feat(image-to-text): switch default model to Qwen2.5-VL-7B-Instruct with 3B cached fallback and cpu offload support

This commit is contained in:
2026-10-29 13:33:46 +05:30
parent 47c35da17e
commit 0db0a4c1de
2 changed files with 46 additions and 12 deletions
+44 -10
View File
@@ -29,7 +29,30 @@ except ImportError:
process_vision_info = None
DEFAULT_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct"
DEFAULT_MODEL_ID = "Qwen/Qwen2.5-VL-7B-Instruct"
FALLBACK_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct"
def is_model_fully_cached(model_id: str) -> bool:
"""Check if all model weights and config files are present in local cache."""
try:
from huggingface_hub import try_to_load_from_cache
import json
f = try_to_load_from_cache(model_id, "config.json")
if not (isinstance(f, str) and Path(f).is_file()):
return False
idx = try_to_load_from_cache(model_id, "model.safetensors.index.json")
if not (isinstance(idx, str) and Path(idx).is_file()):
return False
data = json.loads(Path(idx).read_text())
weight_map = data.get("weight_map", {})
for sf in set(weight_map.values()):
p = try_to_load_from_cache(model_id, sf)
if not (isinstance(p, str) and Path(p).is_file()):
return False
return True
except Exception:
return False
def get_optimal_device() -> torch.device:
@@ -132,7 +155,15 @@ class QwenVLModel:
if self.model is not None and self.processor is not None:
return self
print(f"[*] Loading model from '{self.model_name_or_path}'...")
# Fallback to cached 3B model if 7B is still downloading
effective_model_id = self.model_name_or_path
if effective_model_id == DEFAULT_MODEL_ID and not is_model_fully_cached(effective_model_id):
if is_model_fully_cached(FALLBACK_MODEL_ID):
print(f"[*] '{effective_model_id}' is currently downloading in the background.")
print(f"[*] Seamlessly using cached '{FALLBACK_MODEL_ID}' for now...")
effective_model_id = FALLBACK_MODEL_ID
print(f"[*] Loading model from '{effective_model_id}'...")
# Determine target device
if self.device_str:
@@ -170,9 +201,13 @@ class QwenVLModel:
bnb_4bit_compute_dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16,
bnb_4bit_quant_type="nf4",
bnb_4bit_use_double_quant=True,
llm_int8_enable_fp32_cpu_offload=True,
)
elif selected_quant == "8bit":
quant_config = BitsAndBytesConfig(load_in_8bit=True)
quant_config = BitsAndBytesConfig(
load_in_8bit=True,
llm_int8_enable_fp32_cpu_offload=True,
)
elif selected_quant in ("bf16", "bfloat16"):
self.torch_dtype = torch.bfloat16
elif selected_quant in ("fp16", "float16"):
@@ -181,19 +216,18 @@ class QwenVLModel:
# Load Processor
print("[*] Loading processor...")
self.processor = AutoProcessor.from_pretrained(
self.model_name_or_path,
effective_model_id,
min_pixels=self.min_pixels,
max_pixels=self.max_pixels,
trust_remote_code=True,
use_fast=True,
)
# Inspect config
config = AutoConfig.from_pretrained(self.model_name_or_path, trust_remote_code=True)
config = AutoConfig.from_pretrained(effective_model_id, trust_remote_code=True)
model_type = getattr(config, "model_type", "")
model_kwargs: dict[str, Any] = {
"dtype": self.torch_dtype,
"torch_dtype": self.torch_dtype,
"trust_remote_code": True,
}
@@ -207,18 +241,18 @@ class QwenVLModel:
if model_type == "qwen2_5_vl":
from transformers import Qwen2_5_VLForConditionalGeneration
self.model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
self.model_name_or_path,
effective_model_id,
**model_kwargs,
)
elif model_type == "qwen2_vl":
from transformers import Qwen2VLForConditionalGeneration
self.model = Qwen2VLForConditionalGeneration.from_pretrained(
self.model_name_or_path,
effective_model_id,
**model_kwargs,
)
else:
self.model = AutoModelForVision2Seq.from_pretrained(
self.model_name_or_path,
effective_model_id,
**model_kwargs,
)