diff --git a/cmd/image-to-text/image_to_prompt/model.py b/cmd/image-to-text/image_to_prompt/model.py index dce2199..3371655 100644 --- a/cmd/image-to-text/image_to_prompt/model.py +++ b/cmd/image-to-text/image_to_prompt/model.py @@ -29,7 +29,30 @@ except ImportError: process_vision_info = None -DEFAULT_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct" +DEFAULT_MODEL_ID = "Qwen/Qwen2.5-VL-7B-Instruct" +FALLBACK_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct" + + +def is_model_fully_cached(model_id: str) -> bool: + """Check if all model weights and config files are present in local cache.""" + try: + from huggingface_hub import try_to_load_from_cache + import json + f = try_to_load_from_cache(model_id, "config.json") + if not (isinstance(f, str) and Path(f).is_file()): + return False + idx = try_to_load_from_cache(model_id, "model.safetensors.index.json") + if not (isinstance(idx, str) and Path(idx).is_file()): + return False + data = json.loads(Path(idx).read_text()) + weight_map = data.get("weight_map", {}) + for sf in set(weight_map.values()): + p = try_to_load_from_cache(model_id, sf) + if not (isinstance(p, str) and Path(p).is_file()): + return False + return True + except Exception: + return False def get_optimal_device() -> torch.device: @@ -132,7 +155,15 @@ class QwenVLModel: if self.model is not None and self.processor is not None: return self - print(f"[*] Loading model from '{self.model_name_or_path}'...") + # Fallback to cached 3B model if 7B is still downloading + effective_model_id = self.model_name_or_path + if effective_model_id == DEFAULT_MODEL_ID and not is_model_fully_cached(effective_model_id): + if is_model_fully_cached(FALLBACK_MODEL_ID): + print(f"[*] '{effective_model_id}' is currently downloading in the background.") + print(f"[*] Seamlessly using cached '{FALLBACK_MODEL_ID}' for now...") + effective_model_id = FALLBACK_MODEL_ID + + print(f"[*] Loading model from '{effective_model_id}'...") # Determine target device if self.device_str: @@ -170,9 +201,13 @@ class QwenVLModel: bnb_4bit_compute_dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16, bnb_4bit_quant_type="nf4", bnb_4bit_use_double_quant=True, + llm_int8_enable_fp32_cpu_offload=True, ) elif selected_quant == "8bit": - quant_config = BitsAndBytesConfig(load_in_8bit=True) + quant_config = BitsAndBytesConfig( + load_in_8bit=True, + llm_int8_enable_fp32_cpu_offload=True, + ) elif selected_quant in ("bf16", "bfloat16"): self.torch_dtype = torch.bfloat16 elif selected_quant in ("fp16", "float16"): @@ -181,19 +216,18 @@ class QwenVLModel: # Load Processor print("[*] Loading processor...") self.processor = AutoProcessor.from_pretrained( - self.model_name_or_path, + effective_model_id, min_pixels=self.min_pixels, max_pixels=self.max_pixels, trust_remote_code=True, - use_fast=True, ) # Inspect config - config = AutoConfig.from_pretrained(self.model_name_or_path, trust_remote_code=True) + config = AutoConfig.from_pretrained(effective_model_id, trust_remote_code=True) model_type = getattr(config, "model_type", "") model_kwargs: dict[str, Any] = { - "dtype": self.torch_dtype, + "torch_dtype": self.torch_dtype, "trust_remote_code": True, } @@ -207,18 +241,18 @@ class QwenVLModel: if model_type == "qwen2_5_vl": from transformers import Qwen2_5_VLForConditionalGeneration self.model = Qwen2_5_VLForConditionalGeneration.from_pretrained( - self.model_name_or_path, + effective_model_id, **model_kwargs, ) elif model_type == "qwen2_vl": from transformers import Qwen2VLForConditionalGeneration self.model = Qwen2VLForConditionalGeneration.from_pretrained( - self.model_name_or_path, + effective_model_id, **model_kwargs, ) else: self.model = AutoModelForVision2Seq.from_pretrained( - self.model_name_or_path, + effective_model_id, **model_kwargs, ) diff --git a/web/image-to-text/index.html b/web/image-to-text/index.html index 9546236..910852b 100644 --- a/web/image-to-text/index.html +++ b/web/image-to-text/index.html @@ -62,7 +62,7 @@
- Qwen2.5-VL-3B · 4-bit + Qwen2.5-VL-7B · 4-bit
@@ -599,7 +599,7 @@
Model Architecture
- +