feat(image-to-text): switch default model to Qwen2.5-VL-7B-Instruct with 3B cached fallback and cpu offload support
This commit is contained in:
@@ -29,7 +29,30 @@ except ImportError:
|
||||
process_vision_info = None
|
||||
|
||||
|
||||
DEFAULT_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct"
|
||||
DEFAULT_MODEL_ID = "Qwen/Qwen2.5-VL-7B-Instruct"
|
||||
FALLBACK_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct"
|
||||
|
||||
|
||||
def is_model_fully_cached(model_id: str) -> bool:
|
||||
"""Check if all model weights and config files are present in local cache."""
|
||||
try:
|
||||
from huggingface_hub import try_to_load_from_cache
|
||||
import json
|
||||
f = try_to_load_from_cache(model_id, "config.json")
|
||||
if not (isinstance(f, str) and Path(f).is_file()):
|
||||
return False
|
||||
idx = try_to_load_from_cache(model_id, "model.safetensors.index.json")
|
||||
if not (isinstance(idx, str) and Path(idx).is_file()):
|
||||
return False
|
||||
data = json.loads(Path(idx).read_text())
|
||||
weight_map = data.get("weight_map", {})
|
||||
for sf in set(weight_map.values()):
|
||||
p = try_to_load_from_cache(model_id, sf)
|
||||
if not (isinstance(p, str) and Path(p).is_file()):
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def get_optimal_device() -> torch.device:
|
||||
@@ -132,7 +155,15 @@ class QwenVLModel:
|
||||
if self.model is not None and self.processor is not None:
|
||||
return self
|
||||
|
||||
print(f"[*] Loading model from '{self.model_name_or_path}'...")
|
||||
# Fallback to cached 3B model if 7B is still downloading
|
||||
effective_model_id = self.model_name_or_path
|
||||
if effective_model_id == DEFAULT_MODEL_ID and not is_model_fully_cached(effective_model_id):
|
||||
if is_model_fully_cached(FALLBACK_MODEL_ID):
|
||||
print(f"[*] '{effective_model_id}' is currently downloading in the background.")
|
||||
print(f"[*] Seamlessly using cached '{FALLBACK_MODEL_ID}' for now...")
|
||||
effective_model_id = FALLBACK_MODEL_ID
|
||||
|
||||
print(f"[*] Loading model from '{effective_model_id}'...")
|
||||
|
||||
# Determine target device
|
||||
if self.device_str:
|
||||
@@ -170,9 +201,13 @@ class QwenVLModel:
|
||||
bnb_4bit_compute_dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16,
|
||||
bnb_4bit_quant_type="nf4",
|
||||
bnb_4bit_use_double_quant=True,
|
||||
llm_int8_enable_fp32_cpu_offload=True,
|
||||
)
|
||||
elif selected_quant == "8bit":
|
||||
quant_config = BitsAndBytesConfig(load_in_8bit=True)
|
||||
quant_config = BitsAndBytesConfig(
|
||||
load_in_8bit=True,
|
||||
llm_int8_enable_fp32_cpu_offload=True,
|
||||
)
|
||||
elif selected_quant in ("bf16", "bfloat16"):
|
||||
self.torch_dtype = torch.bfloat16
|
||||
elif selected_quant in ("fp16", "float16"):
|
||||
@@ -181,19 +216,18 @@ class QwenVLModel:
|
||||
# Load Processor
|
||||
print("[*] Loading processor...")
|
||||
self.processor = AutoProcessor.from_pretrained(
|
||||
self.model_name_or_path,
|
||||
effective_model_id,
|
||||
min_pixels=self.min_pixels,
|
||||
max_pixels=self.max_pixels,
|
||||
trust_remote_code=True,
|
||||
use_fast=True,
|
||||
)
|
||||
|
||||
# Inspect config
|
||||
config = AutoConfig.from_pretrained(self.model_name_or_path, trust_remote_code=True)
|
||||
config = AutoConfig.from_pretrained(effective_model_id, trust_remote_code=True)
|
||||
model_type = getattr(config, "model_type", "")
|
||||
|
||||
model_kwargs: dict[str, Any] = {
|
||||
"dtype": self.torch_dtype,
|
||||
"torch_dtype": self.torch_dtype,
|
||||
"trust_remote_code": True,
|
||||
}
|
||||
|
||||
@@ -207,18 +241,18 @@ class QwenVLModel:
|
||||
if model_type == "qwen2_5_vl":
|
||||
from transformers import Qwen2_5_VLForConditionalGeneration
|
||||
self.model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
|
||||
self.model_name_or_path,
|
||||
effective_model_id,
|
||||
**model_kwargs,
|
||||
)
|
||||
elif model_type == "qwen2_vl":
|
||||
from transformers import Qwen2VLForConditionalGeneration
|
||||
self.model = Qwen2VLForConditionalGeneration.from_pretrained(
|
||||
self.model_name_or_path,
|
||||
effective_model_id,
|
||||
**model_kwargs,
|
||||
)
|
||||
else:
|
||||
self.model = AutoModelForVision2Seq.from_pretrained(
|
||||
self.model_name_or_path,
|
||||
effective_model_id,
|
||||
**model_kwargs,
|
||||
)
|
||||
|
||||
|
||||
@@ -62,7 +62,7 @@
|
||||
<!-- Model Status Capsule -->
|
||||
<div class="model-status-capsule glass-pill" id="modelStatusCapsule" title="Click to view model & hardware status">
|
||||
<span class="status-dot online" id="statusDot"></span>
|
||||
<span class="model-name" id="modelNameDisplay">Qwen2.5-VL-3B · 4-bit</span>
|
||||
<span class="model-name" id="modelNameDisplay">Qwen2.5-VL-7B · 4-bit</span>
|
||||
</div>
|
||||
|
||||
<!-- VRAM Meter -->
|
||||
@@ -599,7 +599,7 @@
|
||||
<div class="group-title">Model Architecture</div>
|
||||
<div class="field-item">
|
||||
<label for="modelIdInput" class="field-label">Model Identifier</label>
|
||||
<input type="text" id="modelIdInput" class="glass-input" value="Qwen/Qwen2.5-VL-3B-Instruct">
|
||||
<input type="text" id="modelIdInput" class="glass-input" value="Qwen/Qwen2.5-VL-7B-Instruct">
|
||||
</div>
|
||||
<div class="field-item">
|
||||
<label for="quantSelect" class="field-label">Quantization Mode</label>
|
||||
|
||||
Reference in New Issue
Block a user