feat(image-to-text): explicitly report 4-bit (NF4) quantization status in telemetry and model status
This commit is contained in:
@@ -149,6 +149,7 @@ class QwenVLModel:
|
||||
self.processor = None
|
||||
self.device = None
|
||||
self.torch_dtype = None
|
||||
self.effective_quantization = None
|
||||
|
||||
def load(self):
|
||||
"""Load model and processor into memory."""
|
||||
@@ -196,6 +197,7 @@ class QwenVLModel:
|
||||
print(f"[*] Device: {self.device} | Total VRAM: {total_vram:.1f} GB | Free: {vram_available:.1f} GB | Quantization: {selected_quant}")
|
||||
|
||||
if selected_quant == "4bit":
|
||||
self.effective_quantization = "4-bit (NF4)"
|
||||
quant_config = BitsAndBytesConfig(
|
||||
load_in_4bit=True,
|
||||
bnb_4bit_compute_dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16,
|
||||
@@ -204,14 +206,19 @@ class QwenVLModel:
|
||||
llm_int8_enable_fp32_cpu_offload=True,
|
||||
)
|
||||
elif selected_quant == "8bit":
|
||||
self.effective_quantization = "8-bit"
|
||||
quant_config = BitsAndBytesConfig(
|
||||
load_in_8bit=True,
|
||||
llm_int8_enable_fp32_cpu_offload=True,
|
||||
)
|
||||
elif selected_quant in ("bf16", "bfloat16"):
|
||||
self.effective_quantization = "bf16"
|
||||
self.torch_dtype = torch.bfloat16
|
||||
elif selected_quant in ("fp16", "float16"):
|
||||
self.effective_quantization = "fp16"
|
||||
self.torch_dtype = torch.float16
|
||||
else:
|
||||
self.effective_quantization = "fp32"
|
||||
|
||||
# Load Processor
|
||||
print("[*] Loading processor...")
|
||||
|
||||
@@ -109,11 +109,12 @@ class BatchStartRequest(BaseModel):
|
||||
@app.get("/health")
|
||||
def health():
|
||||
is_loaded = _MODEL_INSTANCE is not None and _MODEL_INSTANCE.model is not None
|
||||
effective_quant = getattr(_MODEL_INSTANCE, "effective_quantization", None) or _CURRENT_QUANT
|
||||
return {
|
||||
"status": "ok",
|
||||
"model_loaded": is_loaded,
|
||||
"model_id": _CURRENT_MODEL_ID,
|
||||
"quantization": _CURRENT_QUANT,
|
||||
"quantization": effective_quant,
|
||||
"vram": get_vram_info(),
|
||||
}
|
||||
|
||||
@@ -121,9 +122,10 @@ def health():
|
||||
@app.get("/model/status")
|
||||
def model_status():
|
||||
is_loaded = _MODEL_INSTANCE is not None and _MODEL_INSTANCE.model is not None
|
||||
effective_quant = getattr(_MODEL_INSTANCE, "effective_quantization", None) or _CURRENT_QUANT
|
||||
return {
|
||||
"model_id": _CURRENT_MODEL_ID,
|
||||
"quantization": _CURRENT_QUANT,
|
||||
"quantization": effective_quant,
|
||||
"loaded": is_loaded,
|
||||
"vram": get_vram_info(),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user