feat(image-to-text): explicitly report 4-bit (NF4) quantization status in telemetry and model status

This commit is contained in:
2026-10-29 13:42:54 +05:30
parent 0db0a4c1de
commit 72ea6cd59d
2 changed files with 11 additions and 2 deletions
@@ -149,6 +149,7 @@ class QwenVLModel:
self.processor = None
self.device = None
self.torch_dtype = None
self.effective_quantization = None
def load(self):
"""Load model and processor into memory."""
@@ -196,6 +197,7 @@ class QwenVLModel:
print(f"[*] Device: {self.device} | Total VRAM: {total_vram:.1f} GB | Free: {vram_available:.1f} GB | Quantization: {selected_quant}")
if selected_quant == "4bit":
self.effective_quantization = "4-bit (NF4)"
quant_config = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_compute_dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16,
@@ -204,14 +206,19 @@ class QwenVLModel:
llm_int8_enable_fp32_cpu_offload=True,
)
elif selected_quant == "8bit":
self.effective_quantization = "8-bit"
quant_config = BitsAndBytesConfig(
load_in_8bit=True,
llm_int8_enable_fp32_cpu_offload=True,
)
elif selected_quant in ("bf16", "bfloat16"):
self.effective_quantization = "bf16"
self.torch_dtype = torch.bfloat16
elif selected_quant in ("fp16", "float16"):
self.effective_quantization = "fp16"
self.torch_dtype = torch.float16
else:
self.effective_quantization = "fp32"
# Load Processor
print("[*] Loading processor...")
+4 -2
View File
@@ -109,11 +109,12 @@ class BatchStartRequest(BaseModel):
@app.get("/health")
def health():
is_loaded = _MODEL_INSTANCE is not None and _MODEL_INSTANCE.model is not None
effective_quant = getattr(_MODEL_INSTANCE, "effective_quantization", None) or _CURRENT_QUANT
return {
"status": "ok",
"model_loaded": is_loaded,
"model_id": _CURRENT_MODEL_ID,
"quantization": _CURRENT_QUANT,
"quantization": effective_quant,
"vram": get_vram_info(),
}
@@ -121,9 +122,10 @@ def health():
@app.get("/model/status")
def model_status():
is_loaded = _MODEL_INSTANCE is not None and _MODEL_INSTANCE.model is not None
effective_quant = getattr(_MODEL_INSTANCE, "effective_quantization", None) or _CURRENT_QUANT
return {
"model_id": _CURRENT_MODEL_ID,
"quantization": _CURRENT_QUANT,
"quantization": effective_quant,
"loaded": is_loaded,
"vram": get_vram_info(),
}