From 72ea6cd59d57319b831b949cb16b07a3acbfd688 Mon Sep 17 00:00:00 2001 From: vshnosngh Date: Thu, 29 Oct 2026 13:42:54 +0530 Subject: [PATCH] feat(image-to-text): explicitly report 4-bit (NF4) quantization status in telemetry and model status --- cmd/image-to-text/image_to_prompt/model.py | 7 +++++++ cmd/image-to-text/worker.py | 6 ++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/cmd/image-to-text/image_to_prompt/model.py b/cmd/image-to-text/image_to_prompt/model.py index 3371655..500cdbe 100644 --- a/cmd/image-to-text/image_to_prompt/model.py +++ b/cmd/image-to-text/image_to_prompt/model.py @@ -149,6 +149,7 @@ class QwenVLModel: self.processor = None self.device = None self.torch_dtype = None + self.effective_quantization = None def load(self): """Load model and processor into memory.""" @@ -196,6 +197,7 @@ class QwenVLModel: print(f"[*] Device: {self.device} | Total VRAM: {total_vram:.1f} GB | Free: {vram_available:.1f} GB | Quantization: {selected_quant}") if selected_quant == "4bit": + self.effective_quantization = "4-bit (NF4)" quant_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16, @@ -204,14 +206,19 @@ class QwenVLModel: llm_int8_enable_fp32_cpu_offload=True, ) elif selected_quant == "8bit": + self.effective_quantization = "8-bit" quant_config = BitsAndBytesConfig( load_in_8bit=True, llm_int8_enable_fp32_cpu_offload=True, ) elif selected_quant in ("bf16", "bfloat16"): + self.effective_quantization = "bf16" self.torch_dtype = torch.bfloat16 elif selected_quant in ("fp16", "float16"): + self.effective_quantization = "fp16" self.torch_dtype = torch.float16 + else: + self.effective_quantization = "fp32" # Load Processor print("[*] Loading processor...") diff --git a/cmd/image-to-text/worker.py b/cmd/image-to-text/worker.py index 4188eeb..220e57d 100644 --- a/cmd/image-to-text/worker.py +++ b/cmd/image-to-text/worker.py @@ -109,11 +109,12 @@ class BatchStartRequest(BaseModel): @app.get("/health") def health(): is_loaded = _MODEL_INSTANCE is not None and _MODEL_INSTANCE.model is not None + effective_quant = getattr(_MODEL_INSTANCE, "effective_quantization", None) or _CURRENT_QUANT return { "status": "ok", "model_loaded": is_loaded, "model_id": _CURRENT_MODEL_ID, - "quantization": _CURRENT_QUANT, + "quantization": effective_quant, "vram": get_vram_info(), } @@ -121,9 +122,10 @@ def health(): @app.get("/model/status") def model_status(): is_loaded = _MODEL_INSTANCE is not None and _MODEL_INSTANCE.model is not None + effective_quant = getattr(_MODEL_INSTANCE, "effective_quantization", None) or _CURRENT_QUANT return { "model_id": _CURRENT_MODEL_ID, - "quantization": _CURRENT_QUANT, + "quantization": effective_quant, "loaded": is_loaded, "vram": get_vram_info(), }