Files
Unified_Dashboard_Suite_Go/cmd/image-to-text/image_to_prompt/model.py
T

350 lines
12 KiB
Python

"""
Model loader and inference wrapper for Qwen Vision-Language models.
Supports Qwen2.5-VL and Qwen2-VL families with bf16, fp16, 4-bit and 8-bit quantization.
"""
from __future__ import annotations
import os
import sys
import gc
from pathlib import Path
from typing import Any, Optional, Union
from PIL import Image
import torch
try:
from transformers import (
AutoConfig,
AutoProcessor,
AutoModelForVision2Seq,
BitsAndBytesConfig,
)
except ImportError:
raise ImportError("transformers is required. Install via pip install transformers")
try:
from qwen_vl_utils import process_vision_info
except ImportError:
process_vision_info = None
DEFAULT_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct"
def get_optimal_device() -> torch.device:
"""Return cuda if available, else cpu."""
if torch.cuda.is_available():
return torch.device("cuda")
return torch.device("cpu")
def get_total_vram_gb() -> float:
"""Return total GPU VRAM in GB if cuda is available, else 0."""
if torch.cuda.is_available():
try:
device = torch.cuda.current_device()
return torch.cuda.get_device_properties(device).total_memory / (1024**3)
except Exception:
return 0.0
return 0.0
def get_vram_gb() -> float:
"""Return free VRAM in GB if cuda is available, else 0."""
if torch.cuda.is_available():
try:
device = torch.cuda.current_device()
free_mem, _ = torch.cuda.mem_get_info(device)
return free_mem / (1024**3)
except Exception:
return 0.0
return 0.0
def get_vram_info() -> dict[str, Any]:
"""Return comprehensive GPU and VRAM telemetry."""
if not torch.cuda.is_available():
return {
"cuda_available": False,
"device_name": "CPU",
"total_gb": 0.0,
"free_gb": 0.0,
"used_gb": 0.0,
"percent_used": 0.0,
}
try:
device = torch.cuda.current_device()
device_name = torch.cuda.get_device_name(device)
total_bytes = torch.cuda.get_device_properties(device).total_memory
free_bytes, _ = torch.cuda.mem_get_info(device)
used_bytes = max(0, total_bytes - free_bytes)
total_gb = total_bytes / (1024**3)
free_gb = free_bytes / (1024**3)
used_gb = used_bytes / (1024**3)
percent = (used_gb / total_gb * 100.0) if total_gb > 0 else 0.0
return {
"cuda_available": True,
"device_name": device_name,
"total_gb": round(total_gb, 2),
"free_gb": round(free_gb, 2),
"used_gb": round(used_gb, 2),
"percent_used": round(percent, 1),
}
except Exception as e:
return {
"cuda_available": True,
"device_name": "CUDA Device",
"error": str(e),
"total_gb": 0.0,
"free_gb": 0.0,
"used_gb": 0.0,
"percent_used": 0.0,
}
class QwenVLModel:
"""
Wrapper for loading and running inference with Qwen VL models.
"""
def __init__(
self,
model_name_or_path: str = DEFAULT_MODEL_ID,
quantization: str = "auto", # 'auto', '4bit', '8bit', 'bf16', 'fp16', 'none'
device: str | None = None,
min_pixels: int = 256 * 28 * 28, # ~200k pixels minimum
max_pixels: int = 1024 * 1024, # ~1M pixels maximum (prevents OOM on 8GB GPU)
):
self.model_name_or_path = model_name_or_path
self.quantization = quantization.lower()
self.device_str = device
self.min_pixels = min_pixels
self.max_pixels = max_pixels
self.model = None
self.processor = None
self.device = None
self.torch_dtype = None
def load(self):
"""Load model and processor into memory."""
if self.model is not None and self.processor is not None:
return self
print(f"[*] Loading model from '{self.model_name_or_path}'...")
# Determine target device
if self.device_str:
self.device = torch.device(self.device_str)
else:
self.device = get_optimal_device()
# Decide quantization and dtype
quant_config = None
vram_available = get_vram_gb()
# Default dtype
if self.device.type == "cuda" and torch.cuda.is_bf16_supported():
self.torch_dtype = torch.bfloat16
elif self.device.type == "cuda":
self.torch_dtype = torch.float16
else:
self.torch_dtype = torch.float32
selected_quant = self.quantization
total_vram = get_total_vram_gb()
if selected_quant == "auto":
# On <= 10GB GPUs (e.g. RTX 5060 8GB), use 4bit so 100% of weights stay on GPU
# and inference is fast without CPU offloading
if self.device.type == "cuda" and total_vram <= 10.0:
selected_quant = "4bit"
else:
selected_quant = "bf16" if torch.cuda.is_bf16_supported() else "fp16"
print(f"[*] Device: {self.device} | Total VRAM: {total_vram:.1f} GB | Free: {vram_available:.1f} GB | Quantization: {selected_quant}")
if selected_quant == "4bit":
quant_config = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_compute_dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16,
bnb_4bit_quant_type="nf4",
bnb_4bit_use_double_quant=True,
)
elif selected_quant == "8bit":
quant_config = BitsAndBytesConfig(load_in_8bit=True)
elif selected_quant in ("bf16", "bfloat16"):
self.torch_dtype = torch.bfloat16
elif selected_quant in ("fp16", "float16"):
self.torch_dtype = torch.float16
# Load Processor
print("[*] Loading processor...")
self.processor = AutoProcessor.from_pretrained(
self.model_name_or_path,
min_pixels=self.min_pixels,
max_pixels=self.max_pixels,
trust_remote_code=True,
use_fast=True,
)
# Inspect config
config = AutoConfig.from_pretrained(self.model_name_or_path, trust_remote_code=True)
model_type = getattr(config, "model_type", "")
model_kwargs: dict[str, Any] = {
"dtype": self.torch_dtype,
"trust_remote_code": True,
}
if quant_config is not None:
model_kwargs["quantization_config"] = quant_config
model_kwargs["device_map"] = "auto"
elif self.device.type == "cuda":
model_kwargs["device_map"] = "auto"
print(f"[*] Instantiating model architecture (model_type='{model_type}')...")
if model_type == "qwen2_5_vl":
from transformers import Qwen2_5_VLForConditionalGeneration
self.model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
self.model_name_or_path,
**model_kwargs,
)
elif model_type == "qwen2_vl":
from transformers import Qwen2VLForConditionalGeneration
self.model = Qwen2VLForConditionalGeneration.from_pretrained(
self.model_name_or_path,
**model_kwargs,
)
else:
self.model = AutoModelForVision2Seq.from_pretrained(
self.model_name_or_path,
**model_kwargs,
)
if quant_config is None and "device_map" not in model_kwargs:
self.model = self.model.to(self.device)
self.model.eval()
print("[+] Model loaded successfully and ready for inference!")
return self
def generate(
self,
image_input: Union[str, Path, Image.Image],
prompt: str,
max_new_tokens: int = 512,
temperature: float = 0.2,
top_p: float = 0.9,
) -> str:
"""
Run inference on a single image with a given text prompt.
Args:
image_input: File path (str/Path) or PIL Image object
prompt: Text prompt / instruction for the model
max_new_tokens: Maximum number of output tokens
temperature: Sampling temperature (0.0 for greedy)
top_p: Nucleus sampling parameter
Returns:
Generated text description / prompt.
"""
if self.model is None or self.processor is None:
self.load()
# Resolve image to PIL.Image if file path
if isinstance(image_input, (str, Path)):
path = Path(image_input)
if not path.is_file():
raise FileNotFoundError(f"Image file not found: {path}")
image = Image.open(path).convert("RGB")
elif isinstance(image_input, Image.Image):
image = image_input.convert("RGB")
else:
raise ValueError(f"Unsupported image input type: {type(image_input)}")
# Construct conversational message structure
messages = [
{
"role": "user",
"content": [
{
"type": "image",
"image": image,
"min_pixels": self.min_pixels,
"max_pixels": self.max_pixels,
},
{"type": "text", "text": prompt},
],
}
]
# Prepare chat template
text = self.processor.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
)
# Process vision inputs using qwen_vl_utils if available
if process_vision_info is not None:
image_inputs, video_inputs = process_vision_info(messages)
inputs = self.processor(
text=[text],
images=image_inputs,
videos=video_inputs,
padding=True,
return_tensors="pt",
)
else:
inputs = self.processor(
text=[text],
images=[image],
padding=True,
return_tensors="pt",
)
# Send inputs to model device
model_device = getattr(self.model, "device", self.device)
inputs = inputs.to(model_device)
# Generation config
gen_kwargs: dict[str, Any] = {
"max_new_tokens": max_new_tokens,
"do_sample": temperature > 0.0,
}
if temperature > 0.0:
gen_kwargs["temperature"] = temperature
gen_kwargs["top_p"] = top_p
with torch.inference_mode():
generated_ids = self.model.generate(**inputs, **gen_kwargs)
# Trim input tokens from generated ids
generated_ids_trimmed = [
out_ids[len(in_ids):]
for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
]
output_text = self.processor.batch_decode(
generated_ids_trimmed,
skip_special_tokens=True,
clean_up_tokenization_spaces=False,
)[0]
return output_text.strip()
def unload(self):
"""Free GPU memory by unloading model and clearing cache."""
if self.model is not None:
del self.model
self.model = None
if self.processor is not None:
del self.processor
self.processor = None
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
print("[*] Model unloaded and GPU cache cleared.")