350 lines
12 KiB
Python
350 lines
12 KiB
Python
"""
|
|
Model loader and inference wrapper for Qwen Vision-Language models.
|
|
Supports Qwen2.5-VL and Qwen2-VL families with bf16, fp16, 4-bit and 8-bit quantization.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
import gc
|
|
from pathlib import Path
|
|
from typing import Any, Optional, Union
|
|
from PIL import Image
|
|
import torch
|
|
|
|
try:
|
|
from transformers import (
|
|
AutoConfig,
|
|
AutoProcessor,
|
|
AutoModelForVision2Seq,
|
|
BitsAndBytesConfig,
|
|
)
|
|
except ImportError:
|
|
raise ImportError("transformers is required. Install via pip install transformers")
|
|
|
|
try:
|
|
from qwen_vl_utils import process_vision_info
|
|
except ImportError:
|
|
process_vision_info = None
|
|
|
|
|
|
DEFAULT_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct"
|
|
|
|
|
|
def get_optimal_device() -> torch.device:
|
|
"""Return cuda if available, else cpu."""
|
|
if torch.cuda.is_available():
|
|
return torch.device("cuda")
|
|
return torch.device("cpu")
|
|
|
|
|
|
def get_total_vram_gb() -> float:
|
|
"""Return total GPU VRAM in GB if cuda is available, else 0."""
|
|
if torch.cuda.is_available():
|
|
try:
|
|
device = torch.cuda.current_device()
|
|
return torch.cuda.get_device_properties(device).total_memory / (1024**3)
|
|
except Exception:
|
|
return 0.0
|
|
return 0.0
|
|
|
|
|
|
def get_vram_gb() -> float:
|
|
"""Return free VRAM in GB if cuda is available, else 0."""
|
|
if torch.cuda.is_available():
|
|
try:
|
|
device = torch.cuda.current_device()
|
|
free_mem, _ = torch.cuda.mem_get_info(device)
|
|
return free_mem / (1024**3)
|
|
except Exception:
|
|
return 0.0
|
|
return 0.0
|
|
|
|
|
|
def get_vram_info() -> dict[str, Any]:
|
|
"""Return comprehensive GPU and VRAM telemetry."""
|
|
if not torch.cuda.is_available():
|
|
return {
|
|
"cuda_available": False,
|
|
"device_name": "CPU",
|
|
"total_gb": 0.0,
|
|
"free_gb": 0.0,
|
|
"used_gb": 0.0,
|
|
"percent_used": 0.0,
|
|
}
|
|
try:
|
|
device = torch.cuda.current_device()
|
|
device_name = torch.cuda.get_device_name(device)
|
|
total_bytes = torch.cuda.get_device_properties(device).total_memory
|
|
free_bytes, _ = torch.cuda.mem_get_info(device)
|
|
used_bytes = max(0, total_bytes - free_bytes)
|
|
total_gb = total_bytes / (1024**3)
|
|
free_gb = free_bytes / (1024**3)
|
|
used_gb = used_bytes / (1024**3)
|
|
percent = (used_gb / total_gb * 100.0) if total_gb > 0 else 0.0
|
|
return {
|
|
"cuda_available": True,
|
|
"device_name": device_name,
|
|
"total_gb": round(total_gb, 2),
|
|
"free_gb": round(free_gb, 2),
|
|
"used_gb": round(used_gb, 2),
|
|
"percent_used": round(percent, 1),
|
|
}
|
|
except Exception as e:
|
|
return {
|
|
"cuda_available": True,
|
|
"device_name": "CUDA Device",
|
|
"error": str(e),
|
|
"total_gb": 0.0,
|
|
"free_gb": 0.0,
|
|
"used_gb": 0.0,
|
|
"percent_used": 0.0,
|
|
}
|
|
|
|
|
|
class QwenVLModel:
|
|
"""
|
|
Wrapper for loading and running inference with Qwen VL models.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
model_name_or_path: str = DEFAULT_MODEL_ID,
|
|
quantization: str = "auto", # 'auto', '4bit', '8bit', 'bf16', 'fp16', 'none'
|
|
device: str | None = None,
|
|
min_pixels: int = 256 * 28 * 28, # ~200k pixels minimum
|
|
max_pixels: int = 1024 * 1024, # ~1M pixels maximum (prevents OOM on 8GB GPU)
|
|
):
|
|
self.model_name_or_path = model_name_or_path
|
|
self.quantization = quantization.lower()
|
|
self.device_str = device
|
|
self.min_pixels = min_pixels
|
|
self.max_pixels = max_pixels
|
|
|
|
self.model = None
|
|
self.processor = None
|
|
self.device = None
|
|
self.torch_dtype = None
|
|
|
|
def load(self):
|
|
"""Load model and processor into memory."""
|
|
if self.model is not None and self.processor is not None:
|
|
return self
|
|
|
|
print(f"[*] Loading model from '{self.model_name_or_path}'...")
|
|
|
|
# Determine target device
|
|
if self.device_str:
|
|
self.device = torch.device(self.device_str)
|
|
else:
|
|
self.device = get_optimal_device()
|
|
|
|
# Decide quantization and dtype
|
|
quant_config = None
|
|
vram_available = get_vram_gb()
|
|
|
|
# Default dtype
|
|
if self.device.type == "cuda" and torch.cuda.is_bf16_supported():
|
|
self.torch_dtype = torch.bfloat16
|
|
elif self.device.type == "cuda":
|
|
self.torch_dtype = torch.float16
|
|
else:
|
|
self.torch_dtype = torch.float32
|
|
|
|
selected_quant = self.quantization
|
|
total_vram = get_total_vram_gb()
|
|
if selected_quant == "auto":
|
|
# On <= 10GB GPUs (e.g. RTX 5060 8GB), use 4bit so 100% of weights stay on GPU
|
|
# and inference is fast without CPU offloading
|
|
if self.device.type == "cuda" and total_vram <= 10.0:
|
|
selected_quant = "4bit"
|
|
else:
|
|
selected_quant = "bf16" if torch.cuda.is_bf16_supported() else "fp16"
|
|
|
|
print(f"[*] Device: {self.device} | Total VRAM: {total_vram:.1f} GB | Free: {vram_available:.1f} GB | Quantization: {selected_quant}")
|
|
|
|
if selected_quant == "4bit":
|
|
quant_config = BitsAndBytesConfig(
|
|
load_in_4bit=True,
|
|
bnb_4bit_compute_dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16,
|
|
bnb_4bit_quant_type="nf4",
|
|
bnb_4bit_use_double_quant=True,
|
|
)
|
|
elif selected_quant == "8bit":
|
|
quant_config = BitsAndBytesConfig(load_in_8bit=True)
|
|
elif selected_quant in ("bf16", "bfloat16"):
|
|
self.torch_dtype = torch.bfloat16
|
|
elif selected_quant in ("fp16", "float16"):
|
|
self.torch_dtype = torch.float16
|
|
|
|
# Load Processor
|
|
print("[*] Loading processor...")
|
|
self.processor = AutoProcessor.from_pretrained(
|
|
self.model_name_or_path,
|
|
min_pixels=self.min_pixels,
|
|
max_pixels=self.max_pixels,
|
|
trust_remote_code=True,
|
|
use_fast=True,
|
|
)
|
|
|
|
# Inspect config
|
|
config = AutoConfig.from_pretrained(self.model_name_or_path, trust_remote_code=True)
|
|
model_type = getattr(config, "model_type", "")
|
|
|
|
model_kwargs: dict[str, Any] = {
|
|
"dtype": self.torch_dtype,
|
|
"trust_remote_code": True,
|
|
}
|
|
|
|
if quant_config is not None:
|
|
model_kwargs["quantization_config"] = quant_config
|
|
model_kwargs["device_map"] = "auto"
|
|
elif self.device.type == "cuda":
|
|
model_kwargs["device_map"] = "auto"
|
|
|
|
print(f"[*] Instantiating model architecture (model_type='{model_type}')...")
|
|
if model_type == "qwen2_5_vl":
|
|
from transformers import Qwen2_5_VLForConditionalGeneration
|
|
self.model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
|
|
self.model_name_or_path,
|
|
**model_kwargs,
|
|
)
|
|
elif model_type == "qwen2_vl":
|
|
from transformers import Qwen2VLForConditionalGeneration
|
|
self.model = Qwen2VLForConditionalGeneration.from_pretrained(
|
|
self.model_name_or_path,
|
|
**model_kwargs,
|
|
)
|
|
else:
|
|
self.model = AutoModelForVision2Seq.from_pretrained(
|
|
self.model_name_or_path,
|
|
**model_kwargs,
|
|
)
|
|
|
|
if quant_config is None and "device_map" not in model_kwargs:
|
|
self.model = self.model.to(self.device)
|
|
|
|
self.model.eval()
|
|
print("[+] Model loaded successfully and ready for inference!")
|
|
return self
|
|
|
|
def generate(
|
|
self,
|
|
image_input: Union[str, Path, Image.Image],
|
|
prompt: str,
|
|
max_new_tokens: int = 512,
|
|
temperature: float = 0.2,
|
|
top_p: float = 0.9,
|
|
) -> str:
|
|
"""
|
|
Run inference on a single image with a given text prompt.
|
|
|
|
Args:
|
|
image_input: File path (str/Path) or PIL Image object
|
|
prompt: Text prompt / instruction for the model
|
|
max_new_tokens: Maximum number of output tokens
|
|
temperature: Sampling temperature (0.0 for greedy)
|
|
top_p: Nucleus sampling parameter
|
|
|
|
Returns:
|
|
Generated text description / prompt.
|
|
"""
|
|
if self.model is None or self.processor is None:
|
|
self.load()
|
|
|
|
# Resolve image to PIL.Image if file path
|
|
if isinstance(image_input, (str, Path)):
|
|
path = Path(image_input)
|
|
if not path.is_file():
|
|
raise FileNotFoundError(f"Image file not found: {path}")
|
|
image = Image.open(path).convert("RGB")
|
|
elif isinstance(image_input, Image.Image):
|
|
image = image_input.convert("RGB")
|
|
else:
|
|
raise ValueError(f"Unsupported image input type: {type(image_input)}")
|
|
|
|
# Construct conversational message structure
|
|
messages = [
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{
|
|
"type": "image",
|
|
"image": image,
|
|
"min_pixels": self.min_pixels,
|
|
"max_pixels": self.max_pixels,
|
|
},
|
|
{"type": "text", "text": prompt},
|
|
],
|
|
}
|
|
]
|
|
|
|
# Prepare chat template
|
|
text = self.processor.apply_chat_template(
|
|
messages,
|
|
tokenize=False,
|
|
add_generation_prompt=True,
|
|
)
|
|
|
|
# Process vision inputs using qwen_vl_utils if available
|
|
if process_vision_info is not None:
|
|
image_inputs, video_inputs = process_vision_info(messages)
|
|
inputs = self.processor(
|
|
text=[text],
|
|
images=image_inputs,
|
|
videos=video_inputs,
|
|
padding=True,
|
|
return_tensors="pt",
|
|
)
|
|
else:
|
|
inputs = self.processor(
|
|
text=[text],
|
|
images=[image],
|
|
padding=True,
|
|
return_tensors="pt",
|
|
)
|
|
|
|
# Send inputs to model device
|
|
model_device = getattr(self.model, "device", self.device)
|
|
inputs = inputs.to(model_device)
|
|
|
|
# Generation config
|
|
gen_kwargs: dict[str, Any] = {
|
|
"max_new_tokens": max_new_tokens,
|
|
"do_sample": temperature > 0.0,
|
|
}
|
|
if temperature > 0.0:
|
|
gen_kwargs["temperature"] = temperature
|
|
gen_kwargs["top_p"] = top_p
|
|
|
|
with torch.inference_mode():
|
|
generated_ids = self.model.generate(**inputs, **gen_kwargs)
|
|
|
|
# Trim input tokens from generated ids
|
|
generated_ids_trimmed = [
|
|
out_ids[len(in_ids):]
|
|
for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
|
|
]
|
|
|
|
output_text = self.processor.batch_decode(
|
|
generated_ids_trimmed,
|
|
skip_special_tokens=True,
|
|
clean_up_tokenization_spaces=False,
|
|
)[0]
|
|
|
|
return output_text.strip()
|
|
|
|
def unload(self):
|
|
"""Free GPU memory by unloading model and clearing cache."""
|
|
if self.model is not None:
|
|
del self.model
|
|
self.model = None
|
|
if self.processor is not None:
|
|
del self.processor
|
|
self.processor = None
|
|
gc.collect()
|
|
if torch.cuda.is_available():
|
|
torch.cuda.empty_cache()
|
|
print("[*] Model unloaded and GPU cache cleared.")
|