feat(image-to-text): add Qwen2.5-VL Vision AI & Prompt Studio with liquid glass UI

This commit is contained in:
2026-10-29 12:39:10 +05:30
parent b74916bd23
commit 457ccadd83
28 changed files with 4978 additions and 1 deletions
@@ -0,0 +1,13 @@
"""
Image to Prompt / Text Generator using Qwen Vision-Language Models.
Supports single image and batch folder processing, multiple prompt modes,
quantization, and CLI / Python API / WebUI interfaces.
"""
__version__ = "1.0.0"
from .model import QwenVLModel
from .processor import ImageProcessor
from .prompts import PROMPT_PRESETS, get_prompt
__all__ = ["QwenVLModel", "ImageProcessor", "PROMPT_PRESETS", "get_prompt"]
+349
View File
@@ -0,0 +1,349 @@
"""
Model loader and inference wrapper for Qwen Vision-Language models.
Supports Qwen2.5-VL and Qwen2-VL families with bf16, fp16, 4-bit and 8-bit quantization.
"""
from __future__ import annotations
import os
import sys
import gc
from pathlib import Path
from typing import Any, Optional, Union
from PIL import Image
import torch
try:
from transformers import (
AutoConfig,
AutoProcessor,
AutoModelForVision2Seq,
BitsAndBytesConfig,
)
except ImportError:
raise ImportError("transformers is required. Install via pip install transformers")
try:
from qwen_vl_utils import process_vision_info
except ImportError:
process_vision_info = None
DEFAULT_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct"
def get_optimal_device() -> torch.device:
"""Return cuda if available, else cpu."""
if torch.cuda.is_available():
return torch.device("cuda")
return torch.device("cpu")
def get_total_vram_gb() -> float:
"""Return total GPU VRAM in GB if cuda is available, else 0."""
if torch.cuda.is_available():
try:
device = torch.cuda.current_device()
return torch.cuda.get_device_properties(device).total_memory / (1024**3)
except Exception:
return 0.0
return 0.0
def get_vram_gb() -> float:
"""Return free VRAM in GB if cuda is available, else 0."""
if torch.cuda.is_available():
try:
device = torch.cuda.current_device()
free_mem, _ = torch.cuda.mem_get_info(device)
return free_mem / (1024**3)
except Exception:
return 0.0
return 0.0
def get_vram_info() -> dict[str, Any]:
"""Return comprehensive GPU and VRAM telemetry."""
if not torch.cuda.is_available():
return {
"cuda_available": False,
"device_name": "CPU",
"total_gb": 0.0,
"free_gb": 0.0,
"used_gb": 0.0,
"percent_used": 0.0,
}
try:
device = torch.cuda.current_device()
device_name = torch.cuda.get_device_name(device)
total_bytes = torch.cuda.get_device_properties(device).total_memory
free_bytes, _ = torch.cuda.mem_get_info(device)
used_bytes = max(0, total_bytes - free_bytes)
total_gb = total_bytes / (1024**3)
free_gb = free_bytes / (1024**3)
used_gb = used_bytes / (1024**3)
percent = (used_gb / total_gb * 100.0) if total_gb > 0 else 0.0
return {
"cuda_available": True,
"device_name": device_name,
"total_gb": round(total_gb, 2),
"free_gb": round(free_gb, 2),
"used_gb": round(used_gb, 2),
"percent_used": round(percent, 1),
}
except Exception as e:
return {
"cuda_available": True,
"device_name": "CUDA Device",
"error": str(e),
"total_gb": 0.0,
"free_gb": 0.0,
"used_gb": 0.0,
"percent_used": 0.0,
}
class QwenVLModel:
"""
Wrapper for loading and running inference with Qwen VL models.
"""
def __init__(
self,
model_name_or_path: str = DEFAULT_MODEL_ID,
quantization: str = "auto", # 'auto', '4bit', '8bit', 'bf16', 'fp16', 'none'
device: str | None = None,
min_pixels: int = 256 * 28 * 28, # ~200k pixels minimum
max_pixels: int = 1024 * 1024, # ~1M pixels maximum (prevents OOM on 8GB GPU)
):
self.model_name_or_path = model_name_or_path
self.quantization = quantization.lower()
self.device_str = device
self.min_pixels = min_pixels
self.max_pixels = max_pixels
self.model = None
self.processor = None
self.device = None
self.torch_dtype = None
def load(self):
"""Load model and processor into memory."""
if self.model is not None and self.processor is not None:
return self
print(f"[*] Loading model from '{self.model_name_or_path}'...")
# Determine target device
if self.device_str:
self.device = torch.device(self.device_str)
else:
self.device = get_optimal_device()
# Decide quantization and dtype
quant_config = None
vram_available = get_vram_gb()
# Default dtype
if self.device.type == "cuda" and torch.cuda.is_bf16_supported():
self.torch_dtype = torch.bfloat16
elif self.device.type == "cuda":
self.torch_dtype = torch.float16
else:
self.torch_dtype = torch.float32
selected_quant = self.quantization
total_vram = get_total_vram_gb()
if selected_quant == "auto":
# On <= 10GB GPUs (e.g. RTX 5060 8GB), use 4bit so 100% of weights stay on GPU
# and inference is fast without CPU offloading
if self.device.type == "cuda" and total_vram <= 10.0:
selected_quant = "4bit"
else:
selected_quant = "bf16" if torch.cuda.is_bf16_supported() else "fp16"
print(f"[*] Device: {self.device} | Total VRAM: {total_vram:.1f} GB | Free: {vram_available:.1f} GB | Quantization: {selected_quant}")
if selected_quant == "4bit":
quant_config = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_compute_dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16,
bnb_4bit_quant_type="nf4",
bnb_4bit_use_double_quant=True,
)
elif selected_quant == "8bit":
quant_config = BitsAndBytesConfig(load_in_8bit=True)
elif selected_quant in ("bf16", "bfloat16"):
self.torch_dtype = torch.bfloat16
elif selected_quant in ("fp16", "float16"):
self.torch_dtype = torch.float16
# Load Processor
print("[*] Loading processor...")
self.processor = AutoProcessor.from_pretrained(
self.model_name_or_path,
min_pixels=self.min_pixels,
max_pixels=self.max_pixels,
trust_remote_code=True,
use_fast=True,
)
# Inspect config
config = AutoConfig.from_pretrained(self.model_name_or_path, trust_remote_code=True)
model_type = getattr(config, "model_type", "")
model_kwargs: dict[str, Any] = {
"dtype": self.torch_dtype,
"trust_remote_code": True,
}
if quant_config is not None:
model_kwargs["quantization_config"] = quant_config
model_kwargs["device_map"] = "auto"
elif self.device.type == "cuda":
model_kwargs["device_map"] = "auto"
print(f"[*] Instantiating model architecture (model_type='{model_type}')...")
if model_type == "qwen2_5_vl":
from transformers import Qwen2_5_VLForConditionalGeneration
self.model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
self.model_name_or_path,
**model_kwargs,
)
elif model_type == "qwen2_vl":
from transformers import Qwen2VLForConditionalGeneration
self.model = Qwen2VLForConditionalGeneration.from_pretrained(
self.model_name_or_path,
**model_kwargs,
)
else:
self.model = AutoModelForVision2Seq.from_pretrained(
self.model_name_or_path,
**model_kwargs,
)
if quant_config is None and "device_map" not in model_kwargs:
self.model = self.model.to(self.device)
self.model.eval()
print("[+] Model loaded successfully and ready for inference!")
return self
def generate(
self,
image_input: Union[str, Path, Image.Image],
prompt: str,
max_new_tokens: int = 512,
temperature: float = 0.2,
top_p: float = 0.9,
) -> str:
"""
Run inference on a single image with a given text prompt.
Args:
image_input: File path (str/Path) or PIL Image object
prompt: Text prompt / instruction for the model
max_new_tokens: Maximum number of output tokens
temperature: Sampling temperature (0.0 for greedy)
top_p: Nucleus sampling parameter
Returns:
Generated text description / prompt.
"""
if self.model is None or self.processor is None:
self.load()
# Resolve image to PIL.Image if file path
if isinstance(image_input, (str, Path)):
path = Path(image_input)
if not path.is_file():
raise FileNotFoundError(f"Image file not found: {path}")
image = Image.open(path).convert("RGB")
elif isinstance(image_input, Image.Image):
image = image_input.convert("RGB")
else:
raise ValueError(f"Unsupported image input type: {type(image_input)}")
# Construct conversational message structure
messages = [
{
"role": "user",
"content": [
{
"type": "image",
"image": image,
"min_pixels": self.min_pixels,
"max_pixels": self.max_pixels,
},
{"type": "text", "text": prompt},
],
}
]
# Prepare chat template
text = self.processor.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
)
# Process vision inputs using qwen_vl_utils if available
if process_vision_info is not None:
image_inputs, video_inputs = process_vision_info(messages)
inputs = self.processor(
text=[text],
images=image_inputs,
videos=video_inputs,
padding=True,
return_tensors="pt",
)
else:
inputs = self.processor(
text=[text],
images=[image],
padding=True,
return_tensors="pt",
)
# Send inputs to model device
model_device = getattr(self.model, "device", self.device)
inputs = inputs.to(model_device)
# Generation config
gen_kwargs: dict[str, Any] = {
"max_new_tokens": max_new_tokens,
"do_sample": temperature > 0.0,
}
if temperature > 0.0:
gen_kwargs["temperature"] = temperature
gen_kwargs["top_p"] = top_p
with torch.inference_mode():
generated_ids = self.model.generate(**inputs, **gen_kwargs)
# Trim input tokens from generated ids
generated_ids_trimmed = [
out_ids[len(in_ids):]
for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
]
output_text = self.processor.batch_decode(
generated_ids_trimmed,
skip_special_tokens=True,
clean_up_tokenization_spaces=False,
)[0]
return output_text.strip()
def unload(self):
"""Free GPU memory by unloading model and clearing cache."""
if self.model is not None:
del self.model
self.model = None
if self.processor is not None:
del self.processor
self.processor = None
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
print("[*] Model unloaded and GPU cache cleared.")
@@ -0,0 +1,297 @@
"""
Batch and single image processing engine.
Handles file discovery, sidecar generation, directory outputs, JSON summaries, and resuming.
"""
from __future__ import annotations
import csv
import json
import time
from pathlib import Path
from typing import Any, Callable, Generator, List, Optional, Union
from PIL import Image
from tqdm import tqdm
from .model import QwenVLModel
from .prompts import get_prompt
SUPPORTED_EXTENSIONS = {
".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tiff", ".tif", ".avif"
}
def find_image_files(
directory: Union[str, Path],
recursive: bool = False,
extensions: set[str] | None = None,
) -> list[Path]:
"""Find all image files in a folder."""
root = Path(directory)
if not root.is_dir():
raise NotADirectoryError(f"Directory not found: {root}")
valid_exts = {ext.lower() if ext.startswith(".") else f".{ext.lower()}"
for ext in (extensions or SUPPORTED_EXTENSIONS)}
pattern = "**/*" if recursive else "*"
image_files: list[Path] = []
for p in root.glob(pattern):
if p.is_file() and p.suffix.lower() in valid_exts:
image_files.append(p)
image_files.sort()
return image_files
class ImageProcessor:
"""
High-level processor that runs single image or batch folder workflows.
"""
def __init__(self, model: QwenVLModel | None = None):
self.model = model or QwenVLModel()
def process_single(
self,
image_path: Union[str, Path, Image.Image],
mode: str = "prompt",
custom_prompt: Optional[str] = None,
max_new_tokens: int = 512,
temperature: float = 0.2,
output_path: Optional[Union[str, Path]] = None,
sidecar: bool = False,
) -> dict[str, Any]:
"""
Process a single image and optionally save output to disk.
Args:
image_path: Path to image file or PIL Image object
mode: Prompt preset mode ('prompt', 'flux', 'detailed', 'short', 'ocr', 'tags', etc.)
custom_prompt: Optional custom prompt to override the preset
max_new_tokens: Maximum tokens to generate
temperature: Sampling temperature
output_path: Optional explicit output file path
sidecar: If True and image_path is a file, write <stem>.txt next to image
Returns:
Dict containing metadata, prompt, generated text, and elapsed seconds.
"""
prompt_text = get_prompt(mode=mode, custom_prompt=custom_prompt)
start_time = time.time()
image_info: dict[str, Any] = {
"source": str(image_path) if isinstance(image_path, (str, Path)) else "PIL.Image",
"mode": mode,
"prompt": prompt_text,
}
# Check image dimensions
if isinstance(image_path, (str, Path)):
p = Path(image_path)
if not p.is_file():
raise FileNotFoundError(f"Image not found: {p}")
with Image.open(p) as img:
image_info["width"], image_info["height"] = img.size
elif isinstance(image_path, Image.Image):
image_info["width"], image_info["height"] = image_path.size
# Run inference
generated_text = self.model.generate(
image_input=image_path,
prompt=prompt_text,
max_new_tokens=max_new_tokens,
temperature=temperature,
)
elapsed = time.time() - start_time
image_info["result"] = generated_text
image_info["elapsed_seconds"] = round(elapsed, 3)
# Handle saving to file
if output_path:
out_file = Path(output_path)
out_file.parent.mkdir(parents=True, exist_ok=True)
out_file.write_text(generated_text, encoding="utf-8")
image_info["saved_to"] = str(out_file)
if sidecar and isinstance(image_path, (str, Path)):
sidecar_file = Path(image_path).with_suffix(".txt")
sidecar_file.write_text(generated_text, encoding="utf-8")
image_info["sidecar_path"] = str(sidecar_file)
return image_info
def process_folder(
self,
folder_path: Union[str, Path],
mode: str = "prompt",
custom_prompt: Optional[str] = None,
recursive: bool = False,
output_dir: Optional[Union[str, Path]] = None,
sidecar: bool = True,
save_json: Optional[Union[str, Path]] = None,
save_csv: Optional[Union[str, Path]] = None,
skip_existing: bool = True,
max_new_tokens: int = 512,
temperature: float = 0.2,
progress_callback: Optional[Callable[[int, int, str], None]] = None,
) -> dict[str, Any]:
"""
Batch process a folder of images.
Args:
folder_path: Directory containing images
mode: Prompt preset mode
custom_prompt: Optional custom prompt
recursive: Whether to scan subdirectories recursively
output_dir: If set, write result .txt files into this directory
sidecar: If True, write result .txt next to each image file
save_json: If set, save aggregated results to this JSON file
save_csv: If set, save aggregated results to this CSV file
skip_existing: If True, skip images that already have output files
max_new_tokens: Maximum tokens per image
temperature: Sampling temperature
progress_callback: Optional callback func(current, total, current_file)
Returns:
Dictionary with batch statistics and list of results.
"""
folder = Path(folder_path)
image_files = find_image_files(folder, recursive=recursive)
total_images = len(image_files)
if total_images == 0:
return {
"total": 0,
"processed": 0,
"skipped": 0,
"failed": 0,
"results": [],
"elapsed_seconds": 0.0,
}
# Ensure output directory exists if requested
out_directory = Path(output_dir) if output_dir else None
if out_directory:
out_directory.mkdir(parents=True, exist_ok=True)
results: list[dict[str, Any]] = []
processed_count = 0
skipped_count = 0
failed_count = 0
start_batch_time = time.time()
print(f"\n[*] Found {total_images} images in '{folder}'")
print(f"[*] Mode: {mode} | Recursive: {recursive} | Skip Existing: {skip_existing}\n")
with tqdm(total=total_images, desc="Processing images", unit="img") as pbar:
for idx, img_path in enumerate(image_files, 1):
pbar.set_postfix_str(img_path.name[:25])
# Determine output target file to check if already done
target_out = None
if out_directory:
if recursive:
rel = img_path.relative_to(folder)
target_out = (out_directory / rel).with_suffix(".txt")
target_out.parent.mkdir(parents=True, exist_ok=True)
else:
target_out = out_directory / f"{img_path.stem}.txt"
elif sidecar:
target_out = img_path.with_suffix(".txt")
if skip_existing and target_out and target_out.is_file() and target_out.stat().st_size > 0:
skipped_count += 1
try:
existing_text = target_out.read_text(encoding="utf-8")
except Exception:
existing_text = ""
results.append({
"file": str(img_path),
"result": existing_text,
"status": "skipped_already_exists",
"output_file": str(target_out),
})
pbar.update(1)
if progress_callback:
progress_callback(idx, total_images, img_path.name)
continue
# Run inference on current image
try:
res = self.process_single(
image_path=img_path,
mode=mode,
custom_prompt=custom_prompt,
max_new_tokens=max_new_tokens,
temperature=temperature,
output_path=target_out,
sidecar=(sidecar and not out_directory),
)
res["file"] = str(img_path)
res["status"] = "success"
results.append(res)
processed_count += 1
except Exception as exc:
failed_count += 1
results.append({
"file": str(img_path),
"status": "failed",
"error": str(exc),
})
print(f"\n[!] Error processing {img_path.name}: {exc}")
pbar.update(1)
if progress_callback:
progress_callback(idx, total_images, img_path.name)
total_elapsed = time.time() - start_batch_time
# Save aggregated JSON
if save_json:
json_file = Path(save_json)
json_file.parent.mkdir(parents=True, exist_ok=True)
with open(json_file, "w", encoding="utf-8") as f:
json.dump(
{
"total": total_images,
"processed": processed_count,
"skipped": skipped_count,
"failed": failed_count,
"mode": mode,
"elapsed_seconds": round(total_elapsed, 2),
"items": results,
},
f,
indent=2,
ensure_ascii=False,
)
print(f"[+] Aggregated JSON saved to: {json_file}")
# Save aggregated CSV
if save_csv:
csv_file = Path(save_csv)
csv_file.parent.mkdir(parents=True, exist_ok=True)
with open(csv_file, "w", newline="", encoding="utf-8") as f:
writer = csv.writer(f)
writer.writerow(["Filename", "Status", "Prompt/Caption", "Elapsed (s)"])
for item in results:
writer.writerow([
item.get("file", ""),
item.get("status", ""),
item.get("result", item.get("error", "")),
item.get("elapsed_seconds", 0.0),
])
print(f"[+] CSV summary saved to: {csv_file}")
print(f"\n[+] Batch complete: {processed_count} processed, {skipped_count} skipped, {failed_count} failed in {total_elapsed:.1f}s")
return {
"total": total_images,
"processed": processed_count,
"skipped": skipped_count,
"failed": failed_count,
"elapsed_seconds": round(total_elapsed, 2),
"results": results,
}
@@ -0,0 +1,61 @@
"""
Preset prompts for different image analysis and prompt generation tasks.
"""
PROMPT_PRESETS = {
"prompt": (
"Analyze this image and write a detailed, high-quality prompt suitable for text-to-image "
"generation (such as Midjourney, Flux, or Stable Diffusion). Describe the subject in detail, "
"the artistic medium and style, composition, camera framing, lighting, color palette, "
"atmosphere, and background details. Output only the prompt without conversational intro or outro."
),
"flux": (
"Generate a highly descriptive prompt for Flux / Midjourney to recreate this exact image. "
"Break down the subject, pose, attire, facial expression, artistic style, camera lens/lighting setup, "
"depth of field, background elements, and overall mood. Output only the prompt description."
),
"detailed": (
"Provide a comprehensive, dense visual description of this image. Describe all visible elements "
"including subjects, positioning, background, textures, lighting, colors, and subtle details. "
"Organize the description clearly."
),
"short": (
"Write a concise, accurate single-sentence caption describing the main subject and action in this image."
),
"ocr": (
"Extract and transcribe all visible text, letters, numbers, signs, labels, watermarks, or logos from this image. "
"If there is no text, reply 'No text detected'."
),
"tags": (
"Analyze this image and generate a comma-separated list of descriptive keywords/tags. "
"Include subject, clothing, colors, setting, art style, mood, and camera angle. "
"Format: tag1, tag2, tag3, ..."
),
"story": (
"Write an imaginative, vivid short paragraph telling the story or setting the narrative mood behind this image."
),
}
DEFAULT_MODE = "prompt"
def get_prompt(mode: str, custom_prompt: str | None = None) -> str:
"""
Get prompt text according to mode or custom user prompt.
Args:
mode: Preset key from PROMPT_PRESETS or 'custom'
custom_prompt: Explicit custom prompt text (overrides preset if provided)
Returns:
The prompt string to pass to the Vision-Language model.
"""
if custom_prompt and custom_prompt.strip():
return custom_prompt.strip()
mode_lower = mode.lower().strip()
if mode_lower in PROMPT_PRESETS:
return PROMPT_PRESETS[mode_lower]
# Fallback to default
return PROMPT_PRESETS[DEFAULT_MODE]
+838
View File
@@ -0,0 +1,838 @@
package main
import (
"archive/zip"
"bytes"
"context"
"encoding/csv"
"encoding/json"
"fmt"
"io"
"log"
"log/slog"
"net"
"net/http"
"os"
"os/exec"
"os/signal"
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
"time"
"unified-dashboard/internal/common"
mymiddleware "unified-dashboard/internal/middleware"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"github.com/go-chi/cors"
"github.com/google/uuid"
)
var (
baseDir string
uploadDir string
historyDir string
exportDir string
historyFile string
workerPort string
workerCmd *exec.Cmd
historyMu sync.RWMutex
httpClient = &http.Client{Timeout: 300 * time.Second}
)
type HistoryItem struct {
ID string `json:"id"`
Timestamp time.Time `json:"timestamp"`
Mode string `json:"mode"`
PromptUsed string `json:"prompt_used"`
Result string `json:"result"`
ElapsedSeconds float64 `json:"elapsed_seconds"`
Width int `json:"width"`
Height int `json:"height"`
ImageFilename string `json:"image_filename"`
ImageURL string `json:"image_url"`
}
func initPaths() {
cwd, _ := os.Getwd()
baseDir = filepath.Join(cwd, "data", "image-to-text")
uploadDir = filepath.Join(baseDir, "uploads")
historyDir = filepath.Join(baseDir, "history")
exportDir = filepath.Join(baseDir, "exports")
historyFile = filepath.Join(historyDir, "history.json")
_ = os.MkdirAll(uploadDir, 0755)
_ = os.MkdirAll(historyDir, 0755)
_ = os.MkdirAll(exportDir, 0755)
}
func getFreePort() (string, error) {
addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
if err != nil {
return "", err
}
l, err := net.ListenTCP("tcp", addr)
if err != nil {
return "", err
}
defer l.Close()
return strconv.Itoa(l.Addr().(*net.TCPAddr).Port), nil
}
func startPythonWorker() {
var err error
workerPort, err = getFreePort()
if err != nil {
log.Fatalf("Failed to allocate free port for Python worker: %v", err)
}
workDir, _ := os.Getwd()
workerScript := filepath.Join(workDir, "cmd", "image-to-text", "worker.py")
pythonBin := "/home/creserver/extras/local_venv/bin/python3"
if _, err := os.Stat(pythonBin); os.IsNotExist(err) {
pythonBin = "python3"
}
workerCmd = exec.Command(pythonBin, workerScript, "--port", workerPort)
workerCmd.Dir = workDir
workerCmd.Stdout = os.Stdout
workerCmd.Stderr = os.Stderr
// Put in independent process group so we can terminate it cleanly
workerCmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
go func() {
slog.Info("Starting Python Vision worker", "port", workerPort, "script", workerScript)
if err := workerCmd.Run(); err != nil {
slog.Warn("Python worker process exited", "error", err)
}
}()
// Wait for worker readiness
workerURL := fmt.Sprintf("http://127.0.0.1:%s/health", workerPort)
ready := false
for i := 0; i < 40; i++ {
resp, err := httpClient.Get(workerURL)
if err == nil && resp.StatusCode == 200 {
resp.Body.Close()
ready = true
slog.Info("Python Vision worker is ready", "port", workerPort)
break
}
if resp != nil {
resp.Body.Close()
}
time.Sleep(500 * time.Millisecond)
}
if !ready {
slog.Error("Python Vision worker failed to start in time", "port", workerPort)
}
}
func stopPythonWorker() {
if workerCmd != nil && workerCmd.Process != nil {
slog.Info("Terminating Python Vision worker...")
pgid, err := syscall.Getpgid(workerCmd.Process.Pid)
if err == nil {
_ = syscall.Kill(-pgid, syscall.SIGTERM)
} else {
_ = workerCmd.Process.Kill()
}
}
}
func writeJSON(w http.ResponseWriter, status int, data any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(data)
}
func readHistory() []HistoryItem {
historyMu.RLock()
defer historyMu.RUnlock()
if _, err := os.Stat(historyFile); os.IsNotExist(err) {
return []HistoryItem{}
}
data, err := os.ReadFile(historyFile)
if err != nil {
return []HistoryItem{}
}
var items []HistoryItem
if err := json.Unmarshal(data, &items); err != nil {
return []HistoryItem{}
}
return items
}
func saveHistoryItem(item HistoryItem) {
historyMu.Lock()
defer historyMu.Unlock()
var items []HistoryItem
if data, err := os.ReadFile(historyFile); err == nil {
_ = json.Unmarshal(data, &items)
}
// Prepend new item (newest first), keep max 100 items
items = append([]HistoryItem{item}, items...)
if len(items) > 100 {
items = items[:100]
}
if data, err := json.MarshalIndent(items, "", " "); err == nil {
_ = os.WriteFile(historyFile, data, 0644)
}
}
// Proxy helper for worker requests
func forwardWorkerRequest(method, path string, body io.Reader, contentType string) (*http.Response, error) {
url := fmt.Sprintf("http://127.0.0.1:%s%s", workerPort, path)
req, err := http.NewRequest(method, url, body)
if err != nil {
return nil, err
}
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
return httpClient.Do(req)
}
func handleHealth(w http.ResponseWriter, r *http.Request) {
resp, err := forwardWorkerRequest("GET", "/health", nil, "")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{
"status": "error",
"worker": "offline",
"error": err.Error(),
})
return
}
defer resp.Body.Close()
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(resp.StatusCode)
_, _ = io.Copy(w, resp.Body)
}
func handleModelStatus(w http.ResponseWriter, r *http.Request) {
resp, err := forwardWorkerRequest("GET", "/model/status", nil, "")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"error": "Worker unavailable: " + err.Error()})
return
}
defer resp.Body.Close()
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(resp.StatusCode)
_, _ = io.Copy(w, resp.Body)
}
func handleModelLoad(w http.ResponseWriter, r *http.Request) {
resp, err := forwardWorkerRequest("POST", "/model/load", r.Body, "application/json")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"error": "Worker unavailable: " + err.Error()})
return
}
defer resp.Body.Close()
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(resp.StatusCode)
_, _ = io.Copy(w, resp.Body)
}
func handleModelUnload(w http.ResponseWriter, r *http.Request) {
resp, err := forwardWorkerRequest("POST", "/model/unload", nil, "")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"error": "Worker unavailable: " + err.Error()})
return
}
defer resp.Body.Close()
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(resp.StatusCode)
_, _ = io.Copy(w, resp.Body)
}
func handlePresets(w http.ResponseWriter, r *http.Request) {
resp, err := forwardWorkerRequest("GET", "/presets", nil, "")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"error": "Worker unavailable: " + err.Error()})
return
}
defer resp.Body.Close()
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(resp.StatusCode)
_, _ = io.Copy(w, resp.Body)
}
func handleGenerate(w http.ResponseWriter, r *http.Request) {
contentType := r.Header.Get("Content-Type")
var imagePath string
var imageFilename string
var imageURL string
var mode = "prompt"
var customPrompt = ""
var maxNewTokens = 512
var temperature = 0.2
var topP = 0.9
if strings.HasPrefix(contentType, "multipart/form-data") {
// Limit to 50MB
if err := r.ParseMultipartForm(50 << 20); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "Failed to parse form: " + err.Error()})
return
}
if m := r.FormValue("mode"); m != "" {
mode = m
}
if cp := r.FormValue("custom_prompt"); cp != "" {
customPrompt = cp
}
if mt := r.FormValue("max_new_tokens"); mt != "" {
if v, err := strconv.Atoi(mt); err == nil {
maxNewTokens = v
}
}
if t := r.FormValue("temperature"); t != "" {
if v, err := strconv.ParseFloat(t, 64); err == nil {
temperature = v
}
}
if tp := r.FormValue("top_p"); tp != "" {
if v, err := strconv.ParseFloat(tp, 64); err == nil {
topP = v
}
}
file, header, err := r.FormFile("image")
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "Image file is required: " + err.Error()})
return
}
defer file.Close()
ext := filepath.Ext(header.Filename)
if ext == "" {
ext = ".jpg"
}
savedName := fmt.Sprintf("%s_%s%s", time.Now().Format("20060102_150405"), uuid.New().String()[:8], ext)
savedPath := filepath.Join(uploadDir, savedName)
out, err := os.Create(savedPath)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "Failed to save file: " + err.Error()})
return
}
defer out.Close()
if _, err := io.Copy(out, file); err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "Failed to write file: " + err.Error()})
return
}
imagePath = savedPath
imageFilename = header.Filename
imageURL = "uploads/" + savedName
} else {
// JSON payload
var reqPayload struct {
ImagePath string `json:"image_path"`
ImageBase64 string `json:"image_base64"`
Mode string `json:"mode"`
CustomPrompt string `json:"custom_prompt"`
MaxNewTokens int `json:"max_new_tokens"`
Temperature float64 `json:"temperature"`
TopP float64 `json:"top_p"`
}
if err := json.NewDecoder(r.Body).Decode(&reqPayload); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "Invalid JSON: " + err.Error()})
return
}
imagePath = reqPayload.ImagePath
if reqPayload.Mode != "" {
mode = reqPayload.Mode
}
customPrompt = reqPayload.CustomPrompt
if reqPayload.MaxNewTokens > 0 {
maxNewTokens = reqPayload.MaxNewTokens
}
if reqPayload.Temperature >= 0 {
temperature = reqPayload.Temperature
}
if reqPayload.TopP > 0 {
topP = reqPayload.TopP
}
// If base64 provided without imagePath, save to uploadDir
if imagePath == "" && reqPayload.ImageBase64 != "" {
b64Data := reqPayload.ImageBase64
if comma := strings.Index(b64Data, ","); comma != -1 {
b64Data = b64Data[comma+1:]
}
rawBytes, err := io.ReadAll(strings.NewReader(b64Data))
if err == nil {
savedName := fmt.Sprintf("%s_%s.png", time.Now().Format("20060102_150405"), uuid.New().String()[:8])
savedPath := filepath.Join(uploadDir, savedName)
_ = os.WriteFile(savedPath, rawBytes, 0644)
imagePath = savedPath
imageFilename = savedName
imageURL = "uploads/" + savedName
}
}
}
// Prepare payload for worker
workerPayload := map[string]any{
"image_path": imagePath,
"mode": mode,
"custom_prompt": customPrompt,
"max_new_tokens": maxNewTokens,
"temperature": temperature,
"top_p": topP,
"save_sidecar": true,
}
payloadBytes, _ := json.Marshal(workerPayload)
resp, err := forwardWorkerRequest("POST", "/generate", bytes.NewReader(payloadBytes), "application/json")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "Inference worker error: " + err.Error()})
return
}
defer resp.Body.Close()
var workerResp struct {
Success bool `json:"success"`
Result string `json:"result"`
ElapsedSeconds float64 `json:"elapsed_seconds"`
Mode string `json:"mode"`
PromptUsed string `json:"prompt_used"`
Width int `json:"width"`
Height int `json:"height"`
SidecarPath string `json:"sidecar_path"`
Detail string `json:"detail"`
}
if err := json.NewDecoder(resp.Body).Decode(&workerResp); err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "Failed to decode worker response: " + err.Error()})
return
}
if !workerResp.Success && workerResp.Detail != "" {
writeJSON(w, resp.StatusCode, map[string]string{"error": workerResp.Detail})
return
}
// Save to history
histItem := HistoryItem{
ID: uuid.New().String(),
Timestamp: time.Now(),
Mode: workerResp.Mode,
PromptUsed: workerResp.PromptUsed,
Result: workerResp.Result,
ElapsedSeconds: workerResp.ElapsedSeconds,
Width: workerResp.Width,
Height: workerResp.Height,
ImageFilename: imageFilename,
ImageURL: imageURL,
}
saveHistoryItem(histItem)
writeJSON(w, http.StatusOK, map[string]any{
"success": true,
"result": workerResp.Result,
"elapsed_seconds": workerResp.ElapsedSeconds,
"mode": workerResp.Mode,
"prompt_used": workerResp.PromptUsed,
"width": workerResp.Width,
"height": workerResp.Height,
"image_url": imageURL,
"image_filename": imageFilename,
"sidecar_path": workerResp.SidecarPath,
})
}
func handleBatch(w http.ResponseWriter, r *http.Request) {
contentType := r.Header.Get("Content-Type")
var folderPath string
var imagePaths []string
var mode = "prompt"
var customPrompt = ""
var recursive = false
var sidecar = true
var skipExisting = true
var maxNewTokens = 512
var temperature = 0.2
if strings.HasPrefix(contentType, "multipart/form-data") {
// Multipart: user uploaded multiple files directly
if err := r.ParseMultipartForm(500 << 20); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "Failed to parse form: " + err.Error()})
return
}
if m := r.FormValue("mode"); m != "" {
mode = m
}
if cp := r.FormValue("custom_prompt"); cp != "" {
customPrompt = cp
}
if r.FormValue("sidecar") == "false" {
sidecar = false
}
if r.FormValue("skip_existing") == "false" {
skipExisting = false
}
if mt := r.FormValue("max_new_tokens"); mt != "" {
if v, err := strconv.Atoi(mt); err == nil {
maxNewTokens = v
}
}
if t := r.FormValue("temperature"); t != "" {
if v, err := strconv.ParseFloat(t, 64); err == nil {
temperature = v
}
}
batchID := uuid.New().String()[:8]
batchUploadDir := filepath.Join(uploadDir, "batch_"+batchID)
_ = os.MkdirAll(batchUploadDir, 0755)
files := r.MultipartForm.File["images"]
if len(files) == 0 {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "No images provided"})
return
}
for _, fileHeader := range files {
f, err := fileHeader.Open()
if err != nil {
continue
}
destPath := filepath.Join(batchUploadDir, fileHeader.Filename)
out, err := os.Create(destPath)
if err == nil {
_, _ = io.Copy(out, f)
out.Close()
imagePaths = append(imagePaths, destPath)
}
f.Close()
}
} else {
// JSON request
var req struct {
FolderPath string `json:"folder_path"`
ImagePaths []string `json:"image_paths"`
Mode string `json:"mode"`
CustomPrompt string `json:"custom_prompt"`
Recursive bool `json:"recursive"`
Sidecar bool `json:"sidecar"`
SkipExisting bool `json:"skip_existing"`
MaxNewTokens int `json:"max_new_tokens"`
Temperature float64 `json:"temperature"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "Invalid JSON: " + err.Error()})
return
}
folderPath = req.FolderPath
imagePaths = req.ImagePaths
if req.Mode != "" {
mode = req.Mode
}
customPrompt = req.CustomPrompt
recursive = req.Recursive
sidecar = req.Sidecar
skipExisting = req.SkipExisting
if req.MaxNewTokens > 0 {
maxNewTokens = req.MaxNewTokens
}
if req.Temperature >= 0 {
temperature = req.Temperature
}
}
workerReq := map[string]any{
"folder_path": folderPath,
"image_paths": imagePaths,
"mode": mode,
"custom_prompt": customPrompt,
"recursive": recursive,
"sidecar": sidecar,
"skip_existing": skipExisting,
"max_new_tokens": maxNewTokens,
"temperature": temperature,
}
payloadBytes, _ := json.Marshal(workerReq)
resp, err := forwardWorkerRequest("POST", "/batch/start", bytes.NewReader(payloadBytes), "application/json")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "Worker error: " + err.Error()})
return
}
defer resp.Body.Close()
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(resp.StatusCode)
_, _ = io.Copy(w, resp.Body)
}
func handleBatchStatus(w http.ResponseWriter, r *http.Request) {
jobID := chi.URLParam(r, "job_id")
if jobID == "" {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "Missing job_id"})
return
}
resp, err := forwardWorkerRequest("GET", "/batch/"+jobID+"/status", nil, "")
if err != nil {
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "Worker error: " + err.Error()})
return
}
defer resp.Body.Close()
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(resp.StatusCode)
_, _ = io.Copy(w, resp.Body)
}
type BatchJobStatusResponse struct {
JobID string `json:"job_id"`
Status string `json:"status"`
Total int `json:"total"`
Processed int `json:"processed"`
Skipped int `json:"skipped"`
Failed int `json:"failed"`
ElapsedSeconds float64 `json:"elapsed_seconds"`
Results []map[string]interface{} `json:"results"`
}
func fetchBatchJobData(jobID string) (*BatchJobStatusResponse, error) {
resp, err := forwardWorkerRequest("GET", "/batch/"+jobID+"/status", nil, "")
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("worker returned status %d", resp.StatusCode)
}
var data BatchJobStatusResponse
if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
return nil, err
}
return &data, nil
}
func handleExportZip(w http.ResponseWriter, r *http.Request) {
jobID := chi.URLParam(r, "job_id")
jobData, err := fetchBatchJobData(jobID)
if err != nil {
writeJSON(w, http.StatusNotFound, map[string]string{"error": "Job not found or inaccessible: " + err.Error()})
return
}
w.Header().Set("Content-Type", "application/zip")
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"dataset_sidecars_%s.zip\"", jobID[:8]))
zipWriter := zip.NewWriter(w)
defer zipWriter.Close()
// 1. Write individual .txt sidecars
for _, item := range jobData.Results {
filename, _ := item["filename"].(string)
if filename == "" {
file, _ := item["file"].(string)
filename = filepath.Base(file)
}
resultText, _ := item["result"].(string)
if resultText == "" {
continue
}
baseName := strings.TrimSuffix(filename, filepath.Ext(filename)) + ".txt"
f, err := zipWriter.Create(baseName)
if err == nil {
_, _ = f.Write([]byte(resultText))
}
}
// 2. Write dataset summary JSON
summaryBytes, err := json.MarshalIndent(jobData, "", " ")
if err == nil {
f, err := zipWriter.Create("dataset_summary.json")
if err == nil {
_, _ = f.Write(summaryBytes)
}
}
}
func handleExportJSON(w http.ResponseWriter, r *http.Request) {
jobID := chi.URLParam(r, "job_id")
jobData, err := fetchBatchJobData(jobID)
if err != nil {
writeJSON(w, http.StatusNotFound, map[string]string{"error": "Job not found or inaccessible: " + err.Error()})
return
}
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"batch_results_%s.json\"", jobID[:8]))
_ = json.NewEncoder(w).Encode(jobData)
}
func handleExportCSV(w http.ResponseWriter, r *http.Request) {
jobID := chi.URLParam(r, "job_id")
jobData, err := fetchBatchJobData(jobID)
if err != nil {
writeJSON(w, http.StatusNotFound, map[string]string{"error": "Job not found or inaccessible: " + err.Error()})
return
}
w.Header().Set("Content-Type", "text/csv")
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"batch_results_%s.csv\"", jobID[:8]))
csvWriter := csv.NewWriter(w)
defer csvWriter.Flush()
_ = csvWriter.Write([]string{"Filename", "Status", "Prompt/Caption/OCR", "Elapsed (s)", "Width", "Height"})
for _, item := range jobData.Results {
fn, _ := item["filename"].(string)
st, _ := item["status"].(string)
res, _ := item["result"].(string)
if res == "" {
res, _ = item["error"].(string)
}
var elStr string
if el, ok := item["elapsed_seconds"].(float64); ok {
elStr = fmt.Sprintf("%.2f", el)
}
var wStr, hStr string
if width, ok := item["width"].(float64); ok {
wStr = strconv.Itoa(int(width))
}
if height, ok := item["height"].(float64); ok {
hStr = strconv.Itoa(int(height))
}
_ = csvWriter.Write([]string{fn, st, res, elStr, wStr, hStr})
}
}
func handleGetHistory(w http.ResponseWriter, r *http.Request) {
items := readHistory()
writeJSON(w, http.StatusOK, map[string]any{
"total": len(items),
"items": items,
})
}
func handleDeleteHistory(w http.ResponseWriter, r *http.Request) {
historyMu.Lock()
defer historyMu.Unlock()
_ = os.WriteFile(historyFile, []byte("[]"), 0644)
writeJSON(w, http.StatusOK, map[string]string{"message": "History cleared"})
}
func main() {
common.InitLogger("image-to-text")
initPaths()
startPythonWorker()
port := os.Getenv("PORT")
if port == "" {
port = "5011"
}
r := chi.NewRouter()
r.Use(middleware.RealIP)
r.Use(middleware.RequestID)
r.Use(mymiddleware.RequestLogger)
r.Use(middleware.Recoverer)
r.Use(cors.Handler(cors.Options{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "X-CSRF-Token"},
}))
if scriptName := os.Getenv("SCRIPT_NAME"); scriptName != "" {
r.Use(common.PrefixMiddleware(scriptName))
}
// Static web assets
workDir, _ := os.Getwd()
webDir := filepath.Join(workDir, "web", "image-to-text")
r.Handle("/assets/*", http.StripPrefix("/assets/", http.FileServer(http.Dir(filepath.Join(webDir, "assets")))))
r.Get("/styles.css", func(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, filepath.Join(webDir, "styles.css"))
})
r.Get("/app.js", func(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, filepath.Join(webDir, "app.js"))
})
r.Get("/", func(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, filepath.Join(webDir, "index.html"))
})
// Uploaded media directory
r.Handle("/uploads/*", http.StripPrefix("/uploads/", http.FileServer(http.Dir(uploadDir))))
// Health and Telemetry API
r.Get("/health", handleHealth)
r.Get("/api/model/status", handleModelStatus)
r.Post("/api/model/load", handleModelLoad)
r.Post("/api/model/unload", handleModelUnload)
r.Get("/api/presets", handlePresets)
// Inference API
r.Post("/api/generate", handleGenerate)
r.Post("/api/batch", handleBatch)
r.Get("/api/batch/{job_id}/status", handleBatchStatus)
r.Get("/api/batch/{job_id}/export-zip", handleExportZip)
r.Head("/api/batch/{job_id}/export-zip", handleExportZip)
r.Get("/api/batch/{job_id}/export-json", handleExportJSON)
r.Head("/api/batch/{job_id}/export-json", handleExportJSON)
r.Get("/api/batch/{job_id}/export-csv", handleExportCSV)
r.Head("/api/batch/{job_id}/export-csv", handleExportCSV)
// History API
r.Get("/api/history", handleGetHistory)
r.Delete("/api/history", handleDeleteHistory)
server := &http.Server{
Addr: ":" + port,
Handler: r,
ReadHeaderTimeout: 30 * time.Second,
ReadTimeout: 300 * time.Second,
WriteTimeout: 0,
IdleTimeout: 120 * time.Second,
}
// Graceful shutdown handling
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
go func() {
s := <-sigChan
slog.Info("Shutdown signal caught", "signal", s)
stopPythonWorker()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = server.Shutdown(ctx)
os.Exit(0)
}()
slog.Info("Image-to-Text Studio starting", "addr", server.Addr)
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
slog.Error("Server error", "error", err)
}
}
+410
View File
@@ -0,0 +1,410 @@
"""
FastAPI Python inference worker for Qwen Vision-Language Prompt & Text Studio.
Managed by Go backend on dynamic or designated port.
"""
from __future__ import annotations
import os
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
os.environ["TF_ENABLE_ONEDNN_OPTS"] = "0"
os.environ["TOKENIZERS_PARALLELISM"] = "false"
import argparse
import base64
import io
import os
import sys
import threading
import time
import uuid
from pathlib import Path
from typing import Any, Dict, List, Optional
from PIL import Image
import uvicorn
from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel, Field
# Ensure current directory is on sys.path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from image_to_prompt.model import QwenVLModel, DEFAULT_MODEL_ID, get_vram_info
from image_to_prompt.processor import ImageProcessor, find_image_files
from image_to_prompt.prompts import PROMPT_PRESETS, get_prompt
app = FastAPI(title="Qwen Image-to-Text Worker", version="1.0.0")
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# Global model state
_CURRENT_MODEL_ID = DEFAULT_MODEL_ID
_CURRENT_QUANT = "auto"
_MODEL_INSTANCE: Optional[QwenVLModel] = None
_PROCESSOR_INSTANCE: Optional[ImageProcessor] = None
_MODEL_LOCK = threading.Lock()
# Batch jobs registry
_BATCH_JOBS: Dict[str, Dict[str, Any]] = {}
_JOBS_LOCK = threading.Lock()
def get_or_create_model(model_name: str = _CURRENT_MODEL_ID, quantization: str = _CURRENT_QUANT) -> QwenVLModel:
global _MODEL_INSTANCE, _PROCESSOR_INSTANCE, _CURRENT_MODEL_ID, _CURRENT_QUANT
with _MODEL_LOCK:
if (_MODEL_INSTANCE is None or
_MODEL_INSTANCE.model_name_or_path != model_name or
_MODEL_INSTANCE.quantization != quantization.lower()):
if _MODEL_INSTANCE is not None:
_MODEL_INSTANCE.unload()
_CURRENT_MODEL_ID = model_name
_CURRENT_QUANT = quantization.lower()
_MODEL_INSTANCE = QwenVLModel(
model_name_or_path=model_name,
quantization=quantization,
)
_PROCESSOR_INSTANCE = ImageProcessor(model=_MODEL_INSTANCE)
return _MODEL_INSTANCE
# --- Request / Response Models ---
class GenerateRequest(BaseModel):
image_path: Optional[str] = None
image_base64: Optional[str] = None
mode: str = "prompt"
custom_prompt: Optional[str] = None
max_new_tokens: int = Field(default=512, ge=16, le=2048)
temperature: float = Field(default=0.2, ge=0.0, le=1.0)
top_p: float = Field(default=0.9, ge=0.1, le=1.0)
save_sidecar: bool = False
class ModelConfigRequest(BaseModel):
model_name: str = DEFAULT_MODEL_ID
quantization: str = "auto"
class BatchStartRequest(BaseModel):
folder_path: Optional[str] = None
image_paths: Optional[List[str]] = None
mode: str = "prompt"
custom_prompt: Optional[str] = None
recursive: bool = False
sidecar: bool = True
skip_existing: bool = True
max_new_tokens: int = 512
temperature: float = 0.2
# --- API Routes ---
@app.get("/health")
def health():
is_loaded = _MODEL_INSTANCE is not None and _MODEL_INSTANCE.model is not None
return {
"status": "ok",
"model_loaded": is_loaded,
"model_id": _CURRENT_MODEL_ID,
"quantization": _CURRENT_QUANT,
"vram": get_vram_info(),
}
@app.get("/model/status")
def model_status():
is_loaded = _MODEL_INSTANCE is not None and _MODEL_INSTANCE.model is not None
return {
"model_id": _CURRENT_MODEL_ID,
"quantization": _CURRENT_QUANT,
"loaded": is_loaded,
"vram": get_vram_info(),
}
@app.post("/model/load")
def load_model(req: ModelConfigRequest):
try:
model = get_or_create_model(req.model_name, req.quantization)
model.load()
return {
"success": True,
"message": f"Model '{req.model_name}' loaded in '{req.quantization}' mode.",
"vram": get_vram_info(),
}
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc))
@app.post("/model/unload")
def unload_model():
global _MODEL_INSTANCE, _PROCESSOR_INSTANCE
with _MODEL_LOCK:
if _MODEL_INSTANCE is not None:
_MODEL_INSTANCE.unload()
_MODEL_INSTANCE = None
_PROCESSOR_INSTANCE = None
return {
"success": True,
"message": "Model unloaded and GPU memory freed.",
"vram": get_vram_info(),
}
@app.get("/presets")
def get_presets():
return {
"default": "prompt",
"presets": PROMPT_PRESETS,
}
@app.post("/generate")
def generate(req: GenerateRequest):
# Resolve input image
image: Optional[Image.Image] = None
img_path: Optional[Path] = None
if req.image_path:
img_path = Path(req.image_path)
if not img_path.is_file():
raise HTTPException(status_code=404, detail=f"Image file not found: {req.image_path}")
try:
image = Image.open(img_path).convert("RGB")
except Exception as exc:
raise HTTPException(status_code=400, detail=f"Failed to read image: {exc}")
elif req.image_base64:
try:
raw_data = req.image_base64
if "," in raw_data:
raw_data = raw_data.split(",", 1)[1]
img_bytes = base64.b64decode(raw_data)
image = Image.open(io.BytesIO(img_bytes)).convert("RGB")
except Exception as exc:
raise HTTPException(status_code=400, detail=f"Failed to decode base64 image: {exc}")
else:
raise HTTPException(status_code=400, detail="Either image_path or image_base64 must be provided")
effective_prompt = get_prompt(req.mode, req.custom_prompt)
model = get_or_create_model()
start_time = time.time()
try:
output_text = model.generate(
image_input=image,
prompt=effective_prompt,
max_new_tokens=req.max_new_tokens,
temperature=req.temperature,
top_p=req.top_p,
)
elapsed = time.time() - start_time
# Save sidecar if requested and image_path is available
sidecar_path = None
if req.save_sidecar and img_path is not None:
sidecar_file = img_path.with_suffix(".txt")
sidecar_file.write_text(output_text, encoding="utf-8")
sidecar_path = str(sidecar_file)
return {
"success": True,
"result": output_text,
"elapsed_seconds": round(elapsed, 3),
"mode": req.mode,
"prompt_used": effective_prompt,
"width": image.width,
"height": image.height,
"sidecar_path": sidecar_path,
}
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Inference error: {exc}")
def _execute_batch_job(job_id: str, req: BatchStartRequest):
with _JOBS_LOCK:
job = _BATCH_JOBS.get(job_id)
if not job:
return
try:
model = get_or_create_model()
# If folder_path provided, find images
image_files: List[Path] = []
if req.folder_path:
p = Path(req.folder_path)
if p.is_dir():
image_files = find_image_files(p, recursive=req.recursive)
elif req.image_paths:
for item in req.image_paths:
ip = Path(item)
if ip.is_file():
image_files.append(ip)
total_files = len(image_files)
with _JOBS_LOCK:
job["total"] = total_files
if total_files == 0:
with _JOBS_LOCK:
job["status"] = "completed"
job["message"] = "No supported image files found"
return
effective_prompt = get_prompt(req.mode, req.custom_prompt)
start_time = time.time()
for idx, img_path in enumerate(image_files, 1):
with _JOBS_LOCK:
job["current_index"] = idx
job["current_file"] = img_path.name
target_sidecar = img_path.with_suffix(".txt") if req.sidecar else None
# Skip existing check
if req.skip_existing and target_sidecar and target_sidecar.is_file() and target_sidecar.stat().st_size > 0:
try:
txt = target_sidecar.read_text(encoding="utf-8")
except Exception:
txt = ""
with _JOBS_LOCK:
job["skipped"] += 1
job["results"].append({
"file": str(img_path),
"filename": img_path.name,
"status": "skipped",
"result": txt,
"sidecar": str(target_sidecar),
"elapsed_seconds": 0.0,
})
continue
# Run inference
t0 = time.time()
try:
img = Image.open(img_path).convert("RGB")
text = model.generate(
image_input=img,
prompt=effective_prompt,
max_new_tokens=req.max_new_tokens,
temperature=req.temperature,
)
dt = round(time.time() - t0, 3)
if req.sidecar and target_sidecar:
target_sidecar.write_text(text, encoding="utf-8")
with _JOBS_LOCK:
job["processed"] += 1
job["results"].append({
"file": str(img_path),
"filename": img_path.name,
"status": "success",
"result": text,
"sidecar": str(target_sidecar) if req.sidecar else None,
"elapsed_seconds": dt,
"width": img.width,
"height": img.height,
})
except Exception as exc:
with _JOBS_LOCK:
job["failed"] += 1
job["results"].append({
"file": str(img_path),
"filename": img_path.name,
"status": "failed",
"error": str(exc),
})
total_elapsed = round(time.time() - start_time, 2)
with _JOBS_LOCK:
job["status"] = "completed"
job["elapsed_seconds"] = total_elapsed
except Exception as exc:
with _JOBS_LOCK:
job["status"] = "failed"
job["error"] = str(exc)
@app.post("/batch/start")
def start_batch(req: BatchStartRequest):
job_id = str(uuid.uuid4())
job_info = {
"job_id": job_id,
"status": "running",
"created_at": time.time(),
"total": 0,
"current_index": 0,
"current_file": "",
"processed": 0,
"skipped": 0,
"failed": 0,
"elapsed_seconds": 0.0,
"mode": req.mode,
"results": [],
}
with _JOBS_LOCK:
_BATCH_JOBS[job_id] = job_info
worker_thread = threading.Thread(
target=_execute_batch_job,
args=(job_id, req),
daemon=True,
)
worker_thread.start()
return {"job_id": job_id, "status": "running"}
@app.get("/batch/{job_id}/status")
def batch_status(job_id: str):
with _JOBS_LOCK:
job = _BATCH_JOBS.get(job_id)
if not job:
raise HTTPException(status_code=404, detail="Batch job not found")
# Compute percentage
total = job.get("total", 0)
curr = job.get("current_index", 0)
percent = round((curr / total * 100.0), 1) if total > 0 else 0.0
return {
"job_id": job["job_id"],
"status": job["status"],
"total": total,
"current_index": curr,
"current_file": job.get("current_file", ""),
"processed": job["processed"],
"skipped": job["skipped"],
"failed": job["failed"],
"percent": percent,
"elapsed_seconds": job.get("elapsed_seconds", 0.0),
"results_count": len(job.get("results", [])),
"results": job.get("results", []),
}
def main():
parser = argparse.ArgumentParser(description="Qwen VL Inference Worker")
parser.add_argument("--port", type=int, default=5012, help="Port to listen on")
parser.add_argument("--host", type=str, default="127.0.0.1", help="Host interface")
parser.add_argument("--preload", action="store_true", help="Preload default model immediately")
args = parser.parse_args()
if args.preload:
print("[*] Preloading model on startup...")
model = get_or_create_model()
model.load()
print(f"[*] Starting Qwen Image-to-Text inference worker on {args.host}:{args.port}")
uvicorn.run(app, host=args.host, port=args.port, log_level="info")
if __name__ == "__main__":
main()