""" FastAPI Python inference worker for Qwen Vision-Language Prompt & Text Studio. Managed by Go backend on dynamic or designated port. """ from __future__ import annotations import os os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" os.environ["TF_ENABLE_ONEDNN_OPTS"] = "0" os.environ["TOKENIZERS_PARALLELISM"] = "false" import argparse import base64 import io import os import sys import threading import time import uuid from pathlib import Path from typing import Any, Dict, List, Optional from PIL import Image import uvicorn from fastapi import FastAPI, HTTPException from fastapi.middleware.cors import CORSMiddleware from pydantic import BaseModel, Field # Ensure current directory is on sys.path sys.path.insert(0, str(Path(__file__).resolve().parent)) from image_to_prompt.model import QwenVLModel, DEFAULT_MODEL_ID, get_vram_info from image_to_prompt.processor import ImageProcessor, find_image_files from image_to_prompt.prompts import PROMPT_PRESETS, get_prompt app = FastAPI(title="Qwen Image-to-Text Worker", version="1.0.0") app.add_middleware( CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) # Global model state _CURRENT_MODEL_ID = DEFAULT_MODEL_ID _CURRENT_QUANT = "auto" _MODEL_INSTANCE: Optional[QwenVLModel] = None _PROCESSOR_INSTANCE: Optional[ImageProcessor] = None _MODEL_LOCK = threading.Lock() # Batch jobs registry _BATCH_JOBS: Dict[str, Dict[str, Any]] = {} _JOBS_LOCK = threading.Lock() def get_or_create_model(model_name: str = _CURRENT_MODEL_ID, quantization: str = _CURRENT_QUANT) -> QwenVLModel: global _MODEL_INSTANCE, _PROCESSOR_INSTANCE, _CURRENT_MODEL_ID, _CURRENT_QUANT with _MODEL_LOCK: if (_MODEL_INSTANCE is None or _MODEL_INSTANCE.model_name_or_path != model_name or _MODEL_INSTANCE.quantization != quantization.lower()): if _MODEL_INSTANCE is not None: _MODEL_INSTANCE.unload() _CURRENT_MODEL_ID = model_name _CURRENT_QUANT = quantization.lower() _MODEL_INSTANCE = QwenVLModel( model_name_or_path=model_name, quantization=quantization, ) _PROCESSOR_INSTANCE = ImageProcessor(model=_MODEL_INSTANCE) return _MODEL_INSTANCE # --- Request / Response Models --- class GenerateRequest(BaseModel): image_path: Optional[str] = None image_base64: Optional[str] = None mode: str = "prompt" custom_prompt: Optional[str] = None max_new_tokens: int = Field(default=512, ge=16, le=2048) temperature: float = Field(default=0.2, ge=0.0, le=1.0) top_p: float = Field(default=0.9, ge=0.1, le=1.0) save_sidecar: bool = False class ModelConfigRequest(BaseModel): model_name: str = DEFAULT_MODEL_ID quantization: str = "auto" class BatchStartRequest(BaseModel): folder_path: Optional[str] = None image_paths: Optional[List[str]] = None mode: str = "prompt" custom_prompt: Optional[str] = None recursive: bool = False sidecar: bool = True skip_existing: bool = True max_new_tokens: int = 512 temperature: float = 0.2 # --- API Routes --- @app.get("/health") def health(): is_loaded = _MODEL_INSTANCE is not None and _MODEL_INSTANCE.model is not None return { "status": "ok", "model_loaded": is_loaded, "model_id": _CURRENT_MODEL_ID, "quantization": _CURRENT_QUANT, "vram": get_vram_info(), } @app.get("/model/status") def model_status(): is_loaded = _MODEL_INSTANCE is not None and _MODEL_INSTANCE.model is not None return { "model_id": _CURRENT_MODEL_ID, "quantization": _CURRENT_QUANT, "loaded": is_loaded, "vram": get_vram_info(), } @app.post("/model/load") def load_model(req: ModelConfigRequest): try: model = get_or_create_model(req.model_name, req.quantization) model.load() return { "success": True, "message": f"Model '{req.model_name}' loaded in '{req.quantization}' mode.", "vram": get_vram_info(), } except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) @app.post("/model/unload") def unload_model(): global _MODEL_INSTANCE, _PROCESSOR_INSTANCE with _MODEL_LOCK: if _MODEL_INSTANCE is not None: _MODEL_INSTANCE.unload() _MODEL_INSTANCE = None _PROCESSOR_INSTANCE = None return { "success": True, "message": "Model unloaded and GPU memory freed.", "vram": get_vram_info(), } @app.get("/presets") def get_presets(): return { "default": "prompt", "presets": PROMPT_PRESETS, } @app.post("/generate") def generate(req: GenerateRequest): # Resolve input image image: Optional[Image.Image] = None img_path: Optional[Path] = None if req.image_path: img_path = Path(req.image_path) if not img_path.is_file(): raise HTTPException(status_code=404, detail=f"Image file not found: {req.image_path}") try: image = Image.open(img_path).convert("RGB") except Exception as exc: raise HTTPException(status_code=400, detail=f"Failed to read image: {exc}") elif req.image_base64: try: raw_data = req.image_base64 if "," in raw_data: raw_data = raw_data.split(",", 1)[1] img_bytes = base64.b64decode(raw_data) image = Image.open(io.BytesIO(img_bytes)).convert("RGB") except Exception as exc: raise HTTPException(status_code=400, detail=f"Failed to decode base64 image: {exc}") else: raise HTTPException(status_code=400, detail="Either image_path or image_base64 must be provided") effective_prompt = get_prompt(req.mode, req.custom_prompt) model = get_or_create_model() start_time = time.time() try: output_text = model.generate( image_input=image, prompt=effective_prompt, max_new_tokens=req.max_new_tokens, temperature=req.temperature, top_p=req.top_p, ) elapsed = time.time() - start_time # Save sidecar if requested and image_path is available sidecar_path = None if req.save_sidecar and img_path is not None: sidecar_file = img_path.with_suffix(".txt") sidecar_file.write_text(output_text, encoding="utf-8") sidecar_path = str(sidecar_file) return { "success": True, "result": output_text, "elapsed_seconds": round(elapsed, 3), "mode": req.mode, "prompt_used": effective_prompt, "width": image.width, "height": image.height, "sidecar_path": sidecar_path, } except Exception as exc: raise HTTPException(status_code=500, detail=f"Inference error: {exc}") def _execute_batch_job(job_id: str, req: BatchStartRequest): with _JOBS_LOCK: job = _BATCH_JOBS.get(job_id) if not job: return try: model = get_or_create_model() # If folder_path provided, find images image_files: List[Path] = [] if req.folder_path: p = Path(req.folder_path) if p.is_dir(): image_files = find_image_files(p, recursive=req.recursive) elif req.image_paths: for item in req.image_paths: ip = Path(item) if ip.is_file(): image_files.append(ip) total_files = len(image_files) with _JOBS_LOCK: job["total"] = total_files if total_files == 0: with _JOBS_LOCK: job["status"] = "completed" job["message"] = "No supported image files found" return effective_prompt = get_prompt(req.mode, req.custom_prompt) start_time = time.time() for idx, img_path in enumerate(image_files, 1): with _JOBS_LOCK: job["current_index"] = idx job["current_file"] = img_path.name target_sidecar = img_path.with_suffix(".txt") if req.sidecar else None # Skip existing check if req.skip_existing and target_sidecar and target_sidecar.is_file() and target_sidecar.stat().st_size > 0: try: txt = target_sidecar.read_text(encoding="utf-8") except Exception: txt = "" with _JOBS_LOCK: job["skipped"] += 1 job["results"].append({ "file": str(img_path), "filename": img_path.name, "status": "skipped", "result": txt, "sidecar": str(target_sidecar), "elapsed_seconds": 0.0, }) continue # Run inference t0 = time.time() try: img = Image.open(img_path).convert("RGB") text = model.generate( image_input=img, prompt=effective_prompt, max_new_tokens=req.max_new_tokens, temperature=req.temperature, ) dt = round(time.time() - t0, 3) if req.sidecar and target_sidecar: target_sidecar.write_text(text, encoding="utf-8") with _JOBS_LOCK: job["processed"] += 1 job["results"].append({ "file": str(img_path), "filename": img_path.name, "status": "success", "result": text, "sidecar": str(target_sidecar) if req.sidecar else None, "elapsed_seconds": dt, "width": img.width, "height": img.height, }) except Exception as exc: with _JOBS_LOCK: job["failed"] += 1 job["results"].append({ "file": str(img_path), "filename": img_path.name, "status": "failed", "error": str(exc), }) total_elapsed = round(time.time() - start_time, 2) with _JOBS_LOCK: job["status"] = "completed" job["elapsed_seconds"] = total_elapsed except Exception as exc: with _JOBS_LOCK: job["status"] = "failed" job["error"] = str(exc) @app.post("/batch/start") def start_batch(req: BatchStartRequest): job_id = str(uuid.uuid4()) job_info = { "job_id": job_id, "status": "running", "created_at": time.time(), "total": 0, "current_index": 0, "current_file": "", "processed": 0, "skipped": 0, "failed": 0, "elapsed_seconds": 0.0, "mode": req.mode, "results": [], } with _JOBS_LOCK: _BATCH_JOBS[job_id] = job_info worker_thread = threading.Thread( target=_execute_batch_job, args=(job_id, req), daemon=True, ) worker_thread.start() return {"job_id": job_id, "status": "running"} @app.get("/batch/{job_id}/status") def batch_status(job_id: str): with _JOBS_LOCK: job = _BATCH_JOBS.get(job_id) if not job: raise HTTPException(status_code=404, detail="Batch job not found") # Compute percentage total = job.get("total", 0) curr = job.get("current_index", 0) percent = round((curr / total * 100.0), 1) if total > 0 else 0.0 return { "job_id": job["job_id"], "status": job["status"], "total": total, "current_index": curr, "current_file": job.get("current_file", ""), "processed": job["processed"], "skipped": job["skipped"], "failed": job["failed"], "percent": percent, "elapsed_seconds": job.get("elapsed_seconds", 0.0), "results_count": len(job.get("results", [])), "results": job.get("results", []), } def main(): parser = argparse.ArgumentParser(description="Qwen VL Inference Worker") parser.add_argument("--port", type=int, default=5012, help="Port to listen on") parser.add_argument("--host", type=str, default="127.0.0.1", help="Host interface") parser.add_argument("--preload", action="store_true", help="Preload default model immediately") args = parser.parse_args() if args.preload: print("[*] Preloading model on startup...") model = get_or_create_model() model.load() print(f"[*] Starting Qwen Image-to-Text inference worker on {args.host}:{args.port}") uvicorn.run(app, host=args.host, port=args.port, log_level="info") if __name__ == "__main__": main()