Files
Unified_Dashboard_Suite_Go/cmd/image-to-text/worker.py
T

419 lines
14 KiB
Python
Raw Normal View History

"""
FastAPI Python inference worker for Qwen Vision-Language Prompt & Text Studio.
Managed by Go backend on dynamic or designated port.
"""
from __future__ import annotations
import os
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
os.environ["TF_ENABLE_ONEDNN_OPTS"] = "0"
os.environ["TOKENIZERS_PARALLELISM"] = "false"
import argparse
import base64
import io
import os
import sys
import threading
import time
import uuid
from pathlib import Path
from typing import Any, Dict, List, Optional
from PIL import Image
import uvicorn
from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel, Field
# Ensure current directory is on sys.path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from image_to_prompt.model import QwenVLModel, DEFAULT_MODEL_ID, get_vram_info
from image_to_prompt.processor import ImageProcessor, find_image_files
from image_to_prompt.prompts import PROMPT_PRESETS, get_prompt
app = FastAPI(title="Qwen Image-to-Text Worker", version="1.0.0")
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# Global model state
_CURRENT_MODEL_ID = DEFAULT_MODEL_ID
_CURRENT_QUANT = "auto"
_MODEL_INSTANCE: Optional[QwenVLModel] = None
_PROCESSOR_INSTANCE: Optional[ImageProcessor] = None
_MODEL_LOCK = threading.Lock()
# Batch jobs registry
_BATCH_JOBS: Dict[str, Dict[str, Any]] = {}
_JOBS_LOCK = threading.Lock()
def get_or_create_model(model_name: str = _CURRENT_MODEL_ID, quantization: str = _CURRENT_QUANT) -> QwenVLModel:
global _MODEL_INSTANCE, _PROCESSOR_INSTANCE, _CURRENT_MODEL_ID, _CURRENT_QUANT
with _MODEL_LOCK:
if (_MODEL_INSTANCE is None or
_MODEL_INSTANCE.model_name_or_path != model_name or
_MODEL_INSTANCE.quantization != quantization.lower()):
if _MODEL_INSTANCE is not None:
_MODEL_INSTANCE.unload()
_CURRENT_MODEL_ID = model_name
_CURRENT_QUANT = quantization.lower()
_MODEL_INSTANCE = QwenVLModel(
model_name_or_path=model_name,
quantization=quantization,
)
_PROCESSOR_INSTANCE = ImageProcessor(model=_MODEL_INSTANCE)
return _MODEL_INSTANCE
# --- Request / Response Models ---
class GenerateRequest(BaseModel):
image_path: Optional[str] = None
image_base64: Optional[str] = None
mode: str = "prompt"
custom_prompt: Optional[str] = None
max_new_tokens: int = Field(default=512, ge=16, le=2048)
temperature: float = Field(default=0.2, ge=0.0, le=1.0)
top_p: float = Field(default=0.9, ge=0.1, le=1.0)
save_sidecar: bool = False
class ModelConfigRequest(BaseModel):
model_name: str = DEFAULT_MODEL_ID
quantization: str = "auto"
class BatchStartRequest(BaseModel):
folder_path: Optional[str] = None
image_paths: Optional[List[str]] = None
mode: str = "prompt"
custom_prompt: Optional[str] = None
recursive: bool = False
sidecar: bool = True
skip_existing: bool = True
max_new_tokens: int = 512
temperature: float = 0.2
# --- API Routes ---
@app.get("/health")
def health():
is_loaded = _MODEL_INSTANCE is not None and _MODEL_INSTANCE.model is not None
effective_quant = getattr(_MODEL_INSTANCE, "effective_quantization", None) or _CURRENT_QUANT
return {
"status": "ok",
"model_loaded": is_loaded,
"model_id": _CURRENT_MODEL_ID,
"quantization": effective_quant,
"vram": get_vram_info(),
}
@app.get("/model/status")
def model_status():
is_loaded = _MODEL_INSTANCE is not None and _MODEL_INSTANCE.model is not None
effective_quant = getattr(_MODEL_INSTANCE, "effective_quantization", None) or _CURRENT_QUANT
return {
"model_id": _CURRENT_MODEL_ID,
"quantization": effective_quant,
"loaded": is_loaded,
"vram": get_vram_info(),
}
@app.post("/model/load")
def load_model(req: ModelConfigRequest):
try:
model = get_or_create_model(req.model_name, req.quantization)
model.load()
return {
"success": True,
"message": f"Model '{req.model_name}' loaded in '{req.quantization}' mode.",
"vram": get_vram_info(),
}
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc))
@app.post("/model/unload")
def unload_model():
global _MODEL_INSTANCE, _PROCESSOR_INSTANCE
with _MODEL_LOCK:
if _MODEL_INSTANCE is not None:
_MODEL_INSTANCE.unload()
_MODEL_INSTANCE = None
_PROCESSOR_INSTANCE = None
return {
"success": True,
"message": "Model unloaded and GPU memory freed.",
"vram": get_vram_info(),
}
@app.get("/presets")
def get_presets():
return {
"default": "prompt",
"presets": PROMPT_PRESETS,
}
@app.post("/generate")
def generate(req: GenerateRequest):
# Resolve input image
image: Optional[Image.Image] = None
img_path: Optional[Path] = None
if req.image_path:
img_path = Path(req.image_path)
if not img_path.is_file():
raise HTTPException(status_code=404, detail=f"Image file not found: {req.image_path}")
try:
image = Image.open(img_path).convert("RGB")
if max(image.width, image.height) > 720:
image.thumbnail((720, 720), Image.Resampling.LANCZOS)
except Exception as exc:
raise HTTPException(status_code=400, detail=f"Failed to read image: {exc}")
elif req.image_base64:
try:
raw_data = req.image_base64
if "," in raw_data:
raw_data = raw_data.split(",", 1)[1]
img_bytes = base64.b64decode(raw_data)
image = Image.open(io.BytesIO(img_bytes)).convert("RGB")
if max(image.width, image.height) > 720:
image.thumbnail((720, 720), Image.Resampling.LANCZOS)
except Exception as exc:
raise HTTPException(status_code=400, detail=f"Failed to decode base64 image: {exc}")
else:
raise HTTPException(status_code=400, detail="Either image_path or image_base64 must be provided")
effective_prompt = get_prompt(req.mode, req.custom_prompt)
model = get_or_create_model()
start_time = time.time()
try:
output_text = model.generate(
image_input=image,
prompt=effective_prompt,
max_new_tokens=req.max_new_tokens,
temperature=req.temperature,
top_p=req.top_p,
)
elapsed = time.time() - start_time
# Save sidecar if requested and image_path is available
sidecar_path = None
if req.save_sidecar and img_path is not None:
sidecar_file = img_path.with_suffix(".txt")
sidecar_file.write_text(output_text, encoding="utf-8")
sidecar_path = str(sidecar_file)
return {
"success": True,
"result": output_text,
"elapsed_seconds": round(elapsed, 3),
"mode": req.mode,
"prompt_used": effective_prompt,
"width": image.width,
"height": image.height,
"sidecar_path": sidecar_path,
}
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Inference error: {exc}")
def _execute_batch_job(job_id: str, req: BatchStartRequest):
with _JOBS_LOCK:
job = _BATCH_JOBS.get(job_id)
if not job:
return
try:
model = get_or_create_model()
# If folder_path provided, find images
image_files: List[Path] = []
if req.folder_path:
p = Path(req.folder_path)
if p.is_dir():
image_files = find_image_files(p, recursive=req.recursive)
elif req.image_paths:
for item in req.image_paths:
ip = Path(item)
if ip.is_file():
image_files.append(ip)
total_files = len(image_files)
with _JOBS_LOCK:
job["total"] = total_files
if total_files == 0:
with _JOBS_LOCK:
job["status"] = "completed"
job["message"] = "No supported image files found"
return
effective_prompt = get_prompt(req.mode, req.custom_prompt)
start_time = time.time()
for idx, img_path in enumerate(image_files, 1):
with _JOBS_LOCK:
job["current_index"] = idx
job["current_file"] = img_path.name
target_sidecar = img_path.with_suffix(".txt") if req.sidecar else None
# Skip existing check
if req.skip_existing and target_sidecar and target_sidecar.is_file() and target_sidecar.stat().st_size > 0:
try:
txt = target_sidecar.read_text(encoding="utf-8")
except Exception:
txt = ""
with _JOBS_LOCK:
job["skipped"] += 1
job["results"].append({
"file": str(img_path),
"filename": img_path.name,
"status": "skipped",
"result": txt,
"sidecar": str(target_sidecar),
"elapsed_seconds": 0.0,
})
continue
# Run inference
t0 = time.time()
try:
img = Image.open(img_path).convert("RGB")
if max(img.width, img.height) > 720:
img.thumbnail((720, 720), Image.Resampling.LANCZOS)
text = model.generate(
image_input=img,
prompt=effective_prompt,
max_new_tokens=req.max_new_tokens,
temperature=req.temperature,
)
dt = round(time.time() - t0, 3)
if req.sidecar and target_sidecar:
target_sidecar.write_text(text, encoding="utf-8")
with _JOBS_LOCK:
job["processed"] += 1
job["results"].append({
"file": str(img_path),
"filename": img_path.name,
"status": "success",
"result": text,
"sidecar": str(target_sidecar) if req.sidecar else None,
"elapsed_seconds": dt,
"width": img.width,
"height": img.height,
})
except Exception as exc:
with _JOBS_LOCK:
job["failed"] += 1
job["results"].append({
"file": str(img_path),
"filename": img_path.name,
"status": "failed",
"error": str(exc),
})
total_elapsed = round(time.time() - start_time, 2)
with _JOBS_LOCK:
job["status"] = "completed"
job["elapsed_seconds"] = total_elapsed
except Exception as exc:
with _JOBS_LOCK:
job["status"] = "failed"
job["error"] = str(exc)
@app.post("/batch/start")
def start_batch(req: BatchStartRequest):
job_id = str(uuid.uuid4())
job_info = {
"job_id": job_id,
"status": "running",
"created_at": time.time(),
"total": 0,
"current_index": 0,
"current_file": "",
"processed": 0,
"skipped": 0,
"failed": 0,
"elapsed_seconds": 0.0,
"mode": req.mode,
"results": [],
}
with _JOBS_LOCK:
_BATCH_JOBS[job_id] = job_info
worker_thread = threading.Thread(
target=_execute_batch_job,
args=(job_id, req),
daemon=True,
)
worker_thread.start()
return {"job_id": job_id, "status": "running"}
@app.get("/batch/{job_id}/status")
def batch_status(job_id: str):
with _JOBS_LOCK:
job = _BATCH_JOBS.get(job_id)
if not job:
raise HTTPException(status_code=404, detail="Batch job not found")
# Compute percentage
total = job.get("total", 0)
curr = job.get("current_index", 0)
percent = round((curr / total * 100.0), 1) if total > 0 else 0.0
return {
"job_id": job["job_id"],
"status": job["status"],
"total": total,
"current_index": curr,
"current_file": job.get("current_file", ""),
"processed": job["processed"],
"skipped": job["skipped"],
"failed": job["failed"],
"percent": percent,
"elapsed_seconds": job.get("elapsed_seconds", 0.0),
"results_count": len(job.get("results", [])),
"results": job.get("results", []),
}
def main():
parser = argparse.ArgumentParser(description="Qwen VL Inference Worker")
parser.add_argument("--port", type=int, default=5012, help="Port to listen on")
parser.add_argument("--host", type=str, default="127.0.0.1", help="Host interface")
parser.add_argument("--preload", action="store_true", help="Preload default model immediately")
args = parser.parse_args()
if args.preload:
print("[*] Preloading model on startup...")
model = get_or_create_model()
model.load()
print(f"[*] Starting Qwen Image-to-Text inference worker on {args.host}:{args.port}")
uvicorn.run(app, host=args.host, port=args.port, log_level="info")
if __name__ == "__main__":
main()