419 lines
14 KiB
Python
419 lines
14 KiB
Python
"""
|
|
FastAPI Python inference worker for Qwen Vision-Language Prompt & Text Studio.
|
|
Managed by Go backend on dynamic or designated port.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
|
|
os.environ["TF_ENABLE_ONEDNN_OPTS"] = "0"
|
|
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
|
|
|
import argparse
|
|
import base64
|
|
import io
|
|
import os
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Optional
|
|
from PIL import Image
|
|
|
|
import uvicorn
|
|
from fastapi import FastAPI, HTTPException
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from pydantic import BaseModel, Field
|
|
|
|
# Ensure current directory is on sys.path
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
|
|
from image_to_prompt.model import QwenVLModel, DEFAULT_MODEL_ID, get_vram_info
|
|
from image_to_prompt.processor import ImageProcessor, find_image_files
|
|
from image_to_prompt.prompts import PROMPT_PRESETS, get_prompt
|
|
|
|
app = FastAPI(title="Qwen Image-to-Text Worker", version="1.0.0")
|
|
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=["*"],
|
|
allow_credentials=True,
|
|
allow_methods=["*"],
|
|
allow_headers=["*"],
|
|
)
|
|
|
|
# Global model state
|
|
_CURRENT_MODEL_ID = DEFAULT_MODEL_ID
|
|
_CURRENT_QUANT = "auto"
|
|
_MODEL_INSTANCE: Optional[QwenVLModel] = None
|
|
_PROCESSOR_INSTANCE: Optional[ImageProcessor] = None
|
|
_MODEL_LOCK = threading.Lock()
|
|
|
|
# Batch jobs registry
|
|
_BATCH_JOBS: Dict[str, Dict[str, Any]] = {}
|
|
_JOBS_LOCK = threading.Lock()
|
|
|
|
|
|
def get_or_create_model(model_name: str = _CURRENT_MODEL_ID, quantization: str = _CURRENT_QUANT) -> QwenVLModel:
|
|
global _MODEL_INSTANCE, _PROCESSOR_INSTANCE, _CURRENT_MODEL_ID, _CURRENT_QUANT
|
|
with _MODEL_LOCK:
|
|
if (_MODEL_INSTANCE is None or
|
|
_MODEL_INSTANCE.model_name_or_path != model_name or
|
|
_MODEL_INSTANCE.quantization != quantization.lower()):
|
|
if _MODEL_INSTANCE is not None:
|
|
_MODEL_INSTANCE.unload()
|
|
_CURRENT_MODEL_ID = model_name
|
|
_CURRENT_QUANT = quantization.lower()
|
|
_MODEL_INSTANCE = QwenVLModel(
|
|
model_name_or_path=model_name,
|
|
quantization=quantization,
|
|
)
|
|
_PROCESSOR_INSTANCE = ImageProcessor(model=_MODEL_INSTANCE)
|
|
return _MODEL_INSTANCE
|
|
|
|
|
|
# --- Request / Response Models ---
|
|
|
|
class GenerateRequest(BaseModel):
|
|
image_path: Optional[str] = None
|
|
image_base64: Optional[str] = None
|
|
mode: str = "prompt"
|
|
custom_prompt: Optional[str] = None
|
|
max_new_tokens: int = Field(default=512, ge=16, le=2048)
|
|
temperature: float = Field(default=0.2, ge=0.0, le=1.0)
|
|
top_p: float = Field(default=0.9, ge=0.1, le=1.0)
|
|
save_sidecar: bool = False
|
|
|
|
|
|
class ModelConfigRequest(BaseModel):
|
|
model_name: str = DEFAULT_MODEL_ID
|
|
quantization: str = "auto"
|
|
|
|
|
|
class BatchStartRequest(BaseModel):
|
|
folder_path: Optional[str] = None
|
|
image_paths: Optional[List[str]] = None
|
|
mode: str = "prompt"
|
|
custom_prompt: Optional[str] = None
|
|
recursive: bool = False
|
|
sidecar: bool = True
|
|
skip_existing: bool = True
|
|
max_new_tokens: int = 512
|
|
temperature: float = 0.2
|
|
|
|
|
|
# --- API Routes ---
|
|
|
|
@app.get("/health")
|
|
def health():
|
|
is_loaded = _MODEL_INSTANCE is not None and _MODEL_INSTANCE.model is not None
|
|
effective_quant = getattr(_MODEL_INSTANCE, "effective_quantization", None) or _CURRENT_QUANT
|
|
return {
|
|
"status": "ok",
|
|
"model_loaded": is_loaded,
|
|
"model_id": _CURRENT_MODEL_ID,
|
|
"quantization": effective_quant,
|
|
"vram": get_vram_info(),
|
|
}
|
|
|
|
|
|
@app.get("/model/status")
|
|
def model_status():
|
|
is_loaded = _MODEL_INSTANCE is not None and _MODEL_INSTANCE.model is not None
|
|
effective_quant = getattr(_MODEL_INSTANCE, "effective_quantization", None) or _CURRENT_QUANT
|
|
return {
|
|
"model_id": _CURRENT_MODEL_ID,
|
|
"quantization": effective_quant,
|
|
"loaded": is_loaded,
|
|
"vram": get_vram_info(),
|
|
}
|
|
|
|
|
|
@app.post("/model/load")
|
|
def load_model(req: ModelConfigRequest):
|
|
try:
|
|
model = get_or_create_model(req.model_name, req.quantization)
|
|
model.load()
|
|
return {
|
|
"success": True,
|
|
"message": f"Model '{req.model_name}' loaded in '{req.quantization}' mode.",
|
|
"vram": get_vram_info(),
|
|
}
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=500, detail=str(exc))
|
|
|
|
|
|
@app.post("/model/unload")
|
|
def unload_model():
|
|
global _MODEL_INSTANCE, _PROCESSOR_INSTANCE
|
|
with _MODEL_LOCK:
|
|
if _MODEL_INSTANCE is not None:
|
|
_MODEL_INSTANCE.unload()
|
|
_MODEL_INSTANCE = None
|
|
_PROCESSOR_INSTANCE = None
|
|
return {
|
|
"success": True,
|
|
"message": "Model unloaded and GPU memory freed.",
|
|
"vram": get_vram_info(),
|
|
}
|
|
|
|
|
|
@app.get("/presets")
|
|
def get_presets():
|
|
return {
|
|
"default": "prompt",
|
|
"presets": PROMPT_PRESETS,
|
|
}
|
|
|
|
|
|
@app.post("/generate")
|
|
def generate(req: GenerateRequest):
|
|
# Resolve input image
|
|
image: Optional[Image.Image] = None
|
|
img_path: Optional[Path] = None
|
|
|
|
if req.image_path:
|
|
img_path = Path(req.image_path)
|
|
if not img_path.is_file():
|
|
raise HTTPException(status_code=404, detail=f"Image file not found: {req.image_path}")
|
|
try:
|
|
image = Image.open(img_path).convert("RGB")
|
|
if max(image.width, image.height) > 720:
|
|
image.thumbnail((720, 720), Image.Resampling.LANCZOS)
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=400, detail=f"Failed to read image: {exc}")
|
|
elif req.image_base64:
|
|
try:
|
|
raw_data = req.image_base64
|
|
if "," in raw_data:
|
|
raw_data = raw_data.split(",", 1)[1]
|
|
img_bytes = base64.b64decode(raw_data)
|
|
image = Image.open(io.BytesIO(img_bytes)).convert("RGB")
|
|
if max(image.width, image.height) > 720:
|
|
image.thumbnail((720, 720), Image.Resampling.LANCZOS)
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=400, detail=f"Failed to decode base64 image: {exc}")
|
|
else:
|
|
raise HTTPException(status_code=400, detail="Either image_path or image_base64 must be provided")
|
|
|
|
effective_prompt = get_prompt(req.mode, req.custom_prompt)
|
|
model = get_or_create_model()
|
|
|
|
start_time = time.time()
|
|
try:
|
|
output_text = model.generate(
|
|
image_input=image,
|
|
prompt=effective_prompt,
|
|
max_new_tokens=req.max_new_tokens,
|
|
temperature=req.temperature,
|
|
top_p=req.top_p,
|
|
)
|
|
elapsed = time.time() - start_time
|
|
|
|
# Save sidecar if requested and image_path is available
|
|
sidecar_path = None
|
|
if req.save_sidecar and img_path is not None:
|
|
sidecar_file = img_path.with_suffix(".txt")
|
|
sidecar_file.write_text(output_text, encoding="utf-8")
|
|
sidecar_path = str(sidecar_file)
|
|
|
|
return {
|
|
"success": True,
|
|
"result": output_text,
|
|
"elapsed_seconds": round(elapsed, 3),
|
|
"mode": req.mode,
|
|
"prompt_used": effective_prompt,
|
|
"width": image.width,
|
|
"height": image.height,
|
|
"sidecar_path": sidecar_path,
|
|
}
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=500, detail=f"Inference error: {exc}")
|
|
|
|
|
|
def _execute_batch_job(job_id: str, req: BatchStartRequest):
|
|
with _JOBS_LOCK:
|
|
job = _BATCH_JOBS.get(job_id)
|
|
if not job:
|
|
return
|
|
|
|
try:
|
|
model = get_or_create_model()
|
|
# If folder_path provided, find images
|
|
image_files: List[Path] = []
|
|
if req.folder_path:
|
|
p = Path(req.folder_path)
|
|
if p.is_dir():
|
|
image_files = find_image_files(p, recursive=req.recursive)
|
|
elif req.image_paths:
|
|
for item in req.image_paths:
|
|
ip = Path(item)
|
|
if ip.is_file():
|
|
image_files.append(ip)
|
|
|
|
total_files = len(image_files)
|
|
with _JOBS_LOCK:
|
|
job["total"] = total_files
|
|
|
|
if total_files == 0:
|
|
with _JOBS_LOCK:
|
|
job["status"] = "completed"
|
|
job["message"] = "No supported image files found"
|
|
return
|
|
|
|
effective_prompt = get_prompt(req.mode, req.custom_prompt)
|
|
start_time = time.time()
|
|
|
|
for idx, img_path in enumerate(image_files, 1):
|
|
with _JOBS_LOCK:
|
|
job["current_index"] = idx
|
|
job["current_file"] = img_path.name
|
|
|
|
target_sidecar = img_path.with_suffix(".txt") if req.sidecar else None
|
|
|
|
# Skip existing check
|
|
if req.skip_existing and target_sidecar and target_sidecar.is_file() and target_sidecar.stat().st_size > 0:
|
|
try:
|
|
txt = target_sidecar.read_text(encoding="utf-8")
|
|
except Exception:
|
|
txt = ""
|
|
with _JOBS_LOCK:
|
|
job["skipped"] += 1
|
|
job["results"].append({
|
|
"file": str(img_path),
|
|
"filename": img_path.name,
|
|
"status": "skipped",
|
|
"result": txt,
|
|
"sidecar": str(target_sidecar),
|
|
"elapsed_seconds": 0.0,
|
|
})
|
|
continue
|
|
|
|
# Run inference
|
|
t0 = time.time()
|
|
try:
|
|
img = Image.open(img_path).convert("RGB")
|
|
if max(img.width, img.height) > 720:
|
|
img.thumbnail((720, 720), Image.Resampling.LANCZOS)
|
|
text = model.generate(
|
|
image_input=img,
|
|
prompt=effective_prompt,
|
|
max_new_tokens=req.max_new_tokens,
|
|
temperature=req.temperature,
|
|
)
|
|
dt = round(time.time() - t0, 3)
|
|
|
|
if req.sidecar and target_sidecar:
|
|
target_sidecar.write_text(text, encoding="utf-8")
|
|
|
|
with _JOBS_LOCK:
|
|
job["processed"] += 1
|
|
job["results"].append({
|
|
"file": str(img_path),
|
|
"filename": img_path.name,
|
|
"status": "success",
|
|
"result": text,
|
|
"sidecar": str(target_sidecar) if req.sidecar else None,
|
|
"elapsed_seconds": dt,
|
|
"width": img.width,
|
|
"height": img.height,
|
|
})
|
|
except Exception as exc:
|
|
with _JOBS_LOCK:
|
|
job["failed"] += 1
|
|
job["results"].append({
|
|
"file": str(img_path),
|
|
"filename": img_path.name,
|
|
"status": "failed",
|
|
"error": str(exc),
|
|
})
|
|
|
|
total_elapsed = round(time.time() - start_time, 2)
|
|
with _JOBS_LOCK:
|
|
job["status"] = "completed"
|
|
job["elapsed_seconds"] = total_elapsed
|
|
|
|
except Exception as exc:
|
|
with _JOBS_LOCK:
|
|
job["status"] = "failed"
|
|
job["error"] = str(exc)
|
|
|
|
|
|
@app.post("/batch/start")
|
|
def start_batch(req: BatchStartRequest):
|
|
job_id = str(uuid.uuid4())
|
|
job_info = {
|
|
"job_id": job_id,
|
|
"status": "running",
|
|
"created_at": time.time(),
|
|
"total": 0,
|
|
"current_index": 0,
|
|
"current_file": "",
|
|
"processed": 0,
|
|
"skipped": 0,
|
|
"failed": 0,
|
|
"elapsed_seconds": 0.0,
|
|
"mode": req.mode,
|
|
"results": [],
|
|
}
|
|
|
|
with _JOBS_LOCK:
|
|
_BATCH_JOBS[job_id] = job_info
|
|
|
|
worker_thread = threading.Thread(
|
|
target=_execute_batch_job,
|
|
args=(job_id, req),
|
|
daemon=True,
|
|
)
|
|
worker_thread.start()
|
|
|
|
return {"job_id": job_id, "status": "running"}
|
|
|
|
|
|
@app.get("/batch/{job_id}/status")
|
|
def batch_status(job_id: str):
|
|
with _JOBS_LOCK:
|
|
job = _BATCH_JOBS.get(job_id)
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="Batch job not found")
|
|
# Compute percentage
|
|
total = job.get("total", 0)
|
|
curr = job.get("current_index", 0)
|
|
percent = round((curr / total * 100.0), 1) if total > 0 else 0.0
|
|
return {
|
|
"job_id": job["job_id"],
|
|
"status": job["status"],
|
|
"total": total,
|
|
"current_index": curr,
|
|
"current_file": job.get("current_file", ""),
|
|
"processed": job["processed"],
|
|
"skipped": job["skipped"],
|
|
"failed": job["failed"],
|
|
"percent": percent,
|
|
"elapsed_seconds": job.get("elapsed_seconds", 0.0),
|
|
"results_count": len(job.get("results", [])),
|
|
"results": job.get("results", []),
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Qwen VL Inference Worker")
|
|
parser.add_argument("--port", type=int, default=5012, help="Port to listen on")
|
|
parser.add_argument("--host", type=str, default="127.0.0.1", help="Host interface")
|
|
parser.add_argument("--preload", action="store_true", help="Preload default model immediately")
|
|
args = parser.parse_args()
|
|
|
|
if args.preload:
|
|
print("[*] Preloading model on startup...")
|
|
model = get_or_create_model()
|
|
model.load()
|
|
|
|
print(f"[*] Starting Qwen Image-to-Text inference worker on {args.host}:{args.port}")
|
|
uvicorn.run(app, host=args.host, port=args.port, log_level="info")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|