322 lines
14 KiB
Python
322 lines
14 KiB
Python
import os
|
|||
|
|
import cv2
|
||
|
|
import numpy as np
|
||
|
|
import onnxruntime
|
||
|
|
import traceback
|
||
|
|
import math
|
||
|
|
import mediapipe as mp
|
||
|
|
from mediapipe.tasks import python
|
||
|
|
from mediapipe.tasks.python import vision
|
||
|
|
|
||
|
|
class FacialRotationDetector:
|
||
|
|
def __init__(self):
|
||
|
|
# Model Path (Relative path to allow running from anywhere)
|
||
|
|
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
|
|
self.model_path = '/home/creserver/Desktop/Vaishno_workspace/Unified_Dashboard_Suite/facial_rotation_detector/Reports/models/best_re_final.onnx'
|
||
|
|
self.mp_task_path = '/home/creserver/Desktop/FastAPI_BACKEND/ML_MODELS_ENDPOINTS/ml_models/architectures/face_landmark/ai_models/face_landmarker.task'
|
||
|
|
self.session = None
|
||
|
|
self.input_name = None
|
||
|
|
self.input_shape = None
|
||
|
|
self.output_names = None
|
||
|
|
# Sensitivity Settings
|
||
|
|
self.conf_threshold = 0.15 # Extremely sensitive to catch all faces
|
||
|
|
self.iou_threshold = 0.45 # Keep overlap tolerance
|
||
|
|
|
||
|
|
self.load_model()
|
||
|
|
|
||
|
|
def load_model(self):
|
||
|
|
try:
|
||
|
|
if not os.path.exists(self.model_path):
|
||
|
|
print(f"Error: Model not found at {self.model_path}")
|
||
|
|
return
|
||
|
|
|
||
|
|
print(f"Loading ONNX model from: {self.model_path}")
|
||
|
|
|
||
|
|
opt_session = onnxruntime.SessionOptions()
|
||
|
|
opt_session.enable_mem_pattern = False
|
||
|
|
opt_session.enable_cpu_mem_arena = False
|
||
|
|
opt_session.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_DISABLE_ALL
|
||
|
|
|
||
|
|
# Using CPU by default to avoid CUDA complexity if not present, but list both
|
||
|
|
providers = ['CUDAExecutionProvider', 'CPUExecutionProvider']
|
||
|
|
self.session = onnxruntime.InferenceSession(self.model_path, opt_session, providers=providers)
|
||
|
|
|
||
|
|
model_inputs = self.session.get_inputs()
|
||
|
|
self.input_name = model_inputs[0].name
|
||
|
|
self.input_shape = model_inputs[0].shape
|
||
|
|
|
||
|
|
model_outputs = self.session.get_outputs()
|
||
|
|
self.output_names = [o.name for o in model_outputs]
|
||
|
|
|
||
|
|
print("✅ ONNX Model loaded successfully")
|
||
|
|
|
||
|
|
# Initialize Mediapipe
|
||
|
|
try:
|
||
|
|
base_options = python.BaseOptions(model_asset_path=self.mp_task_path)
|
||
|
|
options = vision.FaceLandmarkerOptions(base_options=base_options, num_faces=1)
|
||
|
|
self.mp_detector = vision.FaceLandmarker.create_from_options(options)
|
||
|
|
print("✅ Mediapipe FaceLandmarker loaded successfully")
|
||
|
|
except Exception as e:
|
||
|
|
print(f"Error loading Mediapipe: {str(e)}")
|
||
|
|
self.mp_detector = None
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
print(f"Error loading models: {str(e)}")
|
||
|
|
traceback.print_exc()
|
||
|
|
|
||
|
|
def preprocess_image(self, image):
|
||
|
|
# Resize to model input shape
|
||
|
|
input_height, input_width = self.input_shape[2:]
|
||
|
|
image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
|
||
|
|
resized = cv2.resize(image_rgb, (input_width, input_height))
|
||
|
|
|
||
|
|
# Scale input pixel value to 0 to 1
|
||
|
|
input_image = resized / 255.0
|
||
|
|
input_image = input_image.transpose(2, 0, 1)
|
||
|
|
input_tensor = input_image[np.newaxis, :, :, :].astype(np.float32)
|
||
|
|
|
||
|
|
return input_tensor
|
||
|
|
|
||
|
|
def compute_iou(self, box, boxes):
|
||
|
|
# Compute xmin, ymin, xmax, ymax for both boxes
|
||
|
|
xmin = np.maximum(box[0], boxes[:, 0])
|
||
|
|
ymin = np.maximum(box[1], boxes[:, 1])
|
||
|
|
xmax = np.minimum(box[2], boxes[:, 2])
|
||
|
|
ymax = np.minimum(box[3], boxes[:, 3])
|
||
|
|
|
||
|
|
# Compute intersection area
|
||
|
|
intersection_area = np.maximum(0, xmax - xmin) * np.maximum(0, ymax - ymin)
|
||
|
|
|
||
|
|
# Compute union area
|
||
|
|
box_area = (box[2] - box[0]) * (box[3] - box[1])
|
||
|
|
boxes_area = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
|
||
|
|
union_area = box_area + boxes_area - intersection_area
|
||
|
|
|
||
|
|
# Compute IoU
|
||
|
|
iou = intersection_area / union_area
|
||
|
|
return iou
|
||
|
|
|
||
|
|
def nms(self, boxes, scores):
|
||
|
|
# Sort by score
|
||
|
|
sorted_indices = np.argsort(scores)[::-1]
|
||
|
|
keep_boxes = []
|
||
|
|
while sorted_indices.size > 0:
|
||
|
|
# Pick the last box
|
||
|
|
box_id = sorted_indices[0]
|
||
|
|
keep_boxes.append(box_id)
|
||
|
|
# Compute IoU of the picked box with the rest
|
||
|
|
ious = self.compute_iou(boxes[box_id, :], boxes[sorted_indices[1:], :])
|
||
|
|
# Remove boxes with IoU over the threshold
|
||
|
|
keep_indices = np.where(ious < self.iou_threshold)[0]
|
||
|
|
sorted_indices = sorted_indices[keep_indices + 1]
|
||
|
|
|
||
|
|
return keep_boxes
|
||
|
|
|
||
|
|
def process_image(self, image_path, output_path=None, min_faces=0, max_faces=0, conf_threshold=None):
|
||
|
|
threshold = conf_threshold if conf_threshold is not None else self.conf_threshold
|
||
|
|
try:
|
||
|
|
if self.session is None:
|
||
|
|
self.load_model()
|
||
|
|
|
||
|
|
original_image = cv2.imread(image_path)
|
||
|
|
if original_image is None:
|
||
|
|
raise ValueError("Could not read image")
|
||
|
|
|
||
|
|
image_height, image_width = original_image.shape[:2]
|
||
|
|
|
||
|
|
# Preprocess
|
||
|
|
input_tensor = self.preprocess_image(original_image)
|
||
|
|
|
||
|
|
# Inference
|
||
|
|
outputs = self.session.run(self.output_names, {self.input_name: input_tensor})[0]
|
||
|
|
predictions = np.squeeze(outputs).T
|
||
|
|
|
||
|
|
# Debug Scanning
|
||
|
|
raw_scores = np.max(predictions[:, 4:], axis=1)
|
||
|
|
max_raw_score = np.max(raw_scores) if len(raw_scores) > 0 else 0.0
|
||
|
|
print(f"DEBUG: Processing {os.path.basename(image_path)}")
|
||
|
|
print(f"DEBUG: Conf Threshold: {threshold}")
|
||
|
|
print(f"DEBUG: Max Raw Confidence in Image: {max_raw_score}")
|
||
|
|
|
||
|
|
# Filter by confidence
|
||
|
|
scores = np.max(predictions[:, 4:], axis=1)
|
||
|
|
class_ids = np.argmax(predictions[:, 4:], axis=1)
|
||
|
|
|
||
|
|
boxes = predictions[scores > threshold, :4]
|
||
|
|
class_ids = class_ids[scores > threshold]
|
||
|
|
scores = scores[scores > threshold]
|
||
|
|
|
||
|
|
if len(scores) == 0:
|
||
|
|
print("DEBUG: No faces found above threshold.")
|
||
|
|
if output_path:
|
||
|
|
cv2.imwrite(output_path, original_image)
|
||
|
|
warning_msg = (f"Only 0 face(s) found, but {min_faces} required. "
|
||
|
|
f"Try lowering the threshold or add boxes manually.") if min_faces > 0 else None
|
||
|
|
return {
|
||
|
|
'success': True,
|
||
|
|
'warning': warning_msg,
|
||
|
|
'face_count': 0,
|
||
|
|
'coordinate_string': '',
|
||
|
|
'rotation_string': '',
|
||
|
|
'rectangles': []
|
||
|
|
}
|
||
|
|
|
||
|
|
print(f"DEBUG: Found {len(scores)} candidate boxes above threshold.")
|
||
|
|
|
||
|
|
# Rescale Boxes to Original Image Size
|
||
|
|
input_h, input_w = self.input_shape[2:]
|
||
|
|
|
||
|
|
# Apply scaling
|
||
|
|
boxes = np.divide(boxes, np.array([input_w, input_h, input_w, input_h]))
|
||
|
|
boxes *= np.array([image_width, image_height, image_width, image_height])
|
||
|
|
|
||
|
|
# Convert to xyxy (Top-Left, Bottom-Right)
|
||
|
|
xyxy_boxes = np.copy(boxes)
|
||
|
|
xyxy_boxes[..., 0] = boxes[..., 0] - boxes[..., 2] / 2 # x1
|
||
|
|
xyxy_boxes[..., 1] = boxes[..., 1] - boxes[..., 3] / 2 # y1
|
||
|
|
xyxy_boxes[..., 2] = boxes[..., 0] + boxes[..., 2] / 2 # x2
|
||
|
|
xyxy_boxes[..., 3] = boxes[..., 1] + boxes[..., 3] / 2 # y2
|
||
|
|
|
||
|
|
# NMS
|
||
|
|
print(f"DEBUG: Running NMS on {len(scores)} boxes...")
|
||
|
|
indices = self.nms(xyxy_boxes, scores)
|
||
|
|
|
||
|
|
# Sort indices by score descending (most prominent first)
|
||
|
|
indices = sorted(indices, key=lambda i: scores[i], reverse=True)
|
||
|
|
|
||
|
|
# Apply max_faces limit
|
||
|
|
if max_faces > 0:
|
||
|
|
indices = indices[:max_faces]
|
||
|
|
|
||
|
|
print(f"DEBUG: NMS/Limit returned {len(indices)} indices.")
|
||
|
|
|
||
|
|
# Build warning if below minimum — but still return found faces so user can manually fix
|
||
|
|
warning_msg = None
|
||
|
|
if min_faces > 0 and len(indices) < min_faces:
|
||
|
|
warning_msg = (f"Only {len(indices)} face(s) found, but {min_faces} required. "
|
||
|
|
f"Try lowering the threshold or add boxes manually.")
|
||
|
|
|
||
|
|
coord_strs = []
|
||
|
|
rot_point_strs = []
|
||
|
|
final_results = []
|
||
|
|
|
||
|
|
# Prepare Output Image (RGBA for Transparency)
|
||
|
|
image_rgba = cv2.cvtColor(original_image, cv2.COLOR_BGR2BGRA)
|
||
|
|
|
||
|
|
for i in indices:
|
||
|
|
box = xyxy_boxes[i]
|
||
|
|
score = scores[i]
|
||
|
|
cls_id = class_ids[i]
|
||
|
|
|
||
|
|
x1 = int(max(0, box[0]))
|
||
|
|
y1 = int(max(0, box[1]))
|
||
|
|
x2 = int(min(image_width, box[2]))
|
||
|
|
y2 = int(min(image_height, box[3]))
|
||
|
|
|
||
|
|
# --- CALCULATE ANGLE WITH MEDIAPIPE ---
|
||
|
|
angle_deg = 0.0
|
||
|
|
if hasattr(self, 'mp_detector') and self.mp_detector:
|
||
|
|
try:
|
||
|
|
# Crop with small padding for better landmarks
|
||
|
|
pad_x = int((x2 - x1) * 0.1)
|
||
|
|
pad_y = int((y2 - y1) * 0.1)
|
||
|
|
crop_x1 = max(0, x1 - pad_x)
|
||
|
|
crop_y1 = max(0, y1 - pad_y)
|
||
|
|
crop_x2 = min(image_width, x2 + pad_x)
|
||
|
|
crop_y2 = min(image_height, y2 + pad_y)
|
||
|
|
|
||
|
|
crop = original_image[crop_y1:crop_y2, crop_x1:crop_x2]
|
||
|
|
if crop.size > 0:
|
||
|
|
crop_rgb = cv2.cvtColor(crop, cv2.COLOR_BGR2RGB)
|
||
|
|
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=crop_rgb)
|
||
|
|
res = self.mp_detector.detect(mp_image)
|
||
|
|
|
||
|
|
if res.face_landmarks:
|
||
|
|
lms = res.face_landmarks[0]
|
||
|
|
# Use the vertical axis (upward-downward line of the face)
|
||
|
|
# Landmark 10: Top of the face/forehead
|
||
|
|
# Landmark 152: Bottom of the chin
|
||
|
|
top_point = lms[10]
|
||
|
|
bottom_point = lms[152]
|
||
|
|
|
||
|
|
# Calculate differences in pixels
|
||
|
|
dx = (bottom_point.x - top_point.x) * crop.shape[1]
|
||
|
|
dy = (bottom_point.y - top_point.y) * crop.shape[0]
|
||
|
|
|
||
|
|
# Calculate angle from the Y-axis (vertical)
|
||
|
|
# arctan2(dx, dy) gives 0 when dx=0 (straight vertical)
|
||
|
|
# Reversed sign convention per user request
|
||
|
|
angle_deg = -np.degrees(np.arctan2(dx, dy))
|
||
|
|
except Exception as e:
|
||
|
|
print(f"Mediapipe angle error: {e}")
|
||
|
|
|
||
|
|
# Format: x1:y1,x2:y2
|
||
|
|
coord_str = f"{x1}:{y1},{x2}:{y2}"
|
||
|
|
coord_strs.append(coord_str)
|
||
|
|
|
||
|
|
# Rotation Point: Center of Bottom Edge
|
||
|
|
rot_x = int((x1 + x2) / 2)
|
||
|
|
rot_y = y2
|
||
|
|
rot_str = f"{rot_x}:{rot_y}"
|
||
|
|
rot_point_strs.append(rot_str)
|
||
|
|
|
||
|
|
score_val = float(score)
|
||
|
|
if score_val != score_val: # Check for NaN
|
||
|
|
score_val = 0.0
|
||
|
|
if score_val > 1000: # Check for Inf
|
||
|
|
score_val = 1.0
|
||
|
|
|
||
|
|
final_results.append({
|
||
|
|
'corners': {
|
||
|
|
'top_left': {'x': x1, 'y': y1},
|
||
|
|
'top_right': {'x': x2, 'y': y1},
|
||
|
|
'bottom_right': {'x': x2, 'y': y2},
|
||
|
|
'bottom_left': {'x': x1, 'y': y2}
|
||
|
|
},
|
||
|
|
'rotation_point': {'x': rot_x, 'y': rot_y},
|
||
|
|
'score': score_val,
|
||
|
|
'angle': int(round(angle_deg)),
|
||
|
|
'bbox': [x1, y1, x2, y2] # Keep for reference
|
||
|
|
})
|
||
|
|
|
||
|
|
# Draw Bounding Box (Green, Thickness 2)
|
||
|
|
cv2.rectangle(image_rgba, (x1, y1), (x2, y2), (0, 255, 0, 255), 2)
|
||
|
|
|
||
|
|
# Draw Rotation Point (Red Circle, Radius 4)
|
||
|
|
cv2.circle(image_rgba, (rot_x, rot_y), 4, (0, 0, 255, 255), -1)
|
||
|
|
|
||
|
|
# Add convenience properties for frontend
|
||
|
|
final_results[-1]['x'] = x1
|
||
|
|
final_results[-1]['y'] = y1
|
||
|
|
final_results[-1]['width'] = x2 - x1
|
||
|
|
final_results[-1]['height'] = y2 - y1
|
||
|
|
|
||
|
|
print(f"DEBUG: Final Results: {len(final_results)} faces detected.")
|
||
|
|
|
||
|
|
# Save Output
|
||
|
|
if output_path:
|
||
|
|
cv2.imwrite(output_path, image_rgba)
|
||
|
|
|
||
|
|
return {
|
||
|
|
'success': True,
|
||
|
|
'warning': warning_msg,
|
||
|
|
'face_count': len(final_results),
|
||
|
|
'coordinate_string': ';'.join(coord_strs),
|
||
|
|
'rotation_string': ';'.join(rot_point_strs),
|
||
|
|
'rectangles': final_results
|
||
|
|
}
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
print(f"Processing error: {str(e)}")
|
||
|
|
traceback.print_exc()
|
||
|
|
return {'success': False, 'error': str(e)}
|
||
|
|
|
||
|
|
def xywh2xyxy(self, x):
|
||
|
|
# Convert bounding box (x, y, w, h) to bounding box (x1, y1, x2, y2)
|
||
|
|
y = np.copy(x)
|
||
|
|
y[..., 0] = x[..., 0] - x[..., 2] / 2
|
||
|
|
y[..., 1] = x[..., 1] - x[..., 3] / 2
|
||
|
|
y[..., 2] = x[..., 0] + x[..., 2] / 2
|
||
|
|
y[..., 3] = x[..., 1] + x[..., 3] / 2
|
||
|
|
return y
|