389 lines
11 KiB
Go
389 lines
11 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
type JobStatus string
|
|
|
|
const (
|
|
JobStatusQueued JobStatus = "QUEUED"
|
|
JobStatusProcessing JobStatus = "PROCESSING"
|
|
JobStatusCompleted JobStatus = "COMPLETED"
|
|
JobStatusFailed JobStatus = "FAILED"
|
|
JobStatusTimedOut JobStatus = "TIMED_OUT"
|
|
JobStatusCancelled JobStatus = "CANCELLED"
|
|
)
|
|
|
|
type BackgroundJob struct {
|
|
JobID string `json:"job_id"`
|
|
SessionID string `json:"session_id"`
|
|
Operation string `json:"operation"`
|
|
Params map[string]interface{} `json:"params"`
|
|
Status JobStatus `json:"status"`
|
|
TimeLimitSec int `json:"time_limit_sec"`
|
|
CreatedAt time.Time `json:"created_at"`
|
|
StartedAt time.Time `json:"started_at"`
|
|
FinishedAt time.Time `json:"finished_at"`
|
|
ElapsedSec float64 `json:"elapsed_sec"`
|
|
RemainingSec float64 `json:"remaining_sec"`
|
|
ProgressPct int `json:"progress_pct"`
|
|
TotalFiles int `json:"total_files"`
|
|
CompletedFiles int `json:"completed_files"`
|
|
ErrorCount int `json:"error_count"`
|
|
Errors []string `json:"errors"`
|
|
ZipName string `json:"zip_name"`
|
|
OriginalBytes int64 `json:"original_bytes"`
|
|
ProcessedBytes int64 `json:"processed_bytes"`
|
|
Message string `json:"message"`
|
|
|
|
cancelFunc context.CancelFunc `json:"-"`
|
|
}
|
|
|
|
type JobManager struct {
|
|
jobs map[string]*BackgroundJob
|
|
mu sync.RWMutex
|
|
}
|
|
|
|
var globalJobManager = &JobManager{
|
|
jobs: make(map[string]*BackgroundJob),
|
|
}
|
|
|
|
func (jm *JobManager) CreateJob(sessionID, operation string, params map[string]interface{}, timeLimitSec int) *BackgroundJob {
|
|
if timeLimitSec <= 0 {
|
|
timeLimitSec = 120 // Default 2 minutes governance timeout
|
|
}
|
|
if timeLimitSec > 3600 {
|
|
timeLimitSec = 3600 // Max 1 hour
|
|
}
|
|
|
|
jobID := "job_" + uuid.New().String()[:8]
|
|
job := &BackgroundJob{
|
|
JobID: jobID,
|
|
SessionID: sessionID,
|
|
Operation: operation,
|
|
Params: params,
|
|
Status: JobStatusQueued,
|
|
TimeLimitSec: timeLimitSec,
|
|
CreatedAt: time.Now(),
|
|
Errors: []string{},
|
|
Message: "Job queued in enterprise background engine",
|
|
}
|
|
|
|
jm.mu.Lock()
|
|
jm.jobs[jobID] = job
|
|
jm.mu.Unlock()
|
|
|
|
return job
|
|
}
|
|
|
|
func (jm *JobManager) GetJob(jobID string) *BackgroundJob {
|
|
jm.mu.RLock()
|
|
defer jm.mu.RUnlock()
|
|
job, exists := jm.jobs[jobID]
|
|
if !exists {
|
|
return nil
|
|
}
|
|
|
|
// Calculate live elapsed and remaining
|
|
now := time.Now()
|
|
if job.Status == JobStatusProcessing {
|
|
job.ElapsedSec = now.Sub(job.StartedAt).Seconds()
|
|
rem := float64(job.TimeLimitSec) - job.ElapsedSec
|
|
if rem < 0 {
|
|
rem = 0
|
|
}
|
|
job.RemainingSec = rem
|
|
} else if job.Status == JobStatusCompleted || job.Status == JobStatusTimedOut || job.Status == JobStatusCancelled || job.Status == JobStatusFailed {
|
|
job.ElapsedSec = job.FinishedAt.Sub(job.StartedAt).Seconds()
|
|
job.RemainingSec = 0
|
|
}
|
|
|
|
return job
|
|
}
|
|
|
|
func (jm *JobManager) ListJobs() []*BackgroundJob {
|
|
jm.mu.RLock()
|
|
defer jm.mu.RUnlock()
|
|
|
|
var list []*BackgroundJob
|
|
for _, j := range jm.jobs {
|
|
// Update elapsed
|
|
now := time.Now()
|
|
if j.Status == JobStatusProcessing {
|
|
j.ElapsedSec = now.Sub(j.StartedAt).Seconds()
|
|
rem := float64(j.TimeLimitSec) - j.ElapsedSec
|
|
if rem < 0 {
|
|
rem = 0
|
|
}
|
|
j.RemainingSec = rem
|
|
} else if j.FinishedAt.After(j.StartedAt) {
|
|
j.ElapsedSec = j.FinishedAt.Sub(j.StartedAt).Seconds()
|
|
j.RemainingSec = 0
|
|
}
|
|
list = append(list, j)
|
|
}
|
|
|
|
sort.Slice(list, func(i, j int) bool {
|
|
return list[i].CreatedAt.After(list[j].CreatedAt)
|
|
})
|
|
|
|
return list
|
|
}
|
|
|
|
func (jm *JobManager) CancelJob(jobID string) bool {
|
|
jm.mu.Lock()
|
|
defer jm.mu.Unlock()
|
|
|
|
job, exists := jm.jobs[jobID]
|
|
if !exists {
|
|
return false
|
|
}
|
|
|
|
if job.Status == JobStatusProcessing || job.Status == JobStatusQueued {
|
|
if job.cancelFunc != nil {
|
|
job.cancelFunc()
|
|
}
|
|
job.Status = JobStatusCancelled
|
|
job.FinishedAt = time.Now()
|
|
job.Message = "Job cancelled by enterprise user"
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// StartJob executes the job in background goroutine with strict time-limit governance
|
|
func (jm *JobManager) StartJob(job *BackgroundJob) {
|
|
go func() {
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(job.TimeLimitSec)*time.Second)
|
|
job.cancelFunc = cancel
|
|
defer cancel()
|
|
|
|
job.StartedAt = time.Now()
|
|
job.Status = JobStatusProcessing
|
|
job.Message = fmt.Sprintf("Processing started with %d-second governance time limit", job.TimeLimitSec)
|
|
|
|
uploadsDir := getSessionDir(job.SessionID, "uploads")
|
|
resultsDir := getSessionDir(job.SessionID, "results")
|
|
|
|
os.RemoveAll(resultsDir)
|
|
os.MkdirAll(resultsDir, 0755)
|
|
|
|
var allFiles []string
|
|
filepath.Walk(uploadsDir, func(path string, info os.FileInfo, err error) error {
|
|
if err == nil && !info.IsDir() {
|
|
allFiles = append(allFiles, path)
|
|
}
|
|
return nil
|
|
})
|
|
|
|
job.TotalFiles = len(allFiles)
|
|
if len(allFiles) == 0 {
|
|
job.Status = JobStatusFailed
|
|
job.FinishedAt = time.Now()
|
|
job.Message = "No files found in upload session"
|
|
return
|
|
}
|
|
|
|
zipName := fmt.Sprintf("batch_%s_%s.zip", job.Operation, job.JobID)
|
|
job.ZipName = zipName
|
|
|
|
for i, file := range allFiles {
|
|
// Check if context timed out or was cancelled
|
|
select {
|
|
case <-ctx.Done():
|
|
job.FinishedAt = time.Now()
|
|
if ctx.Err() == context.DeadlineExceeded {
|
|
job.Status = JobStatusTimedOut
|
|
job.Message = fmt.Sprintf("Enterprise Governance: Time limit of %d seconds exceeded. Auto-terminated.", job.TimeLimitSec)
|
|
slog.Warn("background job timed out", "job_id", job.JobID, "limit_sec", job.TimeLimitSec, "completed", job.CompletedFiles, "total", job.TotalFiles)
|
|
} else {
|
|
job.Status = JobStatusCancelled
|
|
job.Message = "Job cancelled"
|
|
}
|
|
|
|
// Package whatever files were completed before timeout
|
|
if job.CompletedFiles > 0 {
|
|
createZip(job.SessionID, "results", zipName, "manipulated_partial")
|
|
}
|
|
return
|
|
default:
|
|
}
|
|
|
|
origSize, newSize, err := processSingleBatchFile(file, job.Operation, job.Params, uploadsDir, resultsDir)
|
|
if err != nil {
|
|
job.ErrorCount++
|
|
job.Errors = append(job.Errors, fmt.Sprintf("%s: %v", filepath.Base(file), err))
|
|
} else {
|
|
job.OriginalBytes += origSize
|
|
job.ProcessedBytes += newSize
|
|
}
|
|
|
|
job.CompletedFiles = i + 1
|
|
job.ProgressPct = int((float64(job.CompletedFiles) / float64(job.TotalFiles)) * 100)
|
|
}
|
|
|
|
// Success completion
|
|
job.FinishedAt = time.Now()
|
|
job.Status = JobStatusCompleted
|
|
job.ProgressPct = 100
|
|
job.Message = fmt.Sprintf("Completed! Processed %d/%d files successfully.", job.CompletedFiles-job.ErrorCount, job.TotalFiles)
|
|
|
|
err := createZip(job.SessionID, "results", zipName, "manipulated_output")
|
|
if err != nil {
|
|
slog.Error("failed to create zip for background job", "job_id", job.JobID, "err", err)
|
|
job.Errors = append(job.Errors, fmt.Sprintf("Zip generation error: %v", err))
|
|
}
|
|
}()
|
|
}
|
|
|
|
// Clean old jobs periodically (older than 2 hours)
|
|
func cleanupOldJobs() {
|
|
globalJobManager.mu.Lock()
|
|
defer globalJobManager.mu.Unlock()
|
|
|
|
now := time.Now()
|
|
for id, job := range globalJobManager.jobs {
|
|
if now.Sub(job.CreatedAt) > 2*time.Hour {
|
|
delete(globalJobManager.jobs, id)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════════════════
|
|
// HTTP HANDLERS FOR BACKGROUND JOB GOVERNANCE
|
|
// ═══════════════════════════════════════════════════════════════════
|
|
|
|
func startBackgroundJobHandler(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
|
|
var req struct {
|
|
SessionID string `json:"session_id"`
|
|
Operation string `json:"operation"`
|
|
TimeLimitSec int `json:"time_limit_sec"`
|
|
Params map[string]interface{} `json:"params"`
|
|
}
|
|
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
http.Error(w, `{"error":"Invalid request payload"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
if req.SessionID == "" || !validateSession(req.SessionID) {
|
|
http.Error(w, `{"error":"Invalid or missing session_id"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
if req.Operation == "" {
|
|
http.Error(w, `{"error":"Operation must be specified"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
if req.TimeLimitSec <= 0 {
|
|
req.TimeLimitSec = 120
|
|
}
|
|
|
|
job := globalJobManager.CreateJob(req.SessionID, req.Operation, req.Params, req.TimeLimitSec)
|
|
globalJobManager.StartJob(job)
|
|
|
|
json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"status": "ok",
|
|
"job_id": job.JobID,
|
|
"time_limit_sec": job.TimeLimitSec,
|
|
"message": "Job dispatched to enterprise background worker pool",
|
|
})
|
|
}
|
|
|
|
func getJobStatusHandler(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
|
|
jobID := r.URL.Query().Get("job_id")
|
|
if jobID == "" {
|
|
http.Error(w, `{"error":"job_id parameter required"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
job := globalJobManager.GetJob(jobID)
|
|
if job == nil {
|
|
http.Error(w, `{"error":"Job not found"}`, http.StatusNotFound)
|
|
return
|
|
}
|
|
|
|
json.NewEncoder(w).Encode(job)
|
|
}
|
|
|
|
func listJobsHandler(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
jobs := globalJobManager.ListJobs()
|
|
json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"status": "ok",
|
|
"count": len(jobs),
|
|
"jobs": jobs,
|
|
})
|
|
}
|
|
|
|
func cancelJobHandler(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
|
|
jobID := r.URL.Query().Get("job_id")
|
|
if jobID == "" {
|
|
var req struct {
|
|
JobID string `json:"job_id"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err == nil {
|
|
jobID = req.JobID
|
|
}
|
|
}
|
|
|
|
if jobID == "" {
|
|
http.Error(w, `{"error":"job_id required"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
success := globalJobManager.CancelJob(jobID)
|
|
if !success {
|
|
http.Error(w, `{"error":"Job could not be cancelled or already finished"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"status": "ok",
|
|
"message": "Job cancelled successfully",
|
|
})
|
|
}
|
|
|
|
func downloadJobZipHandler(w http.ResponseWriter, r *http.Request) {
|
|
jobID := r.URL.Query().Get("job_id")
|
|
if jobID == "" {
|
|
http.Error(w, "job_id parameter required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
job := globalJobManager.GetJob(jobID)
|
|
if job == nil {
|
|
http.Error(w, "Job not found", http.StatusNotFound)
|
|
return
|
|
}
|
|
|
|
zipPath := filepath.Join(getSessionDir(job.SessionID), job.ZipName)
|
|
if _, err := os.Stat(zipPath); os.IsNotExist(err) {
|
|
http.Error(w, "Processed ZIP archive not ready or expired", http.StatusNotFound)
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/zip")
|
|
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", job.ZipName))
|
|
http.ServeFile(w, r, zipPath)
|
|
}
|