Files
Unified_Dashboard_Suite_Go/internal/gateway/processmanager.go
T

275 lines
6.4 KiB
Go

package gateway
import (
"context"
"fmt"
"log/slog"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"syscall"
"time"
)
type ProcessState int
const (
StateIdle ProcessState = iota
StateStarting
StateRunning
StateFailed
)
type ManagedProcess struct {
Config ToolConfig
Cmd *exec.Cmd
State ProcessState
mu sync.Mutex
cancel context.CancelFunc
LogFile *os.File
}
type ProcessManager struct {
processes map[string]*ManagedProcess
mu sync.RWMutex
rootCtx context.Context
}
func NewProcessManager(ctx context.Context) *ProcessManager {
return &ProcessManager{
processes: make(map[string]*ManagedProcess),
rootCtx: ctx,
}
}
func (pm *ProcessManager) getConfig(toolID string) *ToolConfig {
for i := range Registry {
if Registry[i].Key == toolID {
return &Registry[i]
}
}
return nil
}
func filterEnv(env []string, keysToRemove ...string) []string {
var res []string
removeMap := make(map[string]bool)
for _, k := range keysToRemove {
removeMap[k] = true
}
for _, e := range env {
parts := strings.SplitN(e, "=", 2)
if len(parts) > 0 && !removeMap[parts[0]] {
res = append(res, e)
}
}
return res
}
func (pm *ProcessManager) cleanup(toolID string) {
if proc, ok := pm.processes[toolID]; ok {
if proc.cancel != nil {
proc.cancel()
}
if proc.LogFile != nil {
proc.LogFile.Close()
}
delete(pm.processes, toolID)
}
}
// IsRunning checks whether a tool's process is currently active and healthy.
func (pm *ProcessManager) IsRunning(toolID string) bool {
if toolID == "docs" || toolID == "khira" {
return true
}
pm.mu.RLock()
defer pm.mu.RUnlock()
proc, ok := pm.processes[toolID]
if !ok || proc.Cmd == nil || proc.Cmd.Process == nil || proc.Cmd.ProcessState != nil {
return false
}
return syscall.Kill(proc.Cmd.Process.Pid, 0) == nil
}
// EnsureRunning launches the tool on demand if it is not currently running.
func (pm *ProcessManager) EnsureRunning(ctx context.Context, toolID string) error {
if pm.IsRunning(toolID) {
return nil
}
cfg := pm.getConfig(toolID)
if cfg == nil {
return fmt.Errorf("unknown tool: %s", toolID)
}
if _, err := pm.Launch(toolID); err != nil {
return err
}
return pm.waitReady(ctx, cfg.Port, 30, 100*time.Millisecond)
}
func (pm *ProcessManager) Launch(toolID string) (string, error) {
if toolID == "docs" {
return "/docs/", nil
}
pm.mu.Lock()
defer pm.mu.Unlock()
cfg := pm.getConfig(toolID)
if cfg == nil {
return "", fmt.Errorf("unknown tool: %s", toolID)
}
// Already running?
if proc, ok := pm.processes[toolID]; ok {
isAlive := false
if proc.Cmd != nil && proc.Cmd.Process != nil && proc.Cmd.ProcessState == nil {
if err := syscall.Kill(proc.Cmd.Process.Pid, 0); err == nil {
isAlive = true
}
}
if isAlive {
slog.Info("tool already running", "tool", toolID, "pid", proc.Cmd.Process.Pid)
return fmt.Sprintf("/tool/%s/", toolID), nil
}
// Dead — clean up
pm.cleanup(toolID)
}
// Spawn
ctx, cancel := context.WithCancel(pm.rootCtx)
logFilePath := cfg.LogFile
if logFilePath == "" {
logFilePath = filepath.Join("logs", toolID+".log")
}
// ensure log directory exists
if lastSlash := strings.LastIndex(logFilePath, "/"); lastSlash != -1 {
os.MkdirAll(logFilePath[:lastSlash], 0755)
}
logFile, err := os.OpenFile(logFilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
slog.Error("failed to open tool log file", "tool", toolID, "path", logFilePath, "err", err)
}
env := os.Environ()
// Remove inherited PORT and Flask reloader vars
env = filterEnv(env, "PORT", "WERKZEUG_SERVER_FD", "WERKZEUG_RUN_MAIN")
env = append(env, fmt.Sprintf("SCRIPT_NAME=/tool/%s", toolID))
env = append(env, fmt.Sprintf("PORT=%d", cfg.Port))
env = append(env, "MANAGED_BY_GATEWAY=1")
env = append(env, fmt.Sprintf("LOG_FILE=%s", logFilePath))
cmd := exec.CommandContext(ctx, cfg.Command, cfg.Args...)
cmd.Dir = cfg.WorkDir
cmd.Env = env
if logFile != nil {
cmd.Stdout = logFile
cmd.Stderr = logFile
} else {
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
}
if err := cmd.Start(); err != nil {
cancel()
if logFile != nil {
logFile.Close()
}
return "", fmt.Errorf("failed to start %s: %w", toolID, err)
}
proc := &ManagedProcess{
Config: *cfg,
Cmd: cmd,
State: StateStarting,
cancel: cancel,
LogFile: logFile,
}
pm.processes[toolID] = proc
slog.Info("process started", "tool", toolID, "pid", cmd.Process.Pid, "log_file", logFilePath)
// Clean up process table when process exits
go func() {
_ = cmd.Wait()
pm.mu.Lock()
if p, exists := pm.processes[toolID]; exists && p == proc {
pm.cleanup(toolID)
}
pm.mu.Unlock()
}()
// Wait for readiness asynchronously
go func() {
if err := pm.waitReady(ctx, cfg.Port, 40, 500*time.Millisecond); err != nil {
slog.Warn("tool slow to respond", "tool", toolID, "err", err)
} else {
slog.Info("tool is ready", "tool", toolID, "port", cfg.Port)
}
proc.State = StateRunning
}()
return fmt.Sprintf("/tool/%s/", toolID), nil
}
// waitReady polls the internal port with exponential backoff
func (pm *ProcessManager) waitReady(ctx context.Context, port int, maxAttempts int, interval time.Duration) error {
client := &http.Client{Timeout: 300 * time.Millisecond}
url := fmt.Sprintf("http://127.0.0.1:%d", port)
for i := 0; i < maxAttempts; i++ {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
resp, err := client.Get(url)
if err == nil && resp.StatusCode < 500 {
resp.Body.Close()
return nil
}
time.Sleep(interval)
}
return fmt.Errorf("tool not ready after %d attempts", maxAttempts)
}
// GracefulShutdown sends SIGTERM to all children and waits with timeout
func (pm *ProcessManager) GracefulShutdown(timeout time.Duration) {
pm.mu.Lock()
defer pm.mu.Unlock()
slog.Info("shutting down all managed processes", "count", len(pm.processes))
var wg sync.WaitGroup
for id, proc := range pm.processes {
wg.Add(1)
go func(id string, p *ManagedProcess) {
defer wg.Done()
p.cancel() // triggers SIGTERM via CommandContext
done := make(chan struct{})
go func() {
if p.Cmd != nil {
p.Cmd.Wait()
}
close(done)
}()
select {
case <-done:
slog.Info("process exited cleanly", "tool", id)
case <-time.After(timeout):
slog.Warn("force-killing process", "tool", id)
if p.Cmd != nil && p.Cmd.Process != nil {
p.Cmd.Process.Kill()
}
}
if p.LogFile != nil {
p.LogFile.Close()
}
}(id, proc)
}
wg.Wait()
}