275 lines
6.4 KiB
Go
275 lines
6.4 KiB
Go
package gateway
|
|||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"fmt"
|
||
|
|
"log/slog"
|
||
|
|
"net/http"
|
||
|
|
"os"
|
||
|
|
"os/exec"
|
||
|
|
"path/filepath"
|
||
|
|
"strings"
|
||
|
|
"sync"
|
||
|
|
"syscall"
|
||
|
|
"time"
|
||
|
|
)
|
||
|
|
|
||
|
|
type ProcessState int
|
||
|
|
|
||
|
|
const (
|
||
|
|
StateIdle ProcessState = iota
|
||
|
|
StateStarting
|
||
|
|
StateRunning
|
||
|
|
StateFailed
|
||
|
|
)
|
||
|
|
|
||
|
|
type ManagedProcess struct {
|
||
|
|
Config ToolConfig
|
||
|
|
Cmd *exec.Cmd
|
||
|
|
State ProcessState
|
||
|
|
mu sync.Mutex
|
||
|
|
cancel context.CancelFunc
|
||
|
|
LogFile *os.File
|
||
|
|
}
|
||
|
|
|
||
|
|
type ProcessManager struct {
|
||
|
|
processes map[string]*ManagedProcess
|
||
|
|
mu sync.RWMutex
|
||
|
|
rootCtx context.Context
|
||
|
|
}
|
||
|
|
|
||
|
|
func NewProcessManager(ctx context.Context) *ProcessManager {
|
||
|
|
return &ProcessManager{
|
||
|
|
processes: make(map[string]*ManagedProcess),
|
||
|
|
rootCtx: ctx,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (pm *ProcessManager) getConfig(toolID string) *ToolConfig {
|
||
|
|
for i := range Registry {
|
||
|
|
if Registry[i].Key == toolID {
|
||
|
|
return &Registry[i]
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func filterEnv(env []string, keysToRemove ...string) []string {
|
||
|
|
var res []string
|
||
|
|
removeMap := make(map[string]bool)
|
||
|
|
for _, k := range keysToRemove {
|
||
|
|
removeMap[k] = true
|
||
|
|
}
|
||
|
|
for _, e := range env {
|
||
|
|
parts := strings.SplitN(e, "=", 2)
|
||
|
|
if len(parts) > 0 && !removeMap[parts[0]] {
|
||
|
|
res = append(res, e)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return res
|
||
|
|
}
|
||
|
|
|
||
|
|
func (pm *ProcessManager) cleanup(toolID string) {
|
||
|
|
if proc, ok := pm.processes[toolID]; ok {
|
||
|
|
if proc.cancel != nil {
|
||
|
|
proc.cancel()
|
||
|
|
}
|
||
|
|
if proc.LogFile != nil {
|
||
|
|
proc.LogFile.Close()
|
||
|
|
}
|
||
|
|
delete(pm.processes, toolID)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// IsRunning checks whether a tool's process is currently active and healthy.
|
||
|
|
func (pm *ProcessManager) IsRunning(toolID string) bool {
|
||
|
|
if toolID == "docs" || toolID == "khira" {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
pm.mu.RLock()
|
||
|
|
defer pm.mu.RUnlock()
|
||
|
|
proc, ok := pm.processes[toolID]
|
||
|
|
if !ok || proc.Cmd == nil || proc.Cmd.Process == nil || proc.Cmd.ProcessState != nil {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
return syscall.Kill(proc.Cmd.Process.Pid, 0) == nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// EnsureRunning launches the tool on demand if it is not currently running.
|
||
|
|
func (pm *ProcessManager) EnsureRunning(ctx context.Context, toolID string) error {
|
||
|
|
if pm.IsRunning(toolID) {
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
cfg := pm.getConfig(toolID)
|
||
|
|
if cfg == nil {
|
||
|
|
return fmt.Errorf("unknown tool: %s", toolID)
|
||
|
|
}
|
||
|
|
if _, err := pm.Launch(toolID); err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
return pm.waitReady(ctx, cfg.Port, 30, 100*time.Millisecond)
|
||
|
|
}
|
||
|
|
|
||
|
|
func (pm *ProcessManager) Launch(toolID string) (string, error) {
|
||
|
|
if toolID == "docs" {
|
||
|
|
return "/docs/", nil
|
||
|
|
}
|
||
|
|
|
||
|
|
pm.mu.Lock()
|
||
|
|
defer pm.mu.Unlock()
|
||
|
|
|
||
|
|
cfg := pm.getConfig(toolID)
|
||
|
|
if cfg == nil {
|
||
|
|
return "", fmt.Errorf("unknown tool: %s", toolID)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Already running?
|
||
|
|
if proc, ok := pm.processes[toolID]; ok {
|
||
|
|
isAlive := false
|
||
|
|
if proc.Cmd != nil && proc.Cmd.Process != nil && proc.Cmd.ProcessState == nil {
|
||
|
|
if err := syscall.Kill(proc.Cmd.Process.Pid, 0); err == nil {
|
||
|
|
isAlive = true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if isAlive {
|
||
|
|
slog.Info("tool already running", "tool", toolID, "pid", proc.Cmd.Process.Pid)
|
||
|
|
return fmt.Sprintf("/tool/%s/", toolID), nil
|
||
|
|
}
|
||
|
|
// Dead — clean up
|
||
|
|
pm.cleanup(toolID)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Spawn
|
||
|
|
ctx, cancel := context.WithCancel(pm.rootCtx)
|
||
|
|
|
||
|
|
logFilePath := cfg.LogFile
|
||
|
|
if logFilePath == "" {
|
||
|
|
logFilePath = filepath.Join("logs", toolID+".log")
|
||
|
|
}
|
||
|
|
|
||
|
|
// ensure log directory exists
|
||
|
|
if lastSlash := strings.LastIndex(logFilePath, "/"); lastSlash != -1 {
|
||
|
|
os.MkdirAll(logFilePath[:lastSlash], 0755)
|
||
|
|
}
|
||
|
|
|
||
|
|
logFile, err := os.OpenFile(logFilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
|
||
|
|
if err != nil {
|
||
|
|
slog.Error("failed to open tool log file", "tool", toolID, "path", logFilePath, "err", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
env := os.Environ()
|
||
|
|
// Remove inherited PORT and Flask reloader vars
|
||
|
|
env = filterEnv(env, "PORT", "WERKZEUG_SERVER_FD", "WERKZEUG_RUN_MAIN")
|
||
|
|
env = append(env, fmt.Sprintf("SCRIPT_NAME=/tool/%s", toolID))
|
||
|
|
env = append(env, fmt.Sprintf("PORT=%d", cfg.Port))
|
||
|
|
env = append(env, "MANAGED_BY_GATEWAY=1")
|
||
|
|
env = append(env, fmt.Sprintf("LOG_FILE=%s", logFilePath))
|
||
|
|
|
||
|
|
cmd := exec.CommandContext(ctx, cfg.Command, cfg.Args...)
|
||
|
|
cmd.Dir = cfg.WorkDir
|
||
|
|
cmd.Env = env
|
||
|
|
if logFile != nil {
|
||
|
|
cmd.Stdout = logFile
|
||
|
|
cmd.Stderr = logFile
|
||
|
|
} else {
|
||
|
|
cmd.Stdout = os.Stdout
|
||
|
|
cmd.Stderr = os.Stderr
|
||
|
|
}
|
||
|
|
|
||
|
|
if err := cmd.Start(); err != nil {
|
||
|
|
cancel()
|
||
|
|
if logFile != nil {
|
||
|
|
logFile.Close()
|
||
|
|
}
|
||
|
|
return "", fmt.Errorf("failed to start %s: %w", toolID, err)
|
||
|
|
}
|
||
|
|
|
||
|
|
proc := &ManagedProcess{
|
||
|
|
Config: *cfg,
|
||
|
|
Cmd: cmd,
|
||
|
|
State: StateStarting,
|
||
|
|
cancel: cancel,
|
||
|
|
LogFile: logFile,
|
||
|
|
}
|
||
|
|
pm.processes[toolID] = proc
|
||
|
|
slog.Info("process started", "tool", toolID, "pid", cmd.Process.Pid, "log_file", logFilePath)
|
||
|
|
|
||
|
|
// Clean up process table when process exits
|
||
|
|
go func() {
|
||
|
|
_ = cmd.Wait()
|
||
|
|
pm.mu.Lock()
|
||
|
|
if p, exists := pm.processes[toolID]; exists && p == proc {
|
||
|
|
pm.cleanup(toolID)
|
||
|
|
}
|
||
|
|
pm.mu.Unlock()
|
||
|
|
}()
|
||
|
|
|
||
|
|
// Wait for readiness asynchronously
|
||
|
|
go func() {
|
||
|
|
if err := pm.waitReady(ctx, cfg.Port, 40, 500*time.Millisecond); err != nil {
|
||
|
|
slog.Warn("tool slow to respond", "tool", toolID, "err", err)
|
||
|
|
} else {
|
||
|
|
slog.Info("tool is ready", "tool", toolID, "port", cfg.Port)
|
||
|
|
}
|
||
|
|
proc.State = StateRunning
|
||
|
|
}()
|
||
|
|
|
||
|
|
return fmt.Sprintf("/tool/%s/", toolID), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// waitReady polls the internal port with exponential backoff
|
||
|
|
func (pm *ProcessManager) waitReady(ctx context.Context, port int, maxAttempts int, interval time.Duration) error {
|
||
|
|
client := &http.Client{Timeout: 300 * time.Millisecond}
|
||
|
|
url := fmt.Sprintf("http://127.0.0.1:%d", port)
|
||
|
|
for i := 0; i < maxAttempts; i++ {
|
||
|
|
select {
|
||
|
|
case <-ctx.Done():
|
||
|
|
return ctx.Err()
|
||
|
|
default:
|
||
|
|
}
|
||
|
|
resp, err := client.Get(url)
|
||
|
|
if err == nil && resp.StatusCode < 500 {
|
||
|
|
resp.Body.Close()
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
time.Sleep(interval)
|
||
|
|
}
|
||
|
|
return fmt.Errorf("tool not ready after %d attempts", maxAttempts)
|
||
|
|
}
|
||
|
|
|
||
|
|
// GracefulShutdown sends SIGTERM to all children and waits with timeout
|
||
|
|
func (pm *ProcessManager) GracefulShutdown(timeout time.Duration) {
|
||
|
|
pm.mu.Lock()
|
||
|
|
defer pm.mu.Unlock()
|
||
|
|
|
||
|
|
slog.Info("shutting down all managed processes", "count", len(pm.processes))
|
||
|
|
var wg sync.WaitGroup
|
||
|
|
for id, proc := range pm.processes {
|
||
|
|
wg.Add(1)
|
||
|
|
go func(id string, p *ManagedProcess) {
|
||
|
|
defer wg.Done()
|
||
|
|
p.cancel() // triggers SIGTERM via CommandContext
|
||
|
|
done := make(chan struct{})
|
||
|
|
go func() {
|
||
|
|
if p.Cmd != nil {
|
||
|
|
p.Cmd.Wait()
|
||
|
|
}
|
||
|
|
close(done)
|
||
|
|
}()
|
||
|
|
select {
|
||
|
|
case <-done:
|
||
|
|
slog.Info("process exited cleanly", "tool", id)
|
||
|
|
case <-time.After(timeout):
|
||
|
|
slog.Warn("force-killing process", "tool", id)
|
||
|
|
if p.Cmd != nil && p.Cmd.Process != nil {
|
||
|
|
p.Cmd.Process.Kill()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if p.LogFile != nil {
|
||
|
|
p.LogFile.Close()
|
||
|
|
}
|
||
|
|
}(id, proc)
|
||
|
|
}
|
||
|
|
wg.Wait()
|
||
|
|
}
|