Initial commit: unified dashboard suite state with Khira and Image Manipulator 3-file setup

This commit is contained in:
2026-10-26 11:17:13 +05:30
commit 6c60b4ecb8
148 changed files with 128149 additions and 0 deletions
+167
View File
@@ -0,0 +1,167 @@
package main
import (
"encoding/json"
"log/slog"
"net/http"
"os"
)
const (
CollectionsFile = "data/bulk-api-tester/collections.json"
SpacesFile = "data/bulk-api-tester/spaces.json"
LastRunFile = "data/bulk-api-tester/last_run.json"
)
func writeJSON(w http.ResponseWriter, status int, data any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
json.NewEncoder(w).Encode(data)
}
func getCollectionsHandler(w http.ResponseWriter, r *http.Request) {
if _, err := os.Stat(CollectionsFile); err == nil {
var data any
if err := readJSON(CollectionsFile, &data); err == nil {
writeJSON(w, http.StatusOK, data)
return
}
}
// Default
writeJSON(w, http.StatusOK, []map[string]any{
{"id": "default", "name": "My Requests", "requests": []any{}},
})
}
func saveCollectionsHandler(w http.ResponseWriter, r *http.Request) {
var data []any
if err := json.NewDecoder(r.Body).Decode(&data); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]any{"success": false, "error": "Invalid data format, expected list"})
return
}
if err := atomicWriteJSON(CollectionsFile, data); err != nil {
slog.Error("failed to save collections", "error", err)
writeJSON(w, http.StatusInternalServerError, map[string]any{"success": false, "error": err.Error()})
return
}
slog.Info("saved collections", "count", len(data))
writeJSON(w, http.StatusOK, map[string]bool{"success": true})
}
func getSpacesHandler(w http.ResponseWriter, r *http.Request) {
var spacesData []map[string]any
// Try loading spaces
if err := readJSON(SpacesFile, &spacesData); err != nil {
// Legacy migration logic
var legacyCollections []any
if err := readJSON(CollectionsFile, &legacyCollections); err == nil && len(legacyCollections) > 0 {
spacesData = []map[string]any{
{
"id": "vaishno",
"name": "Vaishno",
"collections": legacyCollections,
},
{
"id": "aman",
"name": "Aman",
"collections": legacyCollections,
},
}
} else {
// Fallback
defaultCollections := []map[string]any{{"id": "default", "name": "My Requests", "requests": []any{}}}
spacesData = []map[string]any{
{"id": "vaishno", "name": "Vaishno", "collections": defaultCollections},
{"id": "aman", "name": "Aman", "collections": defaultCollections},
}
}
// Save initialized spaces
atomicWriteJSON(SpacesFile, spacesData)
}
writeJSON(w, http.StatusOK, spacesData)
}
func saveSpacesHandler(w http.ResponseWriter, r *http.Request) {
var data []any
if err := json.NewDecoder(r.Body).Decode(&data); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]any{"success": false, "error": "Invalid data format, expected list"})
return
}
if err := atomicWriteJSON(SpacesFile, data); err != nil {
slog.Error("failed to save spaces", "error", err)
writeJSON(w, http.StatusInternalServerError, map[string]any{"success": false, "error": err.Error()})
return
}
slog.Info("saved spaces", "count", len(data))
writeJSON(w, http.StatusOK, map[string]bool{"success": true})
}
func getLastRunHandler(w http.ResponseWriter, r *http.Request) {
spaceID := r.URL.Query().Get("spaceId")
if spaceID == "" {
spaceID = "default"
}
var data map[string]any
if err := readJSON(LastRunFile, &data); err == nil {
if spaceData, ok := data[spaceID]; ok {
writeJSON(w, http.StatusOK, spaceData)
return
}
}
writeJSON(w, http.StatusOK, map[string]any{})
}
func saveLastRunHandler(w http.ResponseWriter, r *http.Request) {
var payload map[string]any
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
slog.Error("invalid saveLastRun payload", "error", err)
writeJSON(w, http.StatusBadRequest, map[string]any{"error": err.Error()})
return
}
spaceID, _ := payload["spaceId"].(string)
if spaceID == "" {
spaceID = "default"
}
var data map[string]any
if err := readJSON(LastRunFile, &data); err != nil {
data = make(map[string]any)
}
// The payload is the entire state data to save for this space
data[spaceID] = payload
if err := atomicWriteJSON(LastRunFile, data); err != nil {
slog.Error("failed to save last run data", "space_id", spaceID, "error", err)
writeJSON(w, http.StatusInternalServerError, map[string]any{"error": err.Error()})
return
}
slog.Info("saved last run state", "space_id", spaceID)
writeJSON(w, http.StatusOK, map[string]bool{"success": true})
}
func getDocsDataHandler(w http.ResponseWriter, r *http.Request) {
paths := []string{
"web/bulk-api-tester/static/data/docs_apis.json",
"web/docs/data.json",
"scratch/faithful_docs_data.json",
}
for _, p := range paths {
if b, err := os.ReadFile(p); err == nil {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusOK)
w.Write(b)
return
}
}
writeJSON(w, http.StatusNotFound, map[string]string{"error": "docs data not found"})
}
+239
View File
@@ -0,0 +1,239 @@
package main
import (
"encoding/base64"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
)
func browseFolderHandler(w http.ResponseWriter, r *http.Request) {
// Try zenity
cmd := exec.Command("zenity", "--file-selection", "--directory", "--title=Select Output Folder")
out, err := cmd.Output()
path := strings.TrimSpace(string(out))
if err == nil && path != "" {
slog.Info("folder selected via zenity", "path", path)
writeJSON(w, http.StatusOK, map[string]any{"success": true, "path": path})
return
}
// Try tkinter fallback
pyCode := `import tkinter as tk; from tkinter import filedialog; root = tk.Tk(); root.withdraw(); root.attributes("-topmost", True); print(filedialog.askdirectory()); root.destroy()`
cmd = exec.Command("python3", "-c", pyCode)
out, err = cmd.Output()
path = strings.TrimSpace(string(out))
if err == nil && path != "" {
slog.Info("folder selected via tkinter", "path", path)
writeJSON(w, http.StatusOK, map[string]any{"success": true, "path": path})
return
}
slog.Warn("no folder selected")
writeJSON(w, http.StatusOK, map[string]any{"success": false, "error": "No folder selected"})
}
func saveFolderHandler(w http.ResponseWriter, r *http.Request) {
var data map[string]any
if err := json.NewDecoder(r.Body).Decode(&data); err != nil {
slog.Error("invalid save folder payload", "error", err)
writeJSON(w, http.StatusBadRequest, map[string]any{"success": false, "error": err.Error()})
return
}
targetDir, _ := data["path"].(string)
if targetDir == "" {
writeJSON(w, http.StatusOK, map[string]any{"success": false, "error": "Invalid path"})
return
}
if _, err := os.Stat(targetDir); os.IsNotExist(err) {
writeJSON(w, http.StatusOK, map[string]any{"success": false, "error": "Path does not exist"})
return
}
results, _ := data["results"].([]any)
savedCount := 0
var errors []string
for _, resAny := range results {
res, ok := resAny.(map[string]any)
if !ok {
continue
}
filename, _ := res["filename"].(string)
if filename == "" {
continue
}
fullPath := filepath.Join(targetDir, filepath.Base(filename))
if strings.Contains(filename, "/") || strings.Contains(filename, string(os.PathSeparator)) {
fullPath = filepath.Join(targetDir, filename)
if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
errors = append(errors, err.Error())
continue
}
}
isBinary, _ := res["is_binary"].(bool)
binaryData, _ := res["binary_data"].(string)
content, hasContent := res["content"]
if isBinary && binaryData != "" {
binaryBytes, err := base64.StdEncoding.DecodeString(binaryData)
if err != nil {
errors = append(errors, fmt.Sprintf("%s: invalid base64", filename))
continue
}
if err := os.WriteFile(fullPath, binaryBytes, 0644); err != nil {
errors = append(errors, fmt.Sprintf("%s: %s", filename, err.Error()))
} else {
savedCount++
}
} else if hasContent {
contentStr := ""
switch v := content.(type) {
case string:
contentStr = v
default:
bytes, _ := json.Marshal(v)
contentStr = string(bytes)
}
if err := os.WriteFile(fullPath, []byte(contentStr), 0644); err != nil {
errors = append(errors, fmt.Sprintf("%s: %s", filename, err.Error()))
} else {
savedCount++
}
}
}
slog.Info("saved bulk results to folder", "path", targetDir, "saved_count", savedCount, "errors_count", len(errors))
writeJSON(w, http.StatusOK, map[string]any{"success": true, "count": savedCount, "errors": errors})
}
func saveFolderFromLastrunHandler(w http.ResponseWriter, r *http.Request) {
var data map[string]any
if err := json.NewDecoder(r.Body).Decode(&data); err != nil {
slog.Error("invalid saveFolderFromLastrun payload", "error", err)
writeJSON(w, http.StatusBadRequest, map[string]any{"success": false, "error": err.Error()})
return
}
targetDir, _ := data["path"].(string)
if targetDir == "" {
writeJSON(w, http.StatusOK, map[string]any{"success": false, "error": "Invalid path"})
return
}
indicesAny, _ := data["indices"].([]any)
indicesMap := make(map[float64]bool)
useIndices := false
if len(indicesAny) > 0 {
useIndices = true
for _, idx := range indicesAny {
if f, ok := idx.(float64); ok {
indicesMap[f] = true
}
}
}
var lastRun map[string]any
if err := readJSON(LastRunFile, &lastRun); err != nil {
writeJSON(w, http.StatusOK, map[string]any{"success": false, "error": "No saved run data found"})
return
}
// Python looks at last_run.get('bulk', {}).get('results', [])
bulkAny, ok := lastRun["bulk"].(map[string]any)
if !ok {
writeJSON(w, http.StatusOK, map[string]any{"success": false, "error": "No batch results found"})
return
}
resultsAny, ok := bulkAny["results"].([]any)
if !ok {
writeJSON(w, http.StatusOK, map[string]any{"success": false, "error": "No batch results found"})
return
}
savedCount := 0
var errors []string
for _, resAny := range resultsAny {
res, ok := resAny.(map[string]any)
if !ok {
continue
}
idx, _ := res["index"].(float64)
if useIndices && !indicesMap[idx] {
continue
}
inputFilename, _ := res["inputFilename"].(string)
if inputFilename == "" {
inputFilename = fmt.Sprintf("result_%v", res["index"])
}
contentType, _ := res["contentType"].(string)
ext := filepath.Ext(inputFilename)
if ext == "" {
ct := strings.ToLower(contentType)
if strings.Contains(ct, "image/png") { ext = ".png" } else
if strings.Contains(ct, "image/jpeg") || strings.Contains(ct, "image/jpg") { ext = ".jpg" } else
if strings.Contains(ct, "image/gif") { ext = ".gif" } else
if strings.Contains(ct, "image/webp") { ext = ".webp" } else
if strings.Contains(ct, "image/svg") { ext = ".svg" } else
if strings.Contains(ct, "application/pdf") { ext = ".pdf" } else
if strings.Contains(ct, "application/json") { ext = ".json" } else
if strings.Contains(ct, "text/html") { ext = ".html" } else
if strings.Contains(ct, "text/plain") { ext = ".txt" }
}
finalFilename := inputFilename
if ext != "" && filepath.Ext(inputFilename) == "" {
finalFilename += ext
}
fullPath := filepath.Join(targetDir, filepath.Base(finalFilename))
binaryData, _ := res["binaryData"].(string)
fullContent, hasFullContent := res["fullContent"]
if binaryData != "" {
binaryBytes, err := base64.StdEncoding.DecodeString(binaryData)
if err != nil {
errors = append(errors, fmt.Sprintf("%s: invalid base64", finalFilename))
continue
}
if err := os.WriteFile(fullPath, binaryBytes, 0644); err != nil {
errors = append(errors, fmt.Sprintf("%s: %s", finalFilename, err.Error()))
} else {
savedCount++
}
} else if hasFullContent {
contentStr := ""
switch v := fullContent.(type) {
case string:
contentStr = v
default:
bytes, _ := json.Marshal(v)
contentStr = string(bytes)
}
if err := os.WriteFile(fullPath, []byte(contentStr), 0644); err != nil {
errors = append(errors, fmt.Sprintf("%s: %s", finalFilename, err.Error()))
} else {
savedCount++
}
}
}
slog.Info("saved last run results to folder", "path", targetDir, "saved_count", savedCount, "errors_count", len(errors))
writeJSON(w, http.StatusOK, map[string]any{"success": true, "count": savedCount, "errors": errors})
}
+60
View File
@@ -0,0 +1,60 @@
package main
import (
"log/slog"
"net/http"
"os"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"unified-dashboard/internal/common"
mymiddleware "unified-dashboard/internal/middleware"
)
func registerRoutes(r chi.Router) {
// Serve static files from web/bulk-api-tester/static
workDir, _ := os.Getwd()
fs := http.FileServer(http.Dir(workDir + "/web/bulk-api-tester"))
// API Endpoints
r.Post("/api/proxy-request", proxyRequestHandler)
r.Get("/api/collections", getCollectionsHandler)
r.Post("/api/collections", saveCollectionsHandler)
r.Get("/api/spaces", getSpacesHandler)
r.Post("/api/spaces", saveSpacesHandler)
r.Post("/api/browse_folder", browseFolderHandler)
r.Post("/api/save_folder", saveFolderHandler)
r.Post("/api/save_folder_from_lastrun", saveFolderFromLastrunHandler)
r.Get("/api/last-run", getLastRunHandler)
r.Post("/api/last-run", saveLastRunHandler)
r.Get("/api/docs-data", getDocsDataHandler)
// Serve static assets and index.html
r.Handle("/static/*", fs)
r.Get("/*", func(w http.ResponseWriter, req *http.Request) {
http.ServeFile(w, req, workDir+"/web/bulk-api-tester/index.html")
})
}
func main() {
common.InitLogger("bulk-api-tester")
port := common.EnvOr("PORT", "5006")
scriptName := os.Getenv("SCRIPT_NAME")
r := chi.NewRouter()
r.Use(middleware.RealIP)
r.Use(middleware.RequestID)
r.Use(mymiddleware.RequestLogger)
r.Use(middleware.Recoverer)
if scriptName != "" {
r.Use(common.PrefixMiddleware(scriptName))
}
registerRoutes(r)
slog.Info("starting bulk-api-tester", "port", port, "prefix", scriptName, "pid", os.Getpid())
if err := http.ListenAndServe("127.0.0.1:"+port, r); err != nil {
slog.Error("server failed", "error", err)
}
}
+225
View File
@@ -0,0 +1,225 @@
package main
import (
"bytes"
"crypto/tls"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"log/slog"
"mime/multipart"
"net/http"
"net/url"
"strings"
"time"
)
var httpClient = &http.Client{
Timeout: 200 * time.Second,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // Allow self-signed
},
}
func proxyRequestHandler(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(512 << 20); err != nil { // 512MB
if err == http.ErrNotMultipart {
if err2 := r.ParseForm(); err2 != nil {
writeJSON(w, http.StatusBadRequest, map[string]any{"status_code": 400, "content": "Failed to parse form: " + err2.Error()})
return
}
} else {
writeJSON(w, http.StatusBadRequest, map[string]any{"status_code": 400, "content": "Failed to parse form: " + err.Error()})
return
}
}
targetURL := r.FormValue("url")
method := r.FormValue("method")
if method == "" {
method = "POST"
}
// Parse headers
headersStr := r.FormValue("headers")
var headers map[string]string
if err := json.Unmarshal([]byte(headersStr), &headers); err != nil {
headers = make(map[string]string)
}
// Parse auth
authStr := r.FormValue("auth")
var auth map[string]string
if err := json.Unmarshal([]byte(authStr), &auth); err != nil {
auth = map[string]string{"type": "none"}
}
// Construct request body
var reqBody io.Reader
var reqContentType string
hasFiles := r.MultipartForm != nil && len(r.MultipartForm.File) > 0
if hasFiles {
// Multipart request
bodyBuf := &bytes.Buffer{}
writer := multipart.NewWriter(bodyBuf)
// Form fields
for key, values := range r.MultipartForm.Value {
if isInternalKey(key) {
continue
}
for _, val := range values {
writer.WriteField(key, val)
}
}
// Files
for key, files := range r.MultipartForm.File {
for _, fileHeader := range files {
part, err := writer.CreateFormFile(key, fileHeader.Filename)
if err != nil {
continue
}
file, err := fileHeader.Open()
if err != nil {
continue
}
io.Copy(part, file)
file.Close()
}
}
writer.Close()
reqBody = bodyBuf
reqContentType = writer.FormDataContentType()
} else if r.FormValue("body_type") == "json" {
bodyField := r.FormValue("body")
if bodyField == "" {
bodyField = r.FormValue("_json_body")
}
reqBody = bytes.NewBufferString(bodyField)
if _, ok := headers["Content-Type"]; !ok {
headers["Content-Type"] = "application/json"
}
} else {
// URL encoded form
form := url.Values{}
srcMap := r.Form
if r.MultipartForm != nil {
srcMap = r.MultipartForm.Value
}
for key, values := range srcMap {
if !isInternalKey(key) {
for _, val := range values {
form.Add(key, val)
}
}
}
encoded := form.Encode()
if encoded != "" {
reqBody = bytes.NewBufferString(encoded)
if _, ok := headers["Content-Type"]; !ok {
headers["Content-Type"] = "application/x-www-form-urlencoded"
}
}
}
// Create Request
outReq, err := http.NewRequest(method, targetURL, reqBody)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]any{"status_code": 500, "content": err.Error()})
return
}
// Auth
authType := auth["type"]
if authType == "bearer" {
outReq.Header.Set("Authorization", "Bearer "+auth["token"])
} else if authType == "basic" {
outReq.SetBasicAuth(auth["username"], auth["password"])
} else if authType == "apikey" {
if key, ok := auth["key"]; ok && key != "" {
outReq.Header.Set(key, auth["value"])
}
}
// Set headers
for k, v := range headers {
if hasFiles && strings.ToLower(k) == "content-type" {
continue // skip custom content-type if multipart
}
outReq.Header.Set(k, v)
}
if hasFiles {
outReq.Header.Set("Content-Type", reqContentType)
}
startTime := time.Now()
slog.Info("proxying request", "method", method, "url", targetURL, "has_files", hasFiles)
// Execute
resp, err := httpClient.Do(outReq)
duration := time.Since(startTime).Milliseconds()
if err != nil {
// Timeout or connection error
statusCode := 500
content := "Internal Proxy Error: " + err.Error()
if urlErr, ok := err.(*url.Error); ok && urlErr.Timeout() {
statusCode = 408
content = "Request Timeout"
}
slog.Error("proxy request failed", "url", targetURL, "status", statusCode, "duration_ms", duration, "error", err)
writeJSON(w, http.StatusOK, map[string]any{
"status_code": statusCode,
"content": content,
"headers": map[string]string{},
}) // Python returns 200 wrapper
return
}
defer resp.Body.Close()
slog.Info("proxy request completed", "url", targetURL, "status", resp.StatusCode, "duration_ms", duration)
// Parse Response
respData := map[string]any{
"status_code": resp.StatusCode,
"headers": make(map[string]string),
"content": nil,
"binary_data": nil,
"content_type": resp.Header.Get("Content-Type"),
}
for k, v := range resp.Header {
if len(v) > 0 {
respData["headers"].(map[string]string)[k] = v[0]
}
}
bodyBytes, _ := io.ReadAll(resp.Body)
contentType := strings.ToLower(resp.Header.Get("Content-Type"))
isBinary := strings.Contains(contentType, "image") || strings.Contains(contentType, "pdf") ||
strings.Contains(contentType, "octet-stream") || strings.Contains(contentType, "audio") ||
strings.Contains(contentType, "video") || strings.Contains(contentType, "zip")
if isBinary {
respData["binary_data"] = base64.StdEncoding.EncodeToString(bodyBytes)
respData["content"] = fmt.Sprintf("[Binary Data: %d bytes]", len(bodyBytes))
} else {
respData["content"] = string(bodyBytes)
}
writeJSON(w, http.StatusOK, respData)
}
func isInternalKey(key string) bool {
keys := []string{"url", "method", "headers", "auth", "body_type", "body", "_json_body"}
for _, k := range keys {
if k == key {
return true
}
}
return false
}
+59
View File
@@ -0,0 +1,59 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"sync"
)
// Use RWMutex so concurrent GETs during bulk runs don't block each other.
// Only writes take an exclusive lock.
var storageMutex sync.RWMutex
// atomicWriteJSON writes data to a temporary file and renames it atomically.
// Uses a compact (no indent) encoder to minimize disk I/O size.
func atomicWriteJSON(path string, data any) error {
storageMutex.Lock()
defer storageMutex.Unlock()
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
tmp := path + ".tmp"
f, err := os.Create(tmp)
if err != nil {
return err
}
// Compact encoding — no pretty-print indentation.
// Saves significant I/O on large payloads (collections with many requests).
encoder := json.NewEncoder(f)
if err := encoder.Encode(data); err != nil {
f.Close()
os.Remove(tmp)
return err
}
if err := f.Close(); err != nil {
os.Remove(tmp)
return err
}
return os.Rename(tmp, path)
}
func readJSON(path string, v any) error {
storageMutex.RLock()
defer storageMutex.RUnlock()
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
return json.NewDecoder(f).Decode(v)
}