Files
Unified_Dashboard_Suite_Go/cmd/json-image-downloader/main.go
T

553 lines
14 KiB
Go

package main
import (
"archive/zip"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"unified-dashboard/internal/common"
mymiddleware "unified-dashboard/internal/middleware"
)
type Progress struct {
Status string `json:"status"`
Total int `json:"total"`
Selected int `json:"selected"`
Current int `json:"current"`
Failed int `json:"failed"`
Message string `json:"message"`
}
type ImageItem struct {
URL string `json:"url"`
Path string `json:"path"`
}
var (
progressMap = make(map[string]*Progress)
controlMap = make(map[string]string)
mapLock sync.Mutex
client = &http.Client{Timeout: 20 * time.Second}
)
func setProgress(sid string, p *Progress) {
mapLock.Lock()
defer mapLock.Unlock()
if existing, ok := progressMap[sid]; ok {
if p.Status != "" { existing.Status = p.Status }
if p.Message != "" { existing.Message = p.Message }
if p.Total > 0 { existing.Total = p.Total }
if p.Selected > 0 { existing.Selected = p.Selected }
existing.Current = p.Current
existing.Failed = p.Failed
} else {
progressMap[sid] = p
}
}
func getControl(sid string) string {
mapLock.Lock()
defer mapLock.Unlock()
if ctrl, ok := controlMap[sid]; ok {
return ctrl
}
return "running"
}
func isImageURL(val string) bool {
lower := strings.ToLower(val)
if !strings.HasPrefix(lower, "http://") && !strings.HasPrefix(lower, "https://") {
return false
}
u, err := url.Parse(val)
if err != nil {
return false
}
path := strings.ToLower(u.Path)
exts := []string{".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", ".tiff", ".avif"}
for _, ext := range exts {
if strings.HasSuffix(path, ext) {
return true
}
}
// Also detect image endpoints like /image/png or query params like format=png
for _, ext := range exts {
rawExt := strings.TrimPrefix(ext, ".")
if strings.HasSuffix(path, "/"+rawExt) || strings.Contains(strings.ToLower(u.RawQuery), "format="+rawExt) {
return true
}
}
return false
}
func analyzeJSON(data interface{}, path string, results map[string][]ImageItem) {
switch v := data.(type) {
case map[string]interface{}:
for key, val := range v {
cur := key
if path != "" {
cur = path + "." + key
}
if strVal, ok := val.(string); ok && isImageURL(strVal) {
results[key] = append(results[key], ImageItem{URL: strVal, Path: cur})
} else {
analyzeJSON(val, cur, results)
}
}
case []interface{}:
for i, val := range v {
cur := fmt.Sprintf("%s[%d]", path, i)
if strVal, ok := val.(string); ok && isImageURL(strVal) {
results["_list_item"] = append(results["_list_item"], ImageItem{URL: strVal, Path: cur})
} else {
analyzeJSON(val, cur, results)
}
}
}
}
func extractImages(data interface{}, selectedFields map[string]bool, maintainStructure bool, base string, out *[]ImageItem) {
switch v := data.(type) {
case map[string]interface{}:
for key, val := range v {
cur := key
if base != "" {
cur = filepath.Join(base, key)
}
if strVal, ok := val.(string); ok && selectedFields[key] && isImageURL(strVal) {
u, _ := url.Parse(strVal)
fname := filepath.Base(u.Path)
if fname == "" || fname == "/" || fname == "." {
fname = fmt.Sprintf("image_%d.jpg", len(*out))
}
savePath := fmt.Sprintf("%s_%d_%s", key, len(*out), fname)
if maintainStructure {
savePath = filepath.Join(cur, fname)
}
*out = append(*out, ImageItem{URL: strVal, Path: savePath})
} else {
extractImages(val, selectedFields, maintainStructure, cur, out)
}
}
case []interface{}:
for i, val := range v {
cur := filepath.Join(base, fmt.Sprintf("%d", i))
extractImages(val, selectedFields, maintainStructure, cur, out)
}
}
}
func writeJSON(w http.ResponseWriter, status int, data any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
json.NewEncoder(w).Encode(data)
}
func main() {
common.InitLogger("json-image-downloader")
port := common.EnvOr("PORT", "5004")
scriptName := os.Getenv("SCRIPT_NAME")
os.MkdirAll("uploads", 0755)
os.MkdirAll("downloads", 0755)
r := chi.NewRouter()
r.Use(middleware.RealIP)
r.Use(middleware.RequestID)
r.Use(mymiddleware.RequestLogger)
r.Use(middleware.Recoverer)
if scriptName != "" {
r.Use(common.PrefixMiddleware(scriptName))
}
workDir, _ := os.Getwd()
webDir := filepath.Join(workDir, "web", "json-image-downloader")
r.Get("/", func(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, filepath.Join(webDir, "index.html"))
})
r.Handle("/static/*", http.StripPrefix("/static/", http.FileServer(http.Dir(filepath.Join(webDir, "static")))))
r.Post("/analyze", analyzeHandler)
r.Post("/preview", previewHandler)
r.Get("/download-progress/{session_id}", downloadProgressHandler)
r.Post("/control/{session_id}", controlHandler)
r.Post("/download", downloadHandler)
r.Post("/cleanup", cleanupHandler)
slog.Info("starting json-image-downloader", "port", port, "pid", os.Getpid())
if err := http.ListenAndServe(":"+port, r); err != nil {
slog.Error("server failed", "error", err)
}
}
func analyzeHandler(w http.ResponseWriter, r *http.Request) {
var data interface{}
ct := strings.ToLower(r.Header.Get("Content-Type"))
if strings.Contains(ct, "application/json") {
var reqBody struct {
JSONData any `json:"json_data"`
}
rawBytes, _ := io.ReadAll(r.Body)
if err := json.Unmarshal(rawBytes, &reqBody); err == nil && reqBody.JSONData != nil {
data = reqBody.JSONData
} else {
if err := json.Unmarshal(rawBytes, &data); err != nil {
writeJSON(w, 400, map[string]string{"error": "Invalid JSON: " + err.Error()})
return
}
}
} else {
_ = r.ParseMultipartForm(100 << 20)
file, _, err := r.FormFile("file")
if err == nil {
defer file.Close()
if err := json.NewDecoder(file).Decode(&data); err != nil {
writeJSON(w, 400, map[string]string{"error": "Invalid JSON in file: " + err.Error()})
return
}
} else if jsonStr := r.FormValue("json_data"); jsonStr != "" {
if err := json.Unmarshal([]byte(jsonStr), &data); err != nil {
writeJSON(w, 400, map[string]string{"error": "Invalid JSON in json_data: " + err.Error()})
return
}
} else {
writeJSON(w, 400, map[string]string{"error": "No JSON file or payload provided"})
return
}
}
if data == nil {
writeJSON(w, 400, map[string]string{"error": "Empty data"})
return
}
results := make(map[string][]ImageItem)
analyzeJSON(data, "", results)
if len(results) == 0 {
writeJSON(w, 400, map[string]string{"error": "No image URLs found in this JSON"})
return
}
fieldsSummary := make(map[string]interface{})
total := 0
for field, entries := range results {
total += len(entries)
sample := []string{}
for i := 0; i < len(entries) && i < 3; i++ {
sample = append(sample, entries[i].URL)
}
fieldsSummary[field] = map[string]interface{}{
"count": len(entries),
"sample_urls": sample,
}
}
sessionID := fmt.Sprintf("%d", time.Now().UnixNano())
uploadPath := filepath.Join("uploads", sessionID+".json")
f, _ := os.Create(uploadPath)
json.NewEncoder(f).Encode(data)
f.Close()
writeJSON(w, 200, map[string]interface{}{
"success": true,
"session_id": sessionID,
"fields": fieldsSummary,
"total_images": total,
})
}
func previewHandler(w http.ResponseWriter, r *http.Request) {
var req struct {
SessionID string `json:"session_id"`
Fields []string `json:"fields"`
MaintainStructure bool `json:"maintain_structure"`
}
json.NewDecoder(r.Body).Decode(&req)
if req.SessionID == "" || len(req.Fields) == 0 {
writeJSON(w, 400, map[string]string{"error": "Missing session_id or fields"})
return
}
uploadPath := filepath.Join("uploads", req.SessionID+".json")
f, err := os.Open(uploadPath)
if err != nil {
writeJSON(w, 404, map[string]string{"error": "Session expired"})
return
}
var data interface{}
json.NewDecoder(f).Decode(&data)
f.Close()
selected := make(map[string]bool)
for _, f := range req.Fields {
selected[f] = true
}
var images []ImageItem
extractImages(data, selected, req.MaintainStructure, "", &images)
var fileList []map[string]interface{}
folders := make(map[string]bool)
for i, img := range images {
folder := filepath.Dir(img.Path)
if folder == "." {
folder = ""
}
if folder != "" {
folders[folder] = true
}
fileList = append(fileList, map[string]interface{}{
"id": i,
"url": img.URL,
"original_path": img.Path,
"folder": folder,
"filename": filepath.Base(img.Path),
"extension": filepath.Ext(img.Path),
})
}
writeJSON(w, 200, map[string]interface{}{
"success": true,
"files": fileList,
"images": fileList,
"stats": map[string]int{
"total_files": len(fileList),
"total_folders": len(folders),
},
})
}
func downloadProgressHandler(w http.ResponseWriter, r *http.Request) {
sessionID := chi.URLParam(r, "session_id")
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
flusher, ok := w.(http.Flusher)
if !ok { return }
for i := 0; i < 60; i++ {
mapLock.Lock()
_, exists := progressMap[sessionID]
mapLock.Unlock()
if exists {
break
}
time.Sleep(500 * time.Millisecond)
}
for {
mapLock.Lock()
prog, exists := progressMap[sessionID]
if !exists {
mapLock.Unlock()
fmt.Fprintf(w, "data: %s\n\n", `{"status":"error","message":"session not found"}`)
flusher.Flush()
break
}
// Copy prog to avoid race
pCopy := *prog
mapLock.Unlock()
data, _ := json.Marshal(pCopy)
fmt.Fprintf(w, "data: %s\n\n", data)
flusher.Flush()
if pCopy.Status == "completed" || pCopy.Status == "stopped" || pCopy.Status == "error" {
break
}
time.Sleep(350 * time.Millisecond)
}
}
func controlHandler(w http.ResponseWriter, r *http.Request) {
sessionID := chi.URLParam(r, "session_id")
var data struct {
Action string `json:"action"`
}
json.NewDecoder(r.Body).Decode(&data)
if data.Action != "running" && data.Action != "paused" && data.Action != "stopped" {
writeJSON(w, 400, map[string]string{"error": "Invalid action"})
return
}
mapLock.Lock()
controlMap[sessionID] = data.Action
if prog, ok := progressMap[sessionID]; ok {
if data.Action == "paused" {
prog.Message = "⏸ Paused"
} else if data.Action == "running" {
prog.Message = "▶ Resumed"
} else if data.Action == "stopped" {
prog.Message = "⏹ Stopping…"
}
}
mapLock.Unlock()
writeJSON(w, 200, map[string]interface{}{"success": true, "action": data.Action})
}
func downloadHandler(w http.ResponseWriter, r *http.Request) {
var req struct {
SessionID string `json:"session_id"`
Fields []string `json:"fields"`
MaintainStructure bool `json:"maintain_structure"`
FileMapping []map[string]interface{} `json:"file_mapping"`
}
json.NewDecoder(r.Body).Decode(&req)
if req.SessionID == "" {
writeJSON(w, 400, map[string]string{"error": "Missing session_id"})
return
}
var images []ImageItem
if len(req.FileMapping) > 0 {
for _, m := range req.FileMapping {
u, _ := m["url"].(string)
s, _ := m["save_as"].(string)
if u != "" && s != "" {
images = append(images, ImageItem{URL: u, Path: s})
}
}
} else {
uploadPath := filepath.Join("uploads", req.SessionID+".json")
f, err := os.Open(uploadPath)
if err != nil {
writeJSON(w, 404, map[string]string{"error": "Session expired"})
return
}
var data interface{}
json.NewDecoder(f).Decode(&data)
f.Close()
selected := make(map[string]bool)
for _, f := range req.Fields { selected[f] = true }
extractImages(data, selected, req.MaintainStructure, "", &images)
}
if len(images) == 0 {
writeJSON(w, 404, map[string]string{"error": "No images in selection"})
return
}
total := len(images)
setProgress(req.SessionID, &Progress{
Status: "starting", Total: total, Selected: total, Current: 0, Failed: 0, Message: "Preparing…",
})
mapLock.Lock()
controlMap[req.SessionID] = "running"
mapLock.Unlock()
zipPath := filepath.Join("downloads", req.SessionID+".zip")
zf, _ := os.Create(zipPath)
zw := zip.NewWriter(zf)
var zipLock sync.Mutex
setProgress(req.SessionID, &Progress{Status: "downloading", Message: "Downloading…"})
taskCh := make(chan ImageItem, total)
for _, img := range images {
taskCh <- img
}
close(taskCh)
var wg sync.WaitGroup
workers := 16
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for img := range taskCh {
for getControl(req.SessionID) == "paused" {
time.Sleep(250 * time.Millisecond)
}
if getControl(req.SessionID) == "stopped" {
break
}
resp, err := client.Get(img.URL)
ok := false
if err == nil && resp.StatusCode == 200 {
data, _ := io.ReadAll(resp.Body)
resp.Body.Close()
zipLock.Lock()
f, _ := zw.Create(img.Path)
f.Write(data)
zipLock.Unlock()
ok = true
}
mapLock.Lock()
if p, exists := progressMap[req.SessionID]; exists {
if ok { p.Current++ } else { p.Failed++ }
}
mapLock.Unlock()
}
}()
}
wg.Wait()
zw.Close()
zf.Close()
finalCtrl := getControl(req.SessionID)
finalStatus := "completed"
dlName := "images.zip"
if finalCtrl == "stopped" {
finalStatus = "stopped"
dlName = "images_partial.zip"
}
setProgress(req.SessionID, &Progress{Status: finalStatus, Message: "Packaging ZIP…"})
os.Remove(filepath.Join("uploads", req.SessionID+".json"))
stat, _ := os.Stat(zipPath)
if stat == nil || stat.Size() == 0 {
writeJSON(w, 500, map[string]string{"error": "No images were downloaded"})
return
}
w.Header().Set("Content-Type", "application/zip")
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, dlName))
http.ServeFile(w, r, zipPath)
os.Remove(zipPath)
mapLock.Lock()
delete(progressMap, req.SessionID)
delete(controlMap, req.SessionID)
mapLock.Unlock()
}
func cleanupHandler(w http.ResponseWriter, r *http.Request) {
var req struct {
SessionID string `json:"session_id"`
}
json.NewDecoder(r.Body).Decode(&req)
if req.SessionID != "" {
os.Remove(filepath.Join("uploads", req.SessionID+".json"))
os.Remove(filepath.Join("downloads", req.SessionID+".zip"))
mapLock.Lock()
delete(progressMap, req.SessionID)
delete(controlMap, req.SessionID)
mapLock.Unlock()
}
writeJSON(w, 200, map[string]bool{"success": true})
}