package main import ( "archive/zip" "encoding/json" "fmt" "io" "log/slog" "net/http" "net/url" "os" "path/filepath" "strings" "sync" "time" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "unified-dashboard/internal/common" mymiddleware "unified-dashboard/internal/middleware" ) type Progress struct { Status string `json:"status"` Total int `json:"total"` Selected int `json:"selected"` Current int `json:"current"` Failed int `json:"failed"` Message string `json:"message"` } type ImageItem struct { URL string `json:"url"` Path string `json:"path"` } var ( progressMap = make(map[string]*Progress) controlMap = make(map[string]string) mapLock sync.Mutex client = &http.Client{Timeout: 20 * time.Second} ) func setProgress(sid string, p *Progress) { mapLock.Lock() defer mapLock.Unlock() if existing, ok := progressMap[sid]; ok { if p.Status != "" { existing.Status = p.Status } if p.Message != "" { existing.Message = p.Message } if p.Total > 0 { existing.Total = p.Total } if p.Selected > 0 { existing.Selected = p.Selected } existing.Current = p.Current existing.Failed = p.Failed } else { progressMap[sid] = p } } func getControl(sid string) string { mapLock.Lock() defer mapLock.Unlock() if ctrl, ok := controlMap[sid]; ok { return ctrl } return "running" } func isImageURL(val string) bool { lower := strings.ToLower(val) if !strings.HasPrefix(lower, "http://") && !strings.HasPrefix(lower, "https://") { return false } u, err := url.Parse(val) if err != nil { return false } path := strings.ToLower(u.Path) exts := []string{".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", ".tiff", ".avif"} for _, ext := range exts { if strings.HasSuffix(path, ext) { return true } } // Also detect image endpoints like /image/png or query params like format=png for _, ext := range exts { rawExt := strings.TrimPrefix(ext, ".") if strings.HasSuffix(path, "/"+rawExt) || strings.Contains(strings.ToLower(u.RawQuery), "format="+rawExt) { return true } } return false } func analyzeJSON(data interface{}, path string, results map[string][]ImageItem) { switch v := data.(type) { case map[string]interface{}: for key, val := range v { cur := key if path != "" { cur = path + "." + key } if strVal, ok := val.(string); ok && isImageURL(strVal) { results[key] = append(results[key], ImageItem{URL: strVal, Path: cur}) } else { analyzeJSON(val, cur, results) } } case []interface{}: for i, val := range v { cur := fmt.Sprintf("%s[%d]", path, i) if strVal, ok := val.(string); ok && isImageURL(strVal) { results["_list_item"] = append(results["_list_item"], ImageItem{URL: strVal, Path: cur}) } else { analyzeJSON(val, cur, results) } } } } func extractImages(data interface{}, selectedFields map[string]bool, maintainStructure bool, base string, out *[]ImageItem) { switch v := data.(type) { case map[string]interface{}: for key, val := range v { cur := key if base != "" { cur = filepath.Join(base, key) } if strVal, ok := val.(string); ok && selectedFields[key] && isImageURL(strVal) { u, _ := url.Parse(strVal) fname := filepath.Base(u.Path) if fname == "" || fname == "/" || fname == "." { fname = fmt.Sprintf("image_%d.jpg", len(*out)) } savePath := fmt.Sprintf("%s_%d_%s", key, len(*out), fname) if maintainStructure { savePath = filepath.Join(cur, fname) } *out = append(*out, ImageItem{URL: strVal, Path: savePath}) } else { extractImages(val, selectedFields, maintainStructure, cur, out) } } case []interface{}: for i, val := range v { cur := filepath.Join(base, fmt.Sprintf("%d", i)) extractImages(val, selectedFields, maintainStructure, cur, out) } } } func writeJSON(w http.ResponseWriter, status int, data any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) json.NewEncoder(w).Encode(data) } func main() { common.InitLogger("json-image-downloader") port := common.EnvOr("PORT", "5004") scriptName := os.Getenv("SCRIPT_NAME") os.MkdirAll("uploads", 0755) os.MkdirAll("downloads", 0755) r := chi.NewRouter() r.Use(middleware.RealIP) r.Use(middleware.RequestID) r.Use(mymiddleware.RequestLogger) r.Use(middleware.Recoverer) if scriptName != "" { r.Use(common.PrefixMiddleware(scriptName)) } workDir, _ := os.Getwd() webDir := filepath.Join(workDir, "web", "json-image-downloader") r.Get("/", func(w http.ResponseWriter, r *http.Request) { http.ServeFile(w, r, filepath.Join(webDir, "index.html")) }) r.Handle("/static/*", http.StripPrefix("/static/", http.FileServer(http.Dir(filepath.Join(webDir, "static"))))) r.Post("/analyze", analyzeHandler) r.Post("/preview", previewHandler) r.Get("/download-progress/{session_id}", downloadProgressHandler) r.Post("/control/{session_id}", controlHandler) r.Post("/download", downloadHandler) r.Post("/cleanup", cleanupHandler) slog.Info("starting json-image-downloader", "port", port, "pid", os.Getpid()) if err := http.ListenAndServe(":"+port, r); err != nil { slog.Error("server failed", "error", err) } } func analyzeHandler(w http.ResponseWriter, r *http.Request) { var data interface{} ct := strings.ToLower(r.Header.Get("Content-Type")) if strings.Contains(ct, "application/json") { var reqBody struct { JSONData any `json:"json_data"` } rawBytes, _ := io.ReadAll(r.Body) if err := json.Unmarshal(rawBytes, &reqBody); err == nil && reqBody.JSONData != nil { data = reqBody.JSONData } else { if err := json.Unmarshal(rawBytes, &data); err != nil { writeJSON(w, 400, map[string]string{"error": "Invalid JSON: " + err.Error()}) return } } } else { _ = r.ParseMultipartForm(100 << 20) file, _, err := r.FormFile("file") if err == nil { defer file.Close() if err := json.NewDecoder(file).Decode(&data); err != nil { writeJSON(w, 400, map[string]string{"error": "Invalid JSON in file: " + err.Error()}) return } } else if jsonStr := r.FormValue("json_data"); jsonStr != "" { if err := json.Unmarshal([]byte(jsonStr), &data); err != nil { writeJSON(w, 400, map[string]string{"error": "Invalid JSON in json_data: " + err.Error()}) return } } else { writeJSON(w, 400, map[string]string{"error": "No JSON file or payload provided"}) return } } if data == nil { writeJSON(w, 400, map[string]string{"error": "Empty data"}) return } results := make(map[string][]ImageItem) analyzeJSON(data, "", results) if len(results) == 0 { writeJSON(w, 400, map[string]string{"error": "No image URLs found in this JSON"}) return } fieldsSummary := make(map[string]interface{}) total := 0 for field, entries := range results { total += len(entries) sample := []string{} for i := 0; i < len(entries) && i < 3; i++ { sample = append(sample, entries[i].URL) } fieldsSummary[field] = map[string]interface{}{ "count": len(entries), "sample_urls": sample, } } sessionID := fmt.Sprintf("%d", time.Now().UnixNano()) uploadPath := filepath.Join("uploads", sessionID+".json") f, _ := os.Create(uploadPath) json.NewEncoder(f).Encode(data) f.Close() writeJSON(w, 200, map[string]interface{}{ "success": true, "session_id": sessionID, "fields": fieldsSummary, "total_images": total, }) } func previewHandler(w http.ResponseWriter, r *http.Request) { var req struct { SessionID string `json:"session_id"` Fields []string `json:"fields"` MaintainStructure bool `json:"maintain_structure"` } json.NewDecoder(r.Body).Decode(&req) if req.SessionID == "" || len(req.Fields) == 0 { writeJSON(w, 400, map[string]string{"error": "Missing session_id or fields"}) return } uploadPath := filepath.Join("uploads", req.SessionID+".json") f, err := os.Open(uploadPath) if err != nil { writeJSON(w, 404, map[string]string{"error": "Session expired"}) return } var data interface{} json.NewDecoder(f).Decode(&data) f.Close() selected := make(map[string]bool) for _, f := range req.Fields { selected[f] = true } var images []ImageItem extractImages(data, selected, req.MaintainStructure, "", &images) var fileList []map[string]interface{} folders := make(map[string]bool) for i, img := range images { folder := filepath.Dir(img.Path) if folder == "." { folder = "" } if folder != "" { folders[folder] = true } fileList = append(fileList, map[string]interface{}{ "id": i, "url": img.URL, "original_path": img.Path, "folder": folder, "filename": filepath.Base(img.Path), "extension": filepath.Ext(img.Path), }) } writeJSON(w, 200, map[string]interface{}{ "success": true, "files": fileList, "images": fileList, "stats": map[string]int{ "total_files": len(fileList), "total_folders": len(folders), }, }) } func downloadProgressHandler(w http.ResponseWriter, r *http.Request) { sessionID := chi.URLParam(r, "session_id") w.Header().Set("Content-Type", "text/event-stream") w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") flusher, ok := w.(http.Flusher) if !ok { return } for i := 0; i < 60; i++ { mapLock.Lock() _, exists := progressMap[sessionID] mapLock.Unlock() if exists { break } time.Sleep(500 * time.Millisecond) } for { mapLock.Lock() prog, exists := progressMap[sessionID] if !exists { mapLock.Unlock() fmt.Fprintf(w, "data: %s\n\n", `{"status":"error","message":"session not found"}`) flusher.Flush() break } // Copy prog to avoid race pCopy := *prog mapLock.Unlock() data, _ := json.Marshal(pCopy) fmt.Fprintf(w, "data: %s\n\n", data) flusher.Flush() if pCopy.Status == "completed" || pCopy.Status == "stopped" || pCopy.Status == "error" { break } time.Sleep(350 * time.Millisecond) } } func controlHandler(w http.ResponseWriter, r *http.Request) { sessionID := chi.URLParam(r, "session_id") var data struct { Action string `json:"action"` } json.NewDecoder(r.Body).Decode(&data) if data.Action != "running" && data.Action != "paused" && data.Action != "stopped" { writeJSON(w, 400, map[string]string{"error": "Invalid action"}) return } mapLock.Lock() controlMap[sessionID] = data.Action if prog, ok := progressMap[sessionID]; ok { if data.Action == "paused" { prog.Message = "⏸ Paused" } else if data.Action == "running" { prog.Message = "▶ Resumed" } else if data.Action == "stopped" { prog.Message = "⏹ Stopping…" } } mapLock.Unlock() writeJSON(w, 200, map[string]interface{}{"success": true, "action": data.Action}) } func downloadHandler(w http.ResponseWriter, r *http.Request) { var req struct { SessionID string `json:"session_id"` Fields []string `json:"fields"` MaintainStructure bool `json:"maintain_structure"` FileMapping []map[string]interface{} `json:"file_mapping"` } json.NewDecoder(r.Body).Decode(&req) if req.SessionID == "" { writeJSON(w, 400, map[string]string{"error": "Missing session_id"}) return } var images []ImageItem if len(req.FileMapping) > 0 { for _, m := range req.FileMapping { u, _ := m["url"].(string) s, _ := m["save_as"].(string) if u != "" && s != "" { images = append(images, ImageItem{URL: u, Path: s}) } } } else { uploadPath := filepath.Join("uploads", req.SessionID+".json") f, err := os.Open(uploadPath) if err != nil { writeJSON(w, 404, map[string]string{"error": "Session expired"}) return } var data interface{} json.NewDecoder(f).Decode(&data) f.Close() selected := make(map[string]bool) for _, f := range req.Fields { selected[f] = true } extractImages(data, selected, req.MaintainStructure, "", &images) } if len(images) == 0 { writeJSON(w, 404, map[string]string{"error": "No images in selection"}) return } total := len(images) setProgress(req.SessionID, &Progress{ Status: "starting", Total: total, Selected: total, Current: 0, Failed: 0, Message: "Preparing…", }) mapLock.Lock() controlMap[req.SessionID] = "running" mapLock.Unlock() zipPath := filepath.Join("downloads", req.SessionID+".zip") zf, _ := os.Create(zipPath) zw := zip.NewWriter(zf) var zipLock sync.Mutex setProgress(req.SessionID, &Progress{Status: "downloading", Message: "Downloading…"}) taskCh := make(chan ImageItem, total) for _, img := range images { taskCh <- img } close(taskCh) var wg sync.WaitGroup workers := 16 for i := 0; i < workers; i++ { wg.Add(1) go func() { defer wg.Done() for img := range taskCh { for getControl(req.SessionID) == "paused" { time.Sleep(250 * time.Millisecond) } if getControl(req.SessionID) == "stopped" { break } resp, err := client.Get(img.URL) ok := false if err == nil && resp.StatusCode == 200 { data, _ := io.ReadAll(resp.Body) resp.Body.Close() zipLock.Lock() f, _ := zw.Create(img.Path) f.Write(data) zipLock.Unlock() ok = true } mapLock.Lock() if p, exists := progressMap[req.SessionID]; exists { if ok { p.Current++ } else { p.Failed++ } } mapLock.Unlock() } }() } wg.Wait() zw.Close() zf.Close() finalCtrl := getControl(req.SessionID) finalStatus := "completed" dlName := "images.zip" if finalCtrl == "stopped" { finalStatus = "stopped" dlName = "images_partial.zip" } setProgress(req.SessionID, &Progress{Status: finalStatus, Message: "Packaging ZIP…"}) os.Remove(filepath.Join("uploads", req.SessionID+".json")) stat, _ := os.Stat(zipPath) if stat == nil || stat.Size() == 0 { writeJSON(w, 500, map[string]string{"error": "No images were downloaded"}) return } w.Header().Set("Content-Type", "application/zip") w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, dlName)) http.ServeFile(w, r, zipPath) os.Remove(zipPath) mapLock.Lock() delete(progressMap, req.SessionID) delete(controlMap, req.SessionID) mapLock.Unlock() } func cleanupHandler(w http.ResponseWriter, r *http.Request) { var req struct { SessionID string `json:"session_id"` } json.NewDecoder(r.Body).Decode(&req) if req.SessionID != "" { os.Remove(filepath.Join("uploads", req.SessionID+".json")) os.Remove(filepath.Join("downloads", req.SessionID+".zip")) mapLock.Lock() delete(progressMap, req.SessionID) delete(controlMap, req.SessionID) mapLock.Unlock() } writeJSON(w, 200, map[string]bool{"success": true}) }