553 lines
14 KiB
Go
553 lines
14 KiB
Go
package main
|
|
|
|
import (
|
|
"archive/zip"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/go-chi/chi/v5"
|
|
"github.com/go-chi/chi/v5/middleware"
|
|
"unified-dashboard/internal/common"
|
|
mymiddleware "unified-dashboard/internal/middleware"
|
|
)
|
|
|
|
type Progress struct {
|
|
Status string `json:"status"`
|
|
Total int `json:"total"`
|
|
Selected int `json:"selected"`
|
|
Current int `json:"current"`
|
|
Failed int `json:"failed"`
|
|
Message string `json:"message"`
|
|
}
|
|
|
|
type ImageItem struct {
|
|
URL string `json:"url"`
|
|
Path string `json:"path"`
|
|
}
|
|
|
|
var (
|
|
progressMap = make(map[string]*Progress)
|
|
controlMap = make(map[string]string)
|
|
mapLock sync.Mutex
|
|
client = &http.Client{Timeout: 20 * time.Second}
|
|
)
|
|
|
|
func setProgress(sid string, p *Progress) {
|
|
mapLock.Lock()
|
|
defer mapLock.Unlock()
|
|
if existing, ok := progressMap[sid]; ok {
|
|
if p.Status != "" { existing.Status = p.Status }
|
|
if p.Message != "" { existing.Message = p.Message }
|
|
if p.Total > 0 { existing.Total = p.Total }
|
|
if p.Selected > 0 { existing.Selected = p.Selected }
|
|
existing.Current = p.Current
|
|
existing.Failed = p.Failed
|
|
} else {
|
|
progressMap[sid] = p
|
|
}
|
|
}
|
|
|
|
func getControl(sid string) string {
|
|
mapLock.Lock()
|
|
defer mapLock.Unlock()
|
|
if ctrl, ok := controlMap[sid]; ok {
|
|
return ctrl
|
|
}
|
|
return "running"
|
|
}
|
|
|
|
func isImageURL(val string) bool {
|
|
lower := strings.ToLower(val)
|
|
if !strings.HasPrefix(lower, "http://") && !strings.HasPrefix(lower, "https://") {
|
|
return false
|
|
}
|
|
u, err := url.Parse(val)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
path := strings.ToLower(u.Path)
|
|
exts := []string{".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", ".tiff", ".avif"}
|
|
for _, ext := range exts {
|
|
if strings.HasSuffix(path, ext) {
|
|
return true
|
|
}
|
|
}
|
|
// Also detect image endpoints like /image/png or query params like format=png
|
|
for _, ext := range exts {
|
|
rawExt := strings.TrimPrefix(ext, ".")
|
|
if strings.HasSuffix(path, "/"+rawExt) || strings.Contains(strings.ToLower(u.RawQuery), "format="+rawExt) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func analyzeJSON(data interface{}, path string, results map[string][]ImageItem) {
|
|
switch v := data.(type) {
|
|
case map[string]interface{}:
|
|
for key, val := range v {
|
|
cur := key
|
|
if path != "" {
|
|
cur = path + "." + key
|
|
}
|
|
if strVal, ok := val.(string); ok && isImageURL(strVal) {
|
|
results[key] = append(results[key], ImageItem{URL: strVal, Path: cur})
|
|
} else {
|
|
analyzeJSON(val, cur, results)
|
|
}
|
|
}
|
|
case []interface{}:
|
|
for i, val := range v {
|
|
cur := fmt.Sprintf("%s[%d]", path, i)
|
|
if strVal, ok := val.(string); ok && isImageURL(strVal) {
|
|
results["_list_item"] = append(results["_list_item"], ImageItem{URL: strVal, Path: cur})
|
|
} else {
|
|
analyzeJSON(val, cur, results)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func extractImages(data interface{}, selectedFields map[string]bool, maintainStructure bool, base string, out *[]ImageItem) {
|
|
switch v := data.(type) {
|
|
case map[string]interface{}:
|
|
for key, val := range v {
|
|
cur := key
|
|
if base != "" {
|
|
cur = filepath.Join(base, key)
|
|
}
|
|
if strVal, ok := val.(string); ok && selectedFields[key] && isImageURL(strVal) {
|
|
u, _ := url.Parse(strVal)
|
|
fname := filepath.Base(u.Path)
|
|
if fname == "" || fname == "/" || fname == "." {
|
|
fname = fmt.Sprintf("image_%d.jpg", len(*out))
|
|
}
|
|
savePath := fmt.Sprintf("%s_%d_%s", key, len(*out), fname)
|
|
if maintainStructure {
|
|
savePath = filepath.Join(cur, fname)
|
|
}
|
|
*out = append(*out, ImageItem{URL: strVal, Path: savePath})
|
|
} else {
|
|
extractImages(val, selectedFields, maintainStructure, cur, out)
|
|
}
|
|
}
|
|
case []interface{}:
|
|
for i, val := range v {
|
|
cur := filepath.Join(base, fmt.Sprintf("%d", i))
|
|
extractImages(val, selectedFields, maintainStructure, cur, out)
|
|
}
|
|
}
|
|
}
|
|
|
|
func writeJSON(w http.ResponseWriter, status int, data any) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(status)
|
|
json.NewEncoder(w).Encode(data)
|
|
}
|
|
|
|
func main() {
|
|
common.InitLogger("json-image-downloader")
|
|
port := common.EnvOr("PORT", "5004")
|
|
scriptName := os.Getenv("SCRIPT_NAME")
|
|
|
|
os.MkdirAll("uploads", 0755)
|
|
os.MkdirAll("downloads", 0755)
|
|
|
|
r := chi.NewRouter()
|
|
r.Use(middleware.RealIP)
|
|
r.Use(middleware.RequestID)
|
|
r.Use(mymiddleware.RequestLogger)
|
|
r.Use(middleware.Recoverer)
|
|
if scriptName != "" {
|
|
r.Use(common.PrefixMiddleware(scriptName))
|
|
}
|
|
|
|
workDir, _ := os.Getwd()
|
|
webDir := filepath.Join(workDir, "web", "json-image-downloader")
|
|
|
|
r.Get("/", func(w http.ResponseWriter, r *http.Request) {
|
|
http.ServeFile(w, r, filepath.Join(webDir, "index.html"))
|
|
})
|
|
r.Handle("/static/*", http.StripPrefix("/static/", http.FileServer(http.Dir(filepath.Join(webDir, "static")))))
|
|
|
|
r.Post("/analyze", analyzeHandler)
|
|
r.Post("/preview", previewHandler)
|
|
r.Get("/download-progress/{session_id}", downloadProgressHandler)
|
|
r.Post("/control/{session_id}", controlHandler)
|
|
r.Post("/download", downloadHandler)
|
|
r.Post("/cleanup", cleanupHandler)
|
|
|
|
slog.Info("starting json-image-downloader", "port", port, "pid", os.Getpid())
|
|
if err := http.ListenAndServe(":"+port, r); err != nil {
|
|
slog.Error("server failed", "error", err)
|
|
}
|
|
}
|
|
|
|
func analyzeHandler(w http.ResponseWriter, r *http.Request) {
|
|
var data interface{}
|
|
ct := strings.ToLower(r.Header.Get("Content-Type"))
|
|
|
|
if strings.Contains(ct, "application/json") {
|
|
var reqBody struct {
|
|
JSONData any `json:"json_data"`
|
|
}
|
|
rawBytes, _ := io.ReadAll(r.Body)
|
|
if err := json.Unmarshal(rawBytes, &reqBody); err == nil && reqBody.JSONData != nil {
|
|
data = reqBody.JSONData
|
|
} else {
|
|
if err := json.Unmarshal(rawBytes, &data); err != nil {
|
|
writeJSON(w, 400, map[string]string{"error": "Invalid JSON: " + err.Error()})
|
|
return
|
|
}
|
|
}
|
|
} else {
|
|
_ = r.ParseMultipartForm(100 << 20)
|
|
file, _, err := r.FormFile("file")
|
|
if err == nil {
|
|
defer file.Close()
|
|
if err := json.NewDecoder(file).Decode(&data); err != nil {
|
|
writeJSON(w, 400, map[string]string{"error": "Invalid JSON in file: " + err.Error()})
|
|
return
|
|
}
|
|
} else if jsonStr := r.FormValue("json_data"); jsonStr != "" {
|
|
if err := json.Unmarshal([]byte(jsonStr), &data); err != nil {
|
|
writeJSON(w, 400, map[string]string{"error": "Invalid JSON in json_data: " + err.Error()})
|
|
return
|
|
}
|
|
} else {
|
|
writeJSON(w, 400, map[string]string{"error": "No JSON file or payload provided"})
|
|
return
|
|
}
|
|
}
|
|
|
|
if data == nil {
|
|
writeJSON(w, 400, map[string]string{"error": "Empty data"})
|
|
return
|
|
}
|
|
|
|
results := make(map[string][]ImageItem)
|
|
analyzeJSON(data, "", results)
|
|
|
|
if len(results) == 0 {
|
|
writeJSON(w, 400, map[string]string{"error": "No image URLs found in this JSON"})
|
|
return
|
|
}
|
|
|
|
fieldsSummary := make(map[string]interface{})
|
|
total := 0
|
|
for field, entries := range results {
|
|
total += len(entries)
|
|
sample := []string{}
|
|
for i := 0; i < len(entries) && i < 3; i++ {
|
|
sample = append(sample, entries[i].URL)
|
|
}
|
|
fieldsSummary[field] = map[string]interface{}{
|
|
"count": len(entries),
|
|
"sample_urls": sample,
|
|
}
|
|
}
|
|
|
|
sessionID := fmt.Sprintf("%d", time.Now().UnixNano())
|
|
uploadPath := filepath.Join("uploads", sessionID+".json")
|
|
|
|
f, _ := os.Create(uploadPath)
|
|
json.NewEncoder(f).Encode(data)
|
|
f.Close()
|
|
|
|
writeJSON(w, 200, map[string]interface{}{
|
|
"success": true,
|
|
"session_id": sessionID,
|
|
"fields": fieldsSummary,
|
|
"total_images": total,
|
|
})
|
|
}
|
|
|
|
func previewHandler(w http.ResponseWriter, r *http.Request) {
|
|
var req struct {
|
|
SessionID string `json:"session_id"`
|
|
Fields []string `json:"fields"`
|
|
MaintainStructure bool `json:"maintain_structure"`
|
|
}
|
|
json.NewDecoder(r.Body).Decode(&req)
|
|
|
|
if req.SessionID == "" || len(req.Fields) == 0 {
|
|
writeJSON(w, 400, map[string]string{"error": "Missing session_id or fields"})
|
|
return
|
|
}
|
|
|
|
uploadPath := filepath.Join("uploads", req.SessionID+".json")
|
|
f, err := os.Open(uploadPath)
|
|
if err != nil {
|
|
writeJSON(w, 404, map[string]string{"error": "Session expired"})
|
|
return
|
|
}
|
|
var data interface{}
|
|
json.NewDecoder(f).Decode(&data)
|
|
f.Close()
|
|
|
|
selected := make(map[string]bool)
|
|
for _, f := range req.Fields {
|
|
selected[f] = true
|
|
}
|
|
|
|
var images []ImageItem
|
|
extractImages(data, selected, req.MaintainStructure, "", &images)
|
|
|
|
var fileList []map[string]interface{}
|
|
folders := make(map[string]bool)
|
|
|
|
for i, img := range images {
|
|
folder := filepath.Dir(img.Path)
|
|
if folder == "." {
|
|
folder = ""
|
|
}
|
|
if folder != "" {
|
|
folders[folder] = true
|
|
}
|
|
fileList = append(fileList, map[string]interface{}{
|
|
"id": i,
|
|
"url": img.URL,
|
|
"original_path": img.Path,
|
|
"folder": folder,
|
|
"filename": filepath.Base(img.Path),
|
|
"extension": filepath.Ext(img.Path),
|
|
})
|
|
}
|
|
|
|
writeJSON(w, 200, map[string]interface{}{
|
|
"success": true,
|
|
"files": fileList,
|
|
"images": fileList,
|
|
"stats": map[string]int{
|
|
"total_files": len(fileList),
|
|
"total_folders": len(folders),
|
|
},
|
|
})
|
|
}
|
|
|
|
func downloadProgressHandler(w http.ResponseWriter, r *http.Request) {
|
|
sessionID := chi.URLParam(r, "session_id")
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
w.Header().Set("Cache-Control", "no-cache")
|
|
w.Header().Set("Connection", "keep-alive")
|
|
|
|
flusher, ok := w.(http.Flusher)
|
|
if !ok { return }
|
|
|
|
for i := 0; i < 60; i++ {
|
|
mapLock.Lock()
|
|
_, exists := progressMap[sessionID]
|
|
mapLock.Unlock()
|
|
if exists {
|
|
break
|
|
}
|
|
time.Sleep(500 * time.Millisecond)
|
|
}
|
|
|
|
for {
|
|
mapLock.Lock()
|
|
prog, exists := progressMap[sessionID]
|
|
if !exists {
|
|
mapLock.Unlock()
|
|
fmt.Fprintf(w, "data: %s\n\n", `{"status":"error","message":"session not found"}`)
|
|
flusher.Flush()
|
|
break
|
|
}
|
|
|
|
// Copy prog to avoid race
|
|
pCopy := *prog
|
|
mapLock.Unlock()
|
|
|
|
data, _ := json.Marshal(pCopy)
|
|
fmt.Fprintf(w, "data: %s\n\n", data)
|
|
flusher.Flush()
|
|
|
|
if pCopy.Status == "completed" || pCopy.Status == "stopped" || pCopy.Status == "error" {
|
|
break
|
|
}
|
|
time.Sleep(350 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func controlHandler(w http.ResponseWriter, r *http.Request) {
|
|
sessionID := chi.URLParam(r, "session_id")
|
|
var data struct {
|
|
Action string `json:"action"`
|
|
}
|
|
json.NewDecoder(r.Body).Decode(&data)
|
|
if data.Action != "running" && data.Action != "paused" && data.Action != "stopped" {
|
|
writeJSON(w, 400, map[string]string{"error": "Invalid action"})
|
|
return
|
|
}
|
|
|
|
mapLock.Lock()
|
|
controlMap[sessionID] = data.Action
|
|
if prog, ok := progressMap[sessionID]; ok {
|
|
if data.Action == "paused" {
|
|
prog.Message = "⏸ Paused"
|
|
} else if data.Action == "running" {
|
|
prog.Message = "▶ Resumed"
|
|
} else if data.Action == "stopped" {
|
|
prog.Message = "⏹ Stopping…"
|
|
}
|
|
}
|
|
mapLock.Unlock()
|
|
writeJSON(w, 200, map[string]interface{}{"success": true, "action": data.Action})
|
|
}
|
|
|
|
func downloadHandler(w http.ResponseWriter, r *http.Request) {
|
|
var req struct {
|
|
SessionID string `json:"session_id"`
|
|
Fields []string `json:"fields"`
|
|
MaintainStructure bool `json:"maintain_structure"`
|
|
FileMapping []map[string]interface{} `json:"file_mapping"`
|
|
}
|
|
json.NewDecoder(r.Body).Decode(&req)
|
|
|
|
if req.SessionID == "" {
|
|
writeJSON(w, 400, map[string]string{"error": "Missing session_id"})
|
|
return
|
|
}
|
|
|
|
var images []ImageItem
|
|
if len(req.FileMapping) > 0 {
|
|
for _, m := range req.FileMapping {
|
|
u, _ := m["url"].(string)
|
|
s, _ := m["save_as"].(string)
|
|
if u != "" && s != "" {
|
|
images = append(images, ImageItem{URL: u, Path: s})
|
|
}
|
|
}
|
|
} else {
|
|
uploadPath := filepath.Join("uploads", req.SessionID+".json")
|
|
f, err := os.Open(uploadPath)
|
|
if err != nil {
|
|
writeJSON(w, 404, map[string]string{"error": "Session expired"})
|
|
return
|
|
}
|
|
var data interface{}
|
|
json.NewDecoder(f).Decode(&data)
|
|
f.Close()
|
|
|
|
selected := make(map[string]bool)
|
|
for _, f := range req.Fields { selected[f] = true }
|
|
extractImages(data, selected, req.MaintainStructure, "", &images)
|
|
}
|
|
|
|
if len(images) == 0 {
|
|
writeJSON(w, 404, map[string]string{"error": "No images in selection"})
|
|
return
|
|
}
|
|
|
|
total := len(images)
|
|
setProgress(req.SessionID, &Progress{
|
|
Status: "starting", Total: total, Selected: total, Current: 0, Failed: 0, Message: "Preparing…",
|
|
})
|
|
mapLock.Lock()
|
|
controlMap[req.SessionID] = "running"
|
|
mapLock.Unlock()
|
|
|
|
zipPath := filepath.Join("downloads", req.SessionID+".zip")
|
|
zf, _ := os.Create(zipPath)
|
|
zw := zip.NewWriter(zf)
|
|
var zipLock sync.Mutex
|
|
|
|
setProgress(req.SessionID, &Progress{Status: "downloading", Message: "Downloading…"})
|
|
|
|
taskCh := make(chan ImageItem, total)
|
|
for _, img := range images {
|
|
taskCh <- img
|
|
}
|
|
close(taskCh)
|
|
|
|
var wg sync.WaitGroup
|
|
workers := 16
|
|
for i := 0; i < workers; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for img := range taskCh {
|
|
for getControl(req.SessionID) == "paused" {
|
|
time.Sleep(250 * time.Millisecond)
|
|
}
|
|
if getControl(req.SessionID) == "stopped" {
|
|
break
|
|
}
|
|
|
|
resp, err := client.Get(img.URL)
|
|
ok := false
|
|
if err == nil && resp.StatusCode == 200 {
|
|
data, _ := io.ReadAll(resp.Body)
|
|
resp.Body.Close()
|
|
zipLock.Lock()
|
|
f, _ := zw.Create(img.Path)
|
|
f.Write(data)
|
|
zipLock.Unlock()
|
|
ok = true
|
|
}
|
|
|
|
mapLock.Lock()
|
|
if p, exists := progressMap[req.SessionID]; exists {
|
|
if ok { p.Current++ } else { p.Failed++ }
|
|
}
|
|
mapLock.Unlock()
|
|
}
|
|
}()
|
|
}
|
|
|
|
wg.Wait()
|
|
zw.Close()
|
|
zf.Close()
|
|
|
|
finalCtrl := getControl(req.SessionID)
|
|
finalStatus := "completed"
|
|
dlName := "images.zip"
|
|
if finalCtrl == "stopped" {
|
|
finalStatus = "stopped"
|
|
dlName = "images_partial.zip"
|
|
}
|
|
setProgress(req.SessionID, &Progress{Status: finalStatus, Message: "Packaging ZIP…"})
|
|
os.Remove(filepath.Join("uploads", req.SessionID+".json"))
|
|
|
|
stat, _ := os.Stat(zipPath)
|
|
if stat == nil || stat.Size() == 0 {
|
|
writeJSON(w, 500, map[string]string{"error": "No images were downloaded"})
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/zip")
|
|
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, dlName))
|
|
http.ServeFile(w, r, zipPath)
|
|
|
|
os.Remove(zipPath)
|
|
mapLock.Lock()
|
|
delete(progressMap, req.SessionID)
|
|
delete(controlMap, req.SessionID)
|
|
mapLock.Unlock()
|
|
}
|
|
|
|
func cleanupHandler(w http.ResponseWriter, r *http.Request) {
|
|
var req struct {
|
|
SessionID string `json:"session_id"`
|
|
}
|
|
json.NewDecoder(r.Body).Decode(&req)
|
|
if req.SessionID != "" {
|
|
os.Remove(filepath.Join("uploads", req.SessionID+".json"))
|
|
os.Remove(filepath.Join("downloads", req.SessionID+".zip"))
|
|
mapLock.Lock()
|
|
delete(progressMap, req.SessionID)
|
|
delete(controlMap, req.SessionID)
|
|
mapLock.Unlock()
|
|
}
|
|
writeJSON(w, 200, map[string]bool{"success": true})
|
|
}
|