perf(bulk-api-tester): optimize draft saving, share link creation, and combining with parallel multipart streaming and instant hard links
This commit is contained in:
+236
-47
@@ -11,6 +11,7 @@ import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
@@ -101,6 +102,20 @@ func copyFile(src, dst string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func linkOrCopyFile(src, dst string) error {
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Try hard link first for instant zero-copy linking
|
||||
if err := os.Link(src, dst); err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Fallback to copy if cross-filesystem or link fails
|
||||
return copyFile(src, dst)
|
||||
}
|
||||
|
||||
func resolveDiskPathFromURL(rawURL string) string {
|
||||
if rawURL == "" {
|
||||
return ""
|
||||
@@ -131,7 +146,7 @@ func listComparisonDraftsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
var drafts []ComparisonDraft
|
||||
drafts := []ComparisonDraft{}
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
@@ -179,6 +194,12 @@ func saveComparisonDraftHandler(w http.ResponseWriter, r *http.Request) {
|
||||
// Set 500MB limit on request body
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 500*1024*1024)
|
||||
|
||||
// High-speed direct binary streaming via multipart
|
||||
if strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data") {
|
||||
handleMultipartDraft(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
var req SaveDraftRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
slog.Error("failed to decode save draft request", "error", err)
|
||||
@@ -227,22 +248,7 @@ func saveComparisonDraftHandler(w http.ResponseWriter, r *http.Request) {
|
||||
SourceDraftID: draftID,
|
||||
}
|
||||
|
||||
isSuccess := false
|
||||
switch v := it.Status.(type) {
|
||||
case float64:
|
||||
if int(v) >= 200 && int(v) < 300 {
|
||||
isSuccess = true
|
||||
}
|
||||
case int:
|
||||
if v >= 200 && v < 300 {
|
||||
isSuccess = true
|
||||
}
|
||||
case string:
|
||||
if strings.HasPrefix(v, "2") {
|
||||
isSuccess = true
|
||||
}
|
||||
}
|
||||
if isSuccess {
|
||||
if isSuccessStatus(it.Status) {
|
||||
successCount++
|
||||
} else {
|
||||
errorCount++
|
||||
@@ -320,6 +326,186 @@ func saveComparisonDraftHandler(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
func handleMultipartDraft(w http.ResponseWriter, r *http.Request) {
|
||||
mr, err := r.MultipartReader()
|
||||
if err != nil {
|
||||
slog.Error("failed to create multipart reader for draft", "error", err)
|
||||
writeJSON(w, http.StatusBadRequest, map[string]any{"success": false, "error": "Invalid multipart payload"})
|
||||
return
|
||||
}
|
||||
|
||||
draftID := fmt.Sprintf("draft_%x_%s", time.Now().Unix(), randomHexString(4))
|
||||
draftDir := filepath.Join("data", "bulk-api-tester", "drafts", draftID)
|
||||
imagesDir := filepath.Join(draftDir, "images")
|
||||
|
||||
if err := os.MkdirAll(imagesDir, 0755); err != nil {
|
||||
slog.Error("failed to create draft directory", "dir", draftDir, "error", err)
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]any{"success": false, "error": "Failed to create storage directory"})
|
||||
return
|
||||
}
|
||||
|
||||
var metaReq SaveDraftRequest
|
||||
filesMap := make(map[string]string) // formName -> savedFilename
|
||||
|
||||
for {
|
||||
part, err := mr.NextPart()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
slog.Warn("error reading draft multipart part", "error", err)
|
||||
break
|
||||
}
|
||||
|
||||
formName := part.FormName()
|
||||
if formName == "meta" {
|
||||
metaBytes, _ := io.ReadAll(part)
|
||||
json.Unmarshal(metaBytes, &metaReq)
|
||||
part.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.HasPrefix(formName, "before_") || strings.HasPrefix(formName, "after_") {
|
||||
fileName := part.FileName()
|
||||
ext := strings.ToLower(filepath.Ext(fileName))
|
||||
if ext == "" {
|
||||
ext = ".png"
|
||||
}
|
||||
saveName := fmt.Sprintf("%s%s", formName, ext)
|
||||
filePath := filepath.Join(imagesDir, saveName)
|
||||
|
||||
outFile, err := os.Create(filePath)
|
||||
if err == nil {
|
||||
io.Copy(outFile, part)
|
||||
outFile.Close()
|
||||
filesMap[formName] = saveName
|
||||
} else {
|
||||
slog.Warn("failed to write draft multipart image", "path", filePath, "error", err)
|
||||
}
|
||||
part.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
part.Close()
|
||||
}
|
||||
|
||||
if metaReq.ID != "" && validIDRegex.MatchString(metaReq.ID) {
|
||||
if metaReq.ID != draftID {
|
||||
targetDir := filepath.Join("data", "bulk-api-tester", "drafts", metaReq.ID)
|
||||
if err := os.Rename(draftDir, targetDir); err == nil {
|
||||
draftID = metaReq.ID
|
||||
draftDir = targetDir
|
||||
imagesDir = filepath.Join(draftDir, "images")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var items []DraftItem
|
||||
var totalLatency int64
|
||||
successCount := 0
|
||||
errorCount := 0
|
||||
|
||||
for i, it := range metaReq.Items {
|
||||
itemIndex := it.Index
|
||||
if itemIndex <= 0 {
|
||||
itemIndex = i + 1
|
||||
}
|
||||
|
||||
dItem := DraftItem{
|
||||
Index: itemIndex,
|
||||
Status: it.Status,
|
||||
Latency: it.Latency,
|
||||
InputFilename: it.InputFilename,
|
||||
OutputFilename: it.OutputFilename,
|
||||
ContentType: it.ContentType,
|
||||
TextContent: it.TextContent,
|
||||
SourceDraftID: draftID,
|
||||
}
|
||||
|
||||
if isSuccessStatus(it.Status) {
|
||||
successCount++
|
||||
} else {
|
||||
errorCount++
|
||||
}
|
||||
totalLatency += it.Latency
|
||||
|
||||
beforeKey := fmt.Sprintf("before_%d", itemIndex)
|
||||
if saveName, ok := filesMap[beforeKey]; ok {
|
||||
dItem.BeforeImage = fmt.Sprintf("/api/comparison-drafts/%s/images/%s", draftID, saveName)
|
||||
dItem.HasBefore = true
|
||||
} else if it.BeforeData != "" {
|
||||
beforeBytes, ext, err := decodeImageData(it.BeforeData)
|
||||
if err == nil && len(beforeBytes) > 0 {
|
||||
filename := fmt.Sprintf("before_%d.%s", itemIndex, ext)
|
||||
filePath := filepath.Join(imagesDir, filename)
|
||||
if err := os.WriteFile(filePath, beforeBytes, 0644); err == nil {
|
||||
dItem.BeforeImage = fmt.Sprintf("/api/comparison-drafts/%s/images/%s", draftID, filename)
|
||||
dItem.HasBefore = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
afterKey := fmt.Sprintf("after_%d", itemIndex)
|
||||
if saveName, ok := filesMap[afterKey]; ok {
|
||||
dItem.AfterImage = fmt.Sprintf("/api/comparison-drafts/%s/images/%s", draftID, saveName)
|
||||
dItem.HasAfter = true
|
||||
} else if it.AfterData != "" {
|
||||
afterBytes, ext, err := decodeImageData(it.AfterData)
|
||||
if err == nil && len(afterBytes) > 0 {
|
||||
filename := fmt.Sprintf("after_%d.%s", itemIndex, ext)
|
||||
filePath := filepath.Join(imagesDir, filename)
|
||||
if err := os.WriteFile(filePath, afterBytes, 0644); err == nil {
|
||||
dItem.AfterImage = fmt.Sprintf("/api/comparison-drafts/%s/images/%s", draftID, filename)
|
||||
dItem.HasAfter = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
items = append(items, dItem)
|
||||
}
|
||||
|
||||
title := strings.TrimSpace(metaReq.Title)
|
||||
if title == "" {
|
||||
title = fmt.Sprintf("Draft Run (%d results) - %s", len(items), time.Now().Format("Jan 02, 15:04"))
|
||||
}
|
||||
|
||||
var avgLatency int64
|
||||
if len(items) > 0 {
|
||||
avgLatency = totalLatency / int64(len(items))
|
||||
}
|
||||
|
||||
nowStr := time.Now().UTC().Format(time.RFC3339)
|
||||
draftData := ComparisonDraft{
|
||||
ID: draftID,
|
||||
Title: title,
|
||||
Notes: metaReq.Notes,
|
||||
Endpoint: metaReq.Endpoint,
|
||||
SpaceName: metaReq.SpaceName,
|
||||
CreatedAt: nowStr,
|
||||
UpdatedAt: nowStr,
|
||||
Total: len(items),
|
||||
SuccessCount: successCount,
|
||||
ErrorCount: errorCount,
|
||||
AvgLatency: avgLatency,
|
||||
Items: items,
|
||||
}
|
||||
|
||||
metaPath := filepath.Join(draftDir, "meta.json")
|
||||
if err := atomicWriteJSON(metaPath, draftData); err != nil {
|
||||
slog.Error("failed to save draft metadata", "path", metaPath, "error", err)
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]any{"success": false, "error": "Failed to save draft metadata"})
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("saved comparison draft via fast multipart", "draft_id", draftID, "items_count", len(items))
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"success": true,
|
||||
"draftId": draftID,
|
||||
"draft": draftData,
|
||||
})
|
||||
}
|
||||
|
||||
func deleteComparisonDraftHandler(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
if id == "" || !validIDRegex.MatchString(id) {
|
||||
@@ -408,7 +594,13 @@ func combineDraftsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
var items []ComparisonItem
|
||||
type fileTask struct {
|
||||
src string
|
||||
dst string
|
||||
}
|
||||
var linkTasks []fileTask
|
||||
|
||||
items := make([]ComparisonItem, len(req.Items))
|
||||
var totalLatency int64
|
||||
successCount := 0
|
||||
errorCount := 0
|
||||
@@ -426,22 +618,7 @@ func combineDraftsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
TextContent: it.TextContent,
|
||||
}
|
||||
|
||||
isSuccess := false
|
||||
switch v := it.Status.(type) {
|
||||
case float64:
|
||||
if int(v) >= 200 && int(v) < 300 {
|
||||
isSuccess = true
|
||||
}
|
||||
case int:
|
||||
if v >= 200 && v < 300 {
|
||||
isSuccess = true
|
||||
}
|
||||
case string:
|
||||
if strings.HasPrefix(v, "2") {
|
||||
isSuccess = true
|
||||
}
|
||||
}
|
||||
if isSuccess {
|
||||
if isSuccessStatus(it.Status) {
|
||||
successCount++
|
||||
} else {
|
||||
errorCount++
|
||||
@@ -468,12 +645,9 @@ func combineDraftsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
filename := fmt.Sprintf("before_%d%s", newIndex, ext)
|
||||
dst := filepath.Join(imagesDir, filename)
|
||||
if err := copyFile(srcDiskPath, dst); err == nil {
|
||||
cItem.BeforeImage = fmt.Sprintf("/api/comparison/%s/images/%s", shareID, filename)
|
||||
cItem.HasBefore = true
|
||||
} else {
|
||||
slog.Warn("failed to copy before image", "src", srcDiskPath, "dst", dst, "error", err)
|
||||
}
|
||||
linkTasks = append(linkTasks, fileTask{src: srcDiskPath, dst: dst})
|
||||
cItem.BeforeImage = fmt.Sprintf("/api/comparison/%s/images/%s", shareID, filename)
|
||||
cItem.HasBefore = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -497,16 +671,31 @@ func combineDraftsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
filename := fmt.Sprintf("after_%d%s", newIndex, ext)
|
||||
dst := filepath.Join(imagesDir, filename)
|
||||
if err := copyFile(srcDiskPath, dst); err == nil {
|
||||
cItem.AfterImage = fmt.Sprintf("/api/comparison/%s/images/%s", shareID, filename)
|
||||
cItem.HasAfter = true
|
||||
} else {
|
||||
slog.Warn("failed to copy after image", "src", srcDiskPath, "dst", dst, "error", err)
|
||||
}
|
||||
linkTasks = append(linkTasks, fileTask{src: srcDiskPath, dst: dst})
|
||||
cItem.AfterImage = fmt.Sprintf("/api/comparison/%s/images/%s", shareID, filename)
|
||||
cItem.HasAfter = true
|
||||
}
|
||||
}
|
||||
|
||||
items = append(items, cItem)
|
||||
items[i] = cItem
|
||||
}
|
||||
|
||||
// Concurrently hard-link or copy files with worker pool
|
||||
if len(linkTasks) > 0 {
|
||||
var wg sync.WaitGroup
|
||||
sem := make(chan struct{}, 16)
|
||||
for _, task := range linkTasks {
|
||||
wg.Add(1)
|
||||
go func(t fileTask) {
|
||||
defer wg.Done()
|
||||
sem <- struct{}{}
|
||||
defer func() { <-sem }()
|
||||
if err := linkOrCopyFile(t.src, t.dst); err != nil {
|
||||
slog.Warn("failed to link/copy draft image", "src", t.src, "dst", t.dst, "error", err)
|
||||
}
|
||||
}(task)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
heading := strings.TrimSpace(req.Heading)
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -19,6 +20,18 @@ import (
|
||||
|
||||
var validIDRegex = regexp.MustCompile(`^[a-zA-Z0-9_\-]+$`)
|
||||
|
||||
func isSuccessStatus(status any) bool {
|
||||
switch v := status.(type) {
|
||||
case float64:
|
||||
return int(v) >= 200 && int(v) < 300
|
||||
case int:
|
||||
return v >= 200 && v < 300
|
||||
case string:
|
||||
return strings.HasPrefix(v, "2")
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type ComparisonItem struct {
|
||||
Index int `json:"index"`
|
||||
Status any `json:"status"`
|
||||
@@ -124,6 +137,12 @@ func createComparisonShareHandler(w http.ResponseWriter, r *http.Request) {
|
||||
// Set 500MB limit on request body for batches of images
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 500*1024*1024)
|
||||
|
||||
// High-speed direct binary streaming via multipart
|
||||
if strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data") {
|
||||
handleMultipartShare(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
var req ShareRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
slog.Error("failed to decode share request", "error", err)
|
||||
@@ -363,3 +382,150 @@ func comparePageHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
w.Write(htmlBytes)
|
||||
}
|
||||
|
||||
func handleMultipartShare(w http.ResponseWriter, r *http.Request) {
|
||||
mr, err := r.MultipartReader()
|
||||
if err != nil {
|
||||
slog.Error("failed to create multipart reader", "error", err)
|
||||
writeJSON(w, http.StatusBadRequest, map[string]any{"success": false, "error": "Invalid multipart payload"})
|
||||
return
|
||||
}
|
||||
|
||||
shareID := fmt.Sprintf("cmp_%x_%s", time.Now().Unix(), randomHexString(4))
|
||||
compDir := filepath.Join("data", "bulk-api-tester", "comparisons", shareID)
|
||||
imagesDir := filepath.Join(compDir, "images")
|
||||
|
||||
if err := os.MkdirAll(imagesDir, 0755); err != nil {
|
||||
slog.Error("failed to create comparison directory", "dir", compDir, "error", err)
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]any{"success": false, "error": "Failed to create storage directory"})
|
||||
return
|
||||
}
|
||||
|
||||
var metaReq ShareRequest
|
||||
filesMap := make(map[string]string) // formName -> savedFilename
|
||||
|
||||
for {
|
||||
part, err := mr.NextPart()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
slog.Warn("error reading multipart part", "error", err)
|
||||
break
|
||||
}
|
||||
|
||||
formName := part.FormName()
|
||||
if formName == "meta" {
|
||||
metaBytes, _ := io.ReadAll(part)
|
||||
json.Unmarshal(metaBytes, &metaReq)
|
||||
part.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.HasPrefix(formName, "before_") || strings.HasPrefix(formName, "after_") {
|
||||
fileName := part.FileName()
|
||||
ext := strings.ToLower(filepath.Ext(fileName))
|
||||
if ext == "" {
|
||||
ext = ".png"
|
||||
}
|
||||
saveName := fmt.Sprintf("%s%s", formName, ext)
|
||||
filePath := filepath.Join(imagesDir, saveName)
|
||||
|
||||
outFile, err := os.Create(filePath)
|
||||
if err == nil {
|
||||
io.Copy(outFile, part)
|
||||
outFile.Close()
|
||||
filesMap[formName] = saveName
|
||||
} else {
|
||||
slog.Warn("failed to write multipart image", "path", filePath, "error", err)
|
||||
}
|
||||
part.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
part.Close()
|
||||
}
|
||||
|
||||
var items []ComparisonItem
|
||||
var totalLatency int64
|
||||
successCount := 0
|
||||
errorCount := 0
|
||||
|
||||
for i, it := range metaReq.Items {
|
||||
itemIndex := it.Index
|
||||
if itemIndex <= 0 {
|
||||
itemIndex = i + 1
|
||||
}
|
||||
|
||||
cItem := ComparisonItem{
|
||||
Index: itemIndex,
|
||||
Status: it.Status,
|
||||
Latency: it.Latency,
|
||||
InputFilename: it.InputFilename,
|
||||
OutputFilename: it.OutputFilename,
|
||||
ContentType: it.ContentType,
|
||||
TextContent: it.TextContent,
|
||||
}
|
||||
|
||||
if isSuccessStatus(it.Status) {
|
||||
successCount++
|
||||
} else {
|
||||
errorCount++
|
||||
}
|
||||
totalLatency += it.Latency
|
||||
|
||||
beforeKey := fmt.Sprintf("before_%d", itemIndex)
|
||||
if saveName, ok := filesMap[beforeKey]; ok {
|
||||
cItem.BeforeImage = fmt.Sprintf("/api/comparison/%s/images/%s", shareID, saveName)
|
||||
cItem.HasBefore = true
|
||||
}
|
||||
|
||||
afterKey := fmt.Sprintf("after_%d", itemIndex)
|
||||
if saveName, ok := filesMap[afterKey]; ok {
|
||||
cItem.AfterImage = fmt.Sprintf("/api/comparison/%s/images/%s", shareID, saveName)
|
||||
cItem.HasAfter = true
|
||||
}
|
||||
|
||||
items = append(items, cItem)
|
||||
}
|
||||
|
||||
title := strings.TrimSpace(metaReq.Title)
|
||||
if title == "" {
|
||||
title = fmt.Sprintf("Batch Comparison (%d results)", len(items))
|
||||
}
|
||||
|
||||
var avgLatency int64
|
||||
if len(items) > 0 {
|
||||
avgLatency = totalLatency / int64(len(items))
|
||||
}
|
||||
|
||||
shareData := ComparisonShare{
|
||||
ID: shareID,
|
||||
Title: title,
|
||||
Endpoint: metaReq.Endpoint,
|
||||
SpaceName: metaReq.SpaceName,
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
Total: len(items),
|
||||
SuccessCount: successCount,
|
||||
ErrorCount: errorCount,
|
||||
AvgLatency: avgLatency,
|
||||
Items: items,
|
||||
}
|
||||
|
||||
metaPath := filepath.Join(compDir, "meta.json")
|
||||
if err := atomicWriteJSON(metaPath, shareData); err != nil {
|
||||
slog.Error("failed to save comparison meta", "path", metaPath, "error", err)
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]any{"success": false, "error": "Failed to save comparison meta"})
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("created shareable comparison via fast multipart", "share_id", shareID, "items_count", len(items))
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"success": true,
|
||||
"shareId": shareID,
|
||||
"url": "/tool/bulk-api-tester/compare/" + shareID,
|
||||
"directUrl": "/compare/" + shareID,
|
||||
"total": len(items),
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user