perf(bulk-api-tester): optimize draft saving, share link creation, and combining with parallel multipart streaming and instant hard links

This commit is contained in:
2026-10-28 16:24:28 +05:30
parent ce75adb467
commit 2fd2c5e582
4 changed files with 643 additions and 239 deletions
+236 -47
View File
@@ -11,6 +11,7 @@ import (
"regexp"
"sort"
"strings"
"sync"
"time"
"github.com/go-chi/chi/v5"
@@ -101,6 +102,20 @@ func copyFile(src, dst string) error {
return err
}
func linkOrCopyFile(src, dst string) error {
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
return err
}
// Try hard link first for instant zero-copy linking
if err := os.Link(src, dst); err == nil {
return nil
}
// Fallback to copy if cross-filesystem or link fails
return copyFile(src, dst)
}
func resolveDiskPathFromURL(rawURL string) string {
if rawURL == "" {
return ""
@@ -131,7 +146,7 @@ func listComparisonDraftsHandler(w http.ResponseWriter, r *http.Request) {
return
}
var drafts []ComparisonDraft
drafts := []ComparisonDraft{}
for _, entry := range entries {
if !entry.IsDir() {
continue
@@ -179,6 +194,12 @@ func saveComparisonDraftHandler(w http.ResponseWriter, r *http.Request) {
// Set 500MB limit on request body
r.Body = http.MaxBytesReader(w, r.Body, 500*1024*1024)
// High-speed direct binary streaming via multipart
if strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data") {
handleMultipartDraft(w, r)
return
}
var req SaveDraftRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
slog.Error("failed to decode save draft request", "error", err)
@@ -227,22 +248,7 @@ func saveComparisonDraftHandler(w http.ResponseWriter, r *http.Request) {
SourceDraftID: draftID,
}
isSuccess := false
switch v := it.Status.(type) {
case float64:
if int(v) >= 200 && int(v) < 300 {
isSuccess = true
}
case int:
if v >= 200 && v < 300 {
isSuccess = true
}
case string:
if strings.HasPrefix(v, "2") {
isSuccess = true
}
}
if isSuccess {
if isSuccessStatus(it.Status) {
successCount++
} else {
errorCount++
@@ -320,6 +326,186 @@ func saveComparisonDraftHandler(w http.ResponseWriter, r *http.Request) {
})
}
func handleMultipartDraft(w http.ResponseWriter, r *http.Request) {
mr, err := r.MultipartReader()
if err != nil {
slog.Error("failed to create multipart reader for draft", "error", err)
writeJSON(w, http.StatusBadRequest, map[string]any{"success": false, "error": "Invalid multipart payload"})
return
}
draftID := fmt.Sprintf("draft_%x_%s", time.Now().Unix(), randomHexString(4))
draftDir := filepath.Join("data", "bulk-api-tester", "drafts", draftID)
imagesDir := filepath.Join(draftDir, "images")
if err := os.MkdirAll(imagesDir, 0755); err != nil {
slog.Error("failed to create draft directory", "dir", draftDir, "error", err)
writeJSON(w, http.StatusInternalServerError, map[string]any{"success": false, "error": "Failed to create storage directory"})
return
}
var metaReq SaveDraftRequest
filesMap := make(map[string]string) // formName -> savedFilename
for {
part, err := mr.NextPart()
if err == io.EOF {
break
}
if err != nil {
slog.Warn("error reading draft multipart part", "error", err)
break
}
formName := part.FormName()
if formName == "meta" {
metaBytes, _ := io.ReadAll(part)
json.Unmarshal(metaBytes, &metaReq)
part.Close()
continue
}
if strings.HasPrefix(formName, "before_") || strings.HasPrefix(formName, "after_") {
fileName := part.FileName()
ext := strings.ToLower(filepath.Ext(fileName))
if ext == "" {
ext = ".png"
}
saveName := fmt.Sprintf("%s%s", formName, ext)
filePath := filepath.Join(imagesDir, saveName)
outFile, err := os.Create(filePath)
if err == nil {
io.Copy(outFile, part)
outFile.Close()
filesMap[formName] = saveName
} else {
slog.Warn("failed to write draft multipart image", "path", filePath, "error", err)
}
part.Close()
continue
}
part.Close()
}
if metaReq.ID != "" && validIDRegex.MatchString(metaReq.ID) {
if metaReq.ID != draftID {
targetDir := filepath.Join("data", "bulk-api-tester", "drafts", metaReq.ID)
if err := os.Rename(draftDir, targetDir); err == nil {
draftID = metaReq.ID
draftDir = targetDir
imagesDir = filepath.Join(draftDir, "images")
}
}
}
var items []DraftItem
var totalLatency int64
successCount := 0
errorCount := 0
for i, it := range metaReq.Items {
itemIndex := it.Index
if itemIndex <= 0 {
itemIndex = i + 1
}
dItem := DraftItem{
Index: itemIndex,
Status: it.Status,
Latency: it.Latency,
InputFilename: it.InputFilename,
OutputFilename: it.OutputFilename,
ContentType: it.ContentType,
TextContent: it.TextContent,
SourceDraftID: draftID,
}
if isSuccessStatus(it.Status) {
successCount++
} else {
errorCount++
}
totalLatency += it.Latency
beforeKey := fmt.Sprintf("before_%d", itemIndex)
if saveName, ok := filesMap[beforeKey]; ok {
dItem.BeforeImage = fmt.Sprintf("/api/comparison-drafts/%s/images/%s", draftID, saveName)
dItem.HasBefore = true
} else if it.BeforeData != "" {
beforeBytes, ext, err := decodeImageData(it.BeforeData)
if err == nil && len(beforeBytes) > 0 {
filename := fmt.Sprintf("before_%d.%s", itemIndex, ext)
filePath := filepath.Join(imagesDir, filename)
if err := os.WriteFile(filePath, beforeBytes, 0644); err == nil {
dItem.BeforeImage = fmt.Sprintf("/api/comparison-drafts/%s/images/%s", draftID, filename)
dItem.HasBefore = true
}
}
}
afterKey := fmt.Sprintf("after_%d", itemIndex)
if saveName, ok := filesMap[afterKey]; ok {
dItem.AfterImage = fmt.Sprintf("/api/comparison-drafts/%s/images/%s", draftID, saveName)
dItem.HasAfter = true
} else if it.AfterData != "" {
afterBytes, ext, err := decodeImageData(it.AfterData)
if err == nil && len(afterBytes) > 0 {
filename := fmt.Sprintf("after_%d.%s", itemIndex, ext)
filePath := filepath.Join(imagesDir, filename)
if err := os.WriteFile(filePath, afterBytes, 0644); err == nil {
dItem.AfterImage = fmt.Sprintf("/api/comparison-drafts/%s/images/%s", draftID, filename)
dItem.HasAfter = true
}
}
}
items = append(items, dItem)
}
title := strings.TrimSpace(metaReq.Title)
if title == "" {
title = fmt.Sprintf("Draft Run (%d results) - %s", len(items), time.Now().Format("Jan 02, 15:04"))
}
var avgLatency int64
if len(items) > 0 {
avgLatency = totalLatency / int64(len(items))
}
nowStr := time.Now().UTC().Format(time.RFC3339)
draftData := ComparisonDraft{
ID: draftID,
Title: title,
Notes: metaReq.Notes,
Endpoint: metaReq.Endpoint,
SpaceName: metaReq.SpaceName,
CreatedAt: nowStr,
UpdatedAt: nowStr,
Total: len(items),
SuccessCount: successCount,
ErrorCount: errorCount,
AvgLatency: avgLatency,
Items: items,
}
metaPath := filepath.Join(draftDir, "meta.json")
if err := atomicWriteJSON(metaPath, draftData); err != nil {
slog.Error("failed to save draft metadata", "path", metaPath, "error", err)
writeJSON(w, http.StatusInternalServerError, map[string]any{"success": false, "error": "Failed to save draft metadata"})
return
}
slog.Info("saved comparison draft via fast multipart", "draft_id", draftID, "items_count", len(items))
writeJSON(w, http.StatusOK, map[string]any{
"success": true,
"draftId": draftID,
"draft": draftData,
})
}
func deleteComparisonDraftHandler(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
if id == "" || !validIDRegex.MatchString(id) {
@@ -408,7 +594,13 @@ func combineDraftsHandler(w http.ResponseWriter, r *http.Request) {
return
}
var items []ComparisonItem
type fileTask struct {
src string
dst string
}
var linkTasks []fileTask
items := make([]ComparisonItem, len(req.Items))
var totalLatency int64
successCount := 0
errorCount := 0
@@ -426,22 +618,7 @@ func combineDraftsHandler(w http.ResponseWriter, r *http.Request) {
TextContent: it.TextContent,
}
isSuccess := false
switch v := it.Status.(type) {
case float64:
if int(v) >= 200 && int(v) < 300 {
isSuccess = true
}
case int:
if v >= 200 && v < 300 {
isSuccess = true
}
case string:
if strings.HasPrefix(v, "2") {
isSuccess = true
}
}
if isSuccess {
if isSuccessStatus(it.Status) {
successCount++
} else {
errorCount++
@@ -468,12 +645,9 @@ func combineDraftsHandler(w http.ResponseWriter, r *http.Request) {
}
filename := fmt.Sprintf("before_%d%s", newIndex, ext)
dst := filepath.Join(imagesDir, filename)
if err := copyFile(srcDiskPath, dst); err == nil {
cItem.BeforeImage = fmt.Sprintf("/api/comparison/%s/images/%s", shareID, filename)
cItem.HasBefore = true
} else {
slog.Warn("failed to copy before image", "src", srcDiskPath, "dst", dst, "error", err)
}
linkTasks = append(linkTasks, fileTask{src: srcDiskPath, dst: dst})
cItem.BeforeImage = fmt.Sprintf("/api/comparison/%s/images/%s", shareID, filename)
cItem.HasBefore = true
}
}
@@ -497,16 +671,31 @@ func combineDraftsHandler(w http.ResponseWriter, r *http.Request) {
}
filename := fmt.Sprintf("after_%d%s", newIndex, ext)
dst := filepath.Join(imagesDir, filename)
if err := copyFile(srcDiskPath, dst); err == nil {
cItem.AfterImage = fmt.Sprintf("/api/comparison/%s/images/%s", shareID, filename)
cItem.HasAfter = true
} else {
slog.Warn("failed to copy after image", "src", srcDiskPath, "dst", dst, "error", err)
}
linkTasks = append(linkTasks, fileTask{src: srcDiskPath, dst: dst})
cItem.AfterImage = fmt.Sprintf("/api/comparison/%s/images/%s", shareID, filename)
cItem.HasAfter = true
}
}
items = append(items, cItem)
items[i] = cItem
}
// Concurrently hard-link or copy files with worker pool
if len(linkTasks) > 0 {
var wg sync.WaitGroup
sem := make(chan struct{}, 16)
for _, task := range linkTasks {
wg.Add(1)
go func(t fileTask) {
defer wg.Done()
sem <- struct{}{}
defer func() { <-sem }()
if err := linkOrCopyFile(t.src, t.dst); err != nil {
slog.Warn("failed to link/copy draft image", "src", t.src, "dst", t.dst, "error", err)
}
}(task)
}
wg.Wait()
}
heading := strings.TrimSpace(req.Heading)