feat: complete dynamic JSON-driven photo tools frontend connected to FastAPI backend
This commit is contained in:
@@ -0,0 +1,333 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import crypto from 'crypto';
|
||||
import { EndpointSpec, ParamSpec, VariantSpec, DiffReport, Localised } from '@/types';
|
||||
|
||||
let cachedEndpoints: EndpointSpec[] | null = null;
|
||||
let lastGoodSnapshot: EndpointSpec[] | null = null;
|
||||
let lastDiffReport: DiffReport | null = null;
|
||||
|
||||
function sanitizeText(str: any): string {
|
||||
if (typeof str !== 'string') return '';
|
||||
// Strip non-printable control characters, trim, collapse whitespace
|
||||
return str.replace(/[\u0000-\u0008\u000B-\u001F\u007F]/g, '').trim();
|
||||
}
|
||||
|
||||
function parseKind(rawType: string): ParamSpec['kind'] {
|
||||
const t = (rawType || '').toLowerCase();
|
||||
if (t.includes('file')) return 'file';
|
||||
if (t.includes('int')) return 'integer';
|
||||
if (t.includes('float') || t.includes('number') || t.includes('double')) return 'number';
|
||||
if (t.includes('bool')) return 'boolean';
|
||||
if (t.includes('json') || t.includes('object') || t.includes('array')) return 'json';
|
||||
return 'string';
|
||||
}
|
||||
|
||||
function parseLocation(lc: string): ParamSpec['location'] {
|
||||
const l = (lc || '').toLowerCase();
|
||||
if (l.includes('multipart') || l.includes('form-data')) return 'multipart';
|
||||
if (l.includes('form') || l.includes('x-www-form-urlencoded')) return 'form';
|
||||
if (l.includes('json')) return 'json';
|
||||
if (l.includes('query')) return 'query';
|
||||
if (l.includes('header')) return 'header';
|
||||
if (l.includes('path')) return 'path';
|
||||
return 'multipart';
|
||||
}
|
||||
|
||||
function isSecretParam(name: string, desc: string): boolean {
|
||||
const n = (name || '').toLowerCase();
|
||||
const d = (desc || '').toLowerCase();
|
||||
if (/secret|token|password|auth|key/.test(n)) return true;
|
||||
if (/authentication|api[-_ ]?key|bearer/.test(d)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
export function loadRawCatalog(): any {
|
||||
const catalogPath = process.env.CATALOG_PATH || path.join(process.cwd(), 'content', 'api.catalog.json');
|
||||
if (!fs.existsSync(catalogPath)) {
|
||||
throw new Error(`Catalogue file not found at ${catalogPath}`);
|
||||
}
|
||||
const rawText = fs.readFileSync(catalogPath, 'utf8');
|
||||
return JSON.parse(rawText);
|
||||
}
|
||||
|
||||
export function loadOverlay(): any {
|
||||
const overlayPath = path.join(process.cwd(), 'content', 'api.overlay.json');
|
||||
if (!fs.existsSync(overlayPath)) {
|
||||
return { schemaVersion: '1.0', defaults: {}, secrets: {}, rateLimits: {}, endpoints: {} };
|
||||
}
|
||||
const rawText = fs.readFileSync(overlayPath, 'utf8');
|
||||
return JSON.parse(rawText);
|
||||
}
|
||||
|
||||
export function extractGroups(raw: any): any[] {
|
||||
if (!raw) return [];
|
||||
// (a) Array of groups
|
||||
if (Array.isArray(raw)) {
|
||||
// Check if it is a bare list of endpoints
|
||||
if (raw.length > 0 && raw[0].full_url && !raw[0].endpoints) {
|
||||
return [{ title: 'Default', endpoints: raw }];
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
// (b) Object with list under any key (apis, groups, data, etc.)
|
||||
for (const key of ['apis', 'groups', 'data', 'catalog', 'endpoints']) {
|
||||
if (Array.isArray(raw[key])) {
|
||||
if (key === 'endpoints') {
|
||||
return [raw];
|
||||
}
|
||||
return raw[key];
|
||||
}
|
||||
}
|
||||
// (c) Single group object
|
||||
if (raw && Array.isArray(raw.endpoints)) {
|
||||
return [raw];
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
export function normaliseCatalog(rawInput?: any, overlayInput?: any): { endpoints: EndpointSpec[]; diff: DiffReport } {
|
||||
try {
|
||||
const raw = rawInput || loadRawCatalog();
|
||||
const overlay = overlayInput || loadOverlay();
|
||||
const defaults = overlay.defaults || {};
|
||||
const overlayEndpoints = overlay.endpoints || {};
|
||||
|
||||
const groups = extractGroups(raw);
|
||||
const seenIds = new Set<string>();
|
||||
const normalised: EndpointSpec[] = [];
|
||||
const misconfigured: string[] = [];
|
||||
|
||||
for (const group of groups) {
|
||||
const groupKey = sanitizeText(group.api_key || group.title || 'default');
|
||||
const groupTitle = sanitizeText(group.title || groupKey);
|
||||
const rawEndpoints = Array.isArray(group.endpoints) ? group.endpoints : [];
|
||||
|
||||
for (const ep of rawEndpoints) {
|
||||
const id = sanitizeText(ep.id);
|
||||
const fullUrl = sanitizeText(ep.full_url);
|
||||
|
||||
if (!id || !fullUrl) {
|
||||
console.warn(`[Catalog] Skipping endpoint without id or full_url`, ep);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (seenIds.has(id)) {
|
||||
console.warn(`[Catalog] Duplicate endpoint ID detected: ${id}. Using first occurrence.`);
|
||||
continue;
|
||||
}
|
||||
seenIds.add(id);
|
||||
|
||||
const epPath = sanitizeText(ep.path || '');
|
||||
let baseUrl = fullUrl;
|
||||
if (epPath && fullUrl.endsWith(epPath)) {
|
||||
baseUrl = fullUrl.slice(0, fullUrl.length - epPath.length);
|
||||
} else {
|
||||
try {
|
||||
const parsedUrl = new URL(fullUrl);
|
||||
baseUrl = `${parsedUrl.protocol}//${parsedUrl.host}`;
|
||||
} catch {
|
||||
baseUrl = fullUrl;
|
||||
}
|
||||
}
|
||||
|
||||
// Overlay overrides for this endpoint
|
||||
const epOverlay = overlayEndpoints[id] || {};
|
||||
const isEnabled = epOverlay.enabled !== false;
|
||||
const isPublicTry = epOverlay.publicTry !== false;
|
||||
|
||||
// Normalise params
|
||||
const paramSources = epOverlay.paramSources || {};
|
||||
const rawParams = Array.isArray(ep.params) ? ep.params : [];
|
||||
const params: ParamSpec[] = [];
|
||||
|
||||
for (const p of rawParams) {
|
||||
const name = sanitizeText(p.n);
|
||||
if (!name) continue;
|
||||
|
||||
const rawTy = sanitizeText(p.ty);
|
||||
const kind = parseKind(rawTy);
|
||||
const location = parseLocation(p.lc);
|
||||
const rawDesc = sanitizeText(p.ds);
|
||||
const isSecret = isSecretParam(name, rawDesc);
|
||||
const required = Boolean(p.rq);
|
||||
const defVal = p.df && p.df !== 'None' && p.df !== 'null' ? sanitizeText(p.df) : undefined;
|
||||
|
||||
let source: ParamSpec['source'] = 'user.file';
|
||||
if (paramSources[name]) {
|
||||
const mapped = paramSources[name];
|
||||
if (mapped === 'user.file') source = 'user.file';
|
||||
else if (mapped === 'secret') source = 'secret';
|
||||
else if (mapped.startsWith('option:')) source = { option: mapped.split(':')[1] };
|
||||
else if (mapped.startsWith('constant:')) source = { constant: mapped.split(':')[1] };
|
||||
} else {
|
||||
// Inference
|
||||
if (kind === 'file') {
|
||||
source = 'user.file';
|
||||
} else if (isSecret) {
|
||||
source = 'secret';
|
||||
} else if (defVal) {
|
||||
source = { constant: defVal };
|
||||
} else if (required) {
|
||||
misconfigured.push(`${id}.${name}`);
|
||||
}
|
||||
}
|
||||
|
||||
params.push({
|
||||
name,
|
||||
alias: p.al ? sanitizeText(p.al) : undefined,
|
||||
description: isSecret ? undefined : rawDesc, // Redact secret hints per PRD §7.11
|
||||
required,
|
||||
kind,
|
||||
rawType: rawTy,
|
||||
default: defVal,
|
||||
location,
|
||||
isSecret,
|
||||
source,
|
||||
});
|
||||
}
|
||||
|
||||
// Normalise variants from subs
|
||||
const rawSubs = Array.isArray(ep.subs) ? ep.subs : [];
|
||||
const variants: VariantSpec[] = [];
|
||||
const overlayVariants: any[] = epOverlay.variants || [];
|
||||
|
||||
for (const sub of rawSubs) {
|
||||
const subPath = sanitizeText(sub.p || '');
|
||||
const segments = subPath.replace(/\/+$/, '').split('/');
|
||||
const variantId = segments[segments.length - 1] || 'default';
|
||||
const url = baseUrl ? `${baseUrl.replace(/\/+$/, '')}/${subPath.replace(/^\/+/, '')}` : subPath;
|
||||
|
||||
const matchedOverlayVar = overlayVariants.find((v: any) => v.id === variantId);
|
||||
const label: Localised = matchedOverlayVar?.label || {
|
||||
en: sanitizeText(sub.d || variantId),
|
||||
};
|
||||
|
||||
variants.push({
|
||||
id: variantId,
|
||||
method: sanitizeText(sub.m || 'POST'),
|
||||
path: subPath,
|
||||
url,
|
||||
description: sanitizeText(sub.d),
|
||||
label,
|
||||
});
|
||||
}
|
||||
|
||||
const limits = {
|
||||
timeoutMs: epOverlay.limits?.timeoutMs ?? defaults.timeoutMs ?? 60000,
|
||||
retries: epOverlay.limits?.retries ?? defaults.retries ?? 0,
|
||||
maxUploadMB: epOverlay.limits?.maxUploadMB ?? defaults.maxUploadMB ?? 12,
|
||||
allowedMime: epOverlay.limits?.allowedMime ?? defaults.allowedMime ?? ['image/jpeg', 'image/png', 'image/webp'],
|
||||
rateLimit: epOverlay.limits?.rateLimit ?? defaults.rateLimit ?? 'tryit-default',
|
||||
};
|
||||
|
||||
const responseKind = epOverlay.response?.kind ?? defaults.response?.kind ?? 'binary-image';
|
||||
const responseDelivery = epOverlay.response?.delivery ?? defaults.response?.delivery ?? 'stream';
|
||||
|
||||
const hash = crypto
|
||||
.createHash('md5')
|
||||
.update(JSON.stringify({ id, fullUrl, params, variants, limits }))
|
||||
.digest('hex');
|
||||
|
||||
normalised.push({
|
||||
id,
|
||||
groupKey,
|
||||
groupTitle,
|
||||
title: sanitizeText(ep.title || id),
|
||||
summary: sanitizeText(ep.summary),
|
||||
description: sanitizeText(ep.desc),
|
||||
method: (ep.method ? sanitizeText(ep.method).toUpperCase() : 'POST') as any,
|
||||
url: fullUrl,
|
||||
baseUrl,
|
||||
path: epPath,
|
||||
responseType: sanitizeText(ep.resp || 'image/jpeg'),
|
||||
params,
|
||||
variants,
|
||||
enabled: isEnabled,
|
||||
publicTry: isPublicTry,
|
||||
limits,
|
||||
response: {
|
||||
kind: responseKind,
|
||||
delivery: responseDelivery,
|
||||
},
|
||||
hash,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Compute Diff against previous snapshot
|
||||
const diff = computeDiff(lastGoodSnapshot || [], normalised, misconfigured);
|
||||
lastGoodSnapshot = normalised;
|
||||
cachedEndpoints = normalised;
|
||||
lastDiffReport = diff;
|
||||
|
||||
return { endpoints: normalised, diff };
|
||||
} catch (err: any) {
|
||||
console.error('[Catalog] Error normalising catalogue:', err);
|
||||
if (lastGoodSnapshot) {
|
||||
console.warn('[Catalog] Falling back to last good snapshot');
|
||||
return { endpoints: lastGoodSnapshot, diff: lastDiffReport || createEmptyDiff() };
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function createEmptyDiff(): DiffReport {
|
||||
return {
|
||||
timestamp: new Date().toISOString(),
|
||||
addedEndpointIds: [],
|
||||
removedEndpointIds: [],
|
||||
changedEndpointIds: [],
|
||||
affectedTools: [],
|
||||
misconfiguredEndpoints: [],
|
||||
totalEndpoints: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function computeDiff(oldList: EndpointSpec[], newList: EndpointSpec[], misconfigured: string[]): DiffReport {
|
||||
const oldMap = new Map(oldList.map((e) => [e.id, e]));
|
||||
const newMap = new Map(newList.map((e) => [e.id, e]));
|
||||
|
||||
const added: string[] = [];
|
||||
const removed: string[] = [];
|
||||
const changed: string[] = [];
|
||||
|
||||
for (const [id, ep] of newMap.entries()) {
|
||||
if (!oldMap.has(id)) {
|
||||
added.push(id);
|
||||
} else if (oldMap.get(id)!.hash !== ep.hash) {
|
||||
changed.push(id);
|
||||
}
|
||||
}
|
||||
|
||||
for (const id of oldMap.keys()) {
|
||||
if (!newMap.has(id)) {
|
||||
removed.push(id);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
timestamp: new Date().toISOString(),
|
||||
addedEndpointIds: added,
|
||||
removedEndpointIds: removed,
|
||||
changedEndpointIds: changed,
|
||||
affectedTools: [],
|
||||
misconfiguredEndpoints: misconfigured,
|
||||
totalEndpoints: newList.length,
|
||||
};
|
||||
}
|
||||
|
||||
export function getEndpoints(): EndpointSpec[] {
|
||||
if (!cachedEndpoints) {
|
||||
normaliseCatalog();
|
||||
}
|
||||
return cachedEndpoints || [];
|
||||
}
|
||||
|
||||
export function getEndpointById(id: string): EndpointSpec | undefined {
|
||||
return getEndpoints().find((e) => e.id === id);
|
||||
}
|
||||
|
||||
export function getLastDiffReport(): DiffReport | null {
|
||||
return lastDiffReport;
|
||||
}
|
||||
Reference in New Issue
Block a user