import fs from 'fs'; import path from 'path'; import crypto from 'crypto'; import { EndpointSpec, ParamSpec, VariantSpec, DiffReport, Localised } from '@/types'; let cachedEndpoints: EndpointSpec[] | null = null; let lastGoodSnapshot: EndpointSpec[] | null = null; let lastDiffReport: DiffReport | null = null; function sanitizeText(str: any): string { if (typeof str !== 'string') return ''; // Strip non-printable control characters, trim, collapse whitespace return str.replace(/[\u0000-\u0008\u000B-\u001F\u007F]/g, '').trim(); } function parseKind(rawType: string): ParamSpec['kind'] { const t = (rawType || '').toLowerCase(); if (t.includes('file')) return 'file'; if (t.includes('int')) return 'integer'; if (t.includes('float') || t.includes('number') || t.includes('double')) return 'number'; if (t.includes('bool')) return 'boolean'; if (t.includes('json') || t.includes('object') || t.includes('array')) return 'json'; return 'string'; } function parseLocation(lc: string): ParamSpec['location'] { const l = (lc || '').toLowerCase(); if (l.includes('multipart') || l.includes('form-data')) return 'multipart'; if (l.includes('form') || l.includes('x-www-form-urlencoded')) return 'form'; if (l.includes('json')) return 'json'; if (l.includes('query')) return 'query'; if (l.includes('header')) return 'header'; if (l.includes('path')) return 'path'; return 'multipart'; } function isSecretParam(name: string, desc: string): boolean { const n = (name || '').toLowerCase(); const d = (desc || '').toLowerCase(); if (/secret|token|password|auth|key/.test(n)) return true; if (/authentication|api[-_ ]?key|bearer/.test(d)) return true; return false; } export function loadRawCatalog(): any { const catalogPath = process.env.CATALOG_PATH || path.join(process.cwd(), 'content', 'api.catalog.json'); if (!fs.existsSync(catalogPath)) { throw new Error(`Catalogue file not found at ${catalogPath}`); } const rawText = fs.readFileSync(catalogPath, 'utf8'); return JSON.parse(rawText); } export function loadOverlay(): any { const overlayPath = path.join(process.cwd(), 'content', 'api.overlay.json'); if (!fs.existsSync(overlayPath)) { return { schemaVersion: '1.0', defaults: {}, secrets: {}, rateLimits: {}, endpoints: {} }; } const rawText = fs.readFileSync(overlayPath, 'utf8'); return JSON.parse(rawText); } export function extractGroups(raw: any): any[] { if (!raw) return []; // (a) Array of groups if (Array.isArray(raw)) { // Check if it is a bare list of endpoints if (raw.length > 0 && raw[0].full_url && !raw[0].endpoints) { return [{ title: 'Default', endpoints: raw }]; } return raw; } // (b) Object with list under any key (apis, groups, data, etc.) for (const key of ['apis', 'groups', 'data', 'catalog', 'endpoints']) { if (Array.isArray(raw[key])) { if (key === 'endpoints') { return [raw]; } return raw[key]; } } // (c) Single group object if (raw && Array.isArray(raw.endpoints)) { return [raw]; } return []; } export function normaliseCatalog(rawInput?: any, overlayInput?: any): { endpoints: EndpointSpec[]; diff: DiffReport } { try { const raw = rawInput || loadRawCatalog(); const overlay = overlayInput || loadOverlay(); const defaults = overlay.defaults || {}; const overlayEndpoints = overlay.endpoints || {}; const groups = extractGroups(raw); const seenIds = new Set(); const normalised: EndpointSpec[] = []; const misconfigured: string[] = []; for (const group of groups) { const groupKey = sanitizeText(group.api_key || group.title || 'default'); const groupTitle = sanitizeText(group.title || groupKey); const rawEndpoints = Array.isArray(group.endpoints) ? group.endpoints : []; for (const ep of rawEndpoints) { const id = sanitizeText(ep.id); const fullUrl = sanitizeText(ep.full_url); if (!id || !fullUrl) { console.warn(`[Catalog] Skipping endpoint without id or full_url`, ep); continue; } if (seenIds.has(id)) { console.warn(`[Catalog] Duplicate endpoint ID detected: ${id}. Using first occurrence.`); continue; } seenIds.add(id); const epPath = sanitizeText(ep.path || ''); let baseUrl = fullUrl; if (epPath && fullUrl.endsWith(epPath)) { baseUrl = fullUrl.slice(0, fullUrl.length - epPath.length); } else { try { const parsedUrl = new URL(fullUrl); baseUrl = `${parsedUrl.protocol}//${parsedUrl.host}`; } catch { baseUrl = fullUrl; } } // Overlay overrides for this endpoint const epOverlay = overlayEndpoints[id] || {}; const isEnabled = epOverlay.enabled !== false; const isPublicTry = epOverlay.publicTry !== false; // Normalise params const paramSources = epOverlay.paramSources || {}; const rawParams = Array.isArray(ep.params) ? ep.params : []; const params: ParamSpec[] = []; for (const p of rawParams) { const name = sanitizeText(p.n); if (!name) continue; const rawTy = sanitizeText(p.ty); const kind = parseKind(rawTy); const location = parseLocation(p.lc); const rawDesc = sanitizeText(p.ds); const isSecret = isSecretParam(name, rawDesc); const required = Boolean(p.rq); const defVal = p.df && p.df !== 'None' && p.df !== 'null' ? sanitizeText(p.df) : undefined; let source: ParamSpec['source'] = 'user.file'; if (paramSources[name]) { const mapped = paramSources[name]; if (mapped === 'user.file') source = 'user.file'; else if (mapped === 'secret') source = 'secret'; else if (mapped.startsWith('option:')) source = { option: mapped.split(':')[1] }; else if (mapped.startsWith('constant:')) source = { constant: mapped.split(':')[1] }; } else { // Inference if (kind === 'file') { source = 'user.file'; } else if (isSecret) { source = 'secret'; } else if (defVal) { source = { constant: defVal }; } else if (required) { misconfigured.push(`${id}.${name}`); } } params.push({ name, alias: p.al ? sanitizeText(p.al) : undefined, description: isSecret ? undefined : rawDesc, // Redact secret hints per PRD ยง7.11 required, kind, rawType: rawTy, default: defVal, location, isSecret, source, }); } // Normalise variants from subs const rawSubs = Array.isArray(ep.subs) ? ep.subs : []; const variants: VariantSpec[] = []; const overlayVariants: any[] = epOverlay.variants || []; for (const sub of rawSubs) { const subPath = sanitizeText(sub.p || ''); const segments = subPath.replace(/\/+$/, '').split('/'); const variantId = segments[segments.length - 1] || 'default'; const url = baseUrl ? `${baseUrl.replace(/\/+$/, '')}/${subPath.replace(/^\/+/, '')}` : subPath; const matchedOverlayVar = overlayVariants.find((v: any) => v.id === variantId); const label: Localised = matchedOverlayVar?.label || { en: sanitizeText(sub.d || variantId), }; variants.push({ id: variantId, method: sanitizeText(sub.m || 'POST'), path: subPath, url, description: sanitizeText(sub.d), label, }); } const limits = { timeoutMs: epOverlay.limits?.timeoutMs ?? defaults.timeoutMs ?? 60000, retries: epOverlay.limits?.retries ?? defaults.retries ?? 0, maxUploadMB: epOverlay.limits?.maxUploadMB ?? defaults.maxUploadMB ?? 12, allowedMime: epOverlay.limits?.allowedMime ?? defaults.allowedMime ?? ['image/jpeg', 'image/png', 'image/webp'], rateLimit: epOverlay.limits?.rateLimit ?? defaults.rateLimit ?? 'tryit-default', }; const responseKind = epOverlay.response?.kind ?? defaults.response?.kind ?? 'binary-image'; const responseDelivery = epOverlay.response?.delivery ?? defaults.response?.delivery ?? 'stream'; const hash = crypto .createHash('md5') .update(JSON.stringify({ id, fullUrl, params, variants, limits })) .digest('hex'); normalised.push({ id, groupKey, groupTitle, title: sanitizeText(ep.title || id), summary: sanitizeText(ep.summary), description: sanitizeText(ep.desc), method: (ep.method ? sanitizeText(ep.method).toUpperCase() : 'POST') as any, url: fullUrl, baseUrl, path: epPath, responseType: sanitizeText(ep.resp || 'image/jpeg'), params, variants, enabled: isEnabled, publicTry: isPublicTry, limits, response: { kind: responseKind, delivery: responseDelivery, }, hash, }); } } // Compute Diff against previous snapshot const diff = computeDiff(lastGoodSnapshot || [], normalised, misconfigured); lastGoodSnapshot = normalised; cachedEndpoints = normalised; lastDiffReport = diff; return { endpoints: normalised, diff }; } catch (err: any) { console.error('[Catalog] Error normalising catalogue:', err); if (lastGoodSnapshot) { console.warn('[Catalog] Falling back to last good snapshot'); return { endpoints: lastGoodSnapshot, diff: lastDiffReport || createEmptyDiff() }; } throw err; } } function createEmptyDiff(): DiffReport { return { timestamp: new Date().toISOString(), addedEndpointIds: [], removedEndpointIds: [], changedEndpointIds: [], affectedTools: [], misconfiguredEndpoints: [], totalEndpoints: 0, }; } function computeDiff(oldList: EndpointSpec[], newList: EndpointSpec[], misconfigured: string[]): DiffReport { const oldMap = new Map(oldList.map((e) => [e.id, e])); const newMap = new Map(newList.map((e) => [e.id, e])); const added: string[] = []; const removed: string[] = []; const changed: string[] = []; for (const [id, ep] of newMap.entries()) { if (!oldMap.has(id)) { added.push(id); } else if (oldMap.get(id)!.hash !== ep.hash) { changed.push(id); } } for (const id of oldMap.keys()) { if (!newMap.has(id)) { removed.push(id); } } return { timestamp: new Date().toISOString(), addedEndpointIds: added, removedEndpointIds: removed, changedEndpointIds: changed, affectedTools: [], misconfiguredEndpoints: misconfigured, totalEndpoints: newList.length, }; } export function getEndpoints(): EndpointSpec[] { if (!cachedEndpoints) { normaliseCatalog(); } return cachedEndpoints || []; } export function getEndpointById(id: string): EndpointSpec | undefined { return getEndpoints().find((e) => e.id === id); } export function getLastDiffReport(): DiffReport | null { return lastDiffReport; }