334 lines
11 KiB
TypeScript
334 lines
11 KiB
TypeScript
import fs from 'fs';
|
|
import path from 'path';
|
|
import crypto from 'crypto';
|
|
import { EndpointSpec, ParamSpec, VariantSpec, DiffReport, Localised } from '@/types';
|
|
|
|
let cachedEndpoints: EndpointSpec[] | null = null;
|
|
let lastGoodSnapshot: EndpointSpec[] | null = null;
|
|
let lastDiffReport: DiffReport | null = null;
|
|
|
|
function sanitizeText(str: any): string {
|
|
if (typeof str !== 'string') return '';
|
|
// Strip non-printable control characters, trim, collapse whitespace
|
|
return str.replace(/[\u0000-\u0008\u000B-\u001F\u007F]/g, '').trim();
|
|
}
|
|
|
|
function parseKind(rawType: string): ParamSpec['kind'] {
|
|
const t = (rawType || '').toLowerCase();
|
|
if (t.includes('file')) return 'file';
|
|
if (t.includes('int')) return 'integer';
|
|
if (t.includes('float') || t.includes('number') || t.includes('double')) return 'number';
|
|
if (t.includes('bool')) return 'boolean';
|
|
if (t.includes('json') || t.includes('object') || t.includes('array')) return 'json';
|
|
return 'string';
|
|
}
|
|
|
|
function parseLocation(lc: string): ParamSpec['location'] {
|
|
const l = (lc || '').toLowerCase();
|
|
if (l.includes('multipart') || l.includes('form-data')) return 'multipart';
|
|
if (l.includes('form') || l.includes('x-www-form-urlencoded')) return 'form';
|
|
if (l.includes('json')) return 'json';
|
|
if (l.includes('query')) return 'query';
|
|
if (l.includes('header')) return 'header';
|
|
if (l.includes('path')) return 'path';
|
|
return 'multipart';
|
|
}
|
|
|
|
function isSecretParam(name: string, desc: string): boolean {
|
|
const n = (name || '').toLowerCase();
|
|
const d = (desc || '').toLowerCase();
|
|
if (/secret|token|password|auth|key/.test(n)) return true;
|
|
if (/authentication|api[-_ ]?key|bearer/.test(d)) return true;
|
|
return false;
|
|
}
|
|
|
|
export function loadRawCatalog(): any {
|
|
const catalogPath = process.env.CATALOG_PATH || path.join(process.cwd(), 'content', 'api.catalog.json');
|
|
if (!fs.existsSync(catalogPath)) {
|
|
throw new Error(`Catalogue file not found at ${catalogPath}`);
|
|
}
|
|
const rawText = fs.readFileSync(catalogPath, 'utf8');
|
|
return JSON.parse(rawText);
|
|
}
|
|
|
|
export function loadOverlay(): any {
|
|
const overlayPath = path.join(process.cwd(), 'content', 'api.overlay.json');
|
|
if (!fs.existsSync(overlayPath)) {
|
|
return { schemaVersion: '1.0', defaults: {}, secrets: {}, rateLimits: {}, endpoints: {} };
|
|
}
|
|
const rawText = fs.readFileSync(overlayPath, 'utf8');
|
|
return JSON.parse(rawText);
|
|
}
|
|
|
|
export function extractGroups(raw: any): any[] {
|
|
if (!raw) return [];
|
|
// (a) Array of groups
|
|
if (Array.isArray(raw)) {
|
|
// Check if it is a bare list of endpoints
|
|
if (raw.length > 0 && raw[0].full_url && !raw[0].endpoints) {
|
|
return [{ title: 'Default', endpoints: raw }];
|
|
}
|
|
return raw;
|
|
}
|
|
// (b) Object with list under any key (apis, groups, data, etc.)
|
|
for (const key of ['apis', 'groups', 'data', 'catalog', 'endpoints']) {
|
|
if (Array.isArray(raw[key])) {
|
|
if (key === 'endpoints') {
|
|
return [raw];
|
|
}
|
|
return raw[key];
|
|
}
|
|
}
|
|
// (c) Single group object
|
|
if (raw && Array.isArray(raw.endpoints)) {
|
|
return [raw];
|
|
}
|
|
return [];
|
|
}
|
|
|
|
export function normaliseCatalog(rawInput?: any, overlayInput?: any): { endpoints: EndpointSpec[]; diff: DiffReport } {
|
|
try {
|
|
const raw = rawInput || loadRawCatalog();
|
|
const overlay = overlayInput || loadOverlay();
|
|
const defaults = overlay.defaults || {};
|
|
const overlayEndpoints = overlay.endpoints || {};
|
|
|
|
const groups = extractGroups(raw);
|
|
const seenIds = new Set<string>();
|
|
const normalised: EndpointSpec[] = [];
|
|
const misconfigured: string[] = [];
|
|
|
|
for (const group of groups) {
|
|
const groupKey = sanitizeText(group.api_key || group.title || 'default');
|
|
const groupTitle = sanitizeText(group.title || groupKey);
|
|
const rawEndpoints = Array.isArray(group.endpoints) ? group.endpoints : [];
|
|
|
|
for (const ep of rawEndpoints) {
|
|
const id = sanitizeText(ep.id);
|
|
const fullUrl = sanitizeText(ep.full_url);
|
|
|
|
if (!id || !fullUrl) {
|
|
console.warn(`[Catalog] Skipping endpoint without id or full_url`, ep);
|
|
continue;
|
|
}
|
|
|
|
if (seenIds.has(id)) {
|
|
console.warn(`[Catalog] Duplicate endpoint ID detected: ${id}. Using first occurrence.`);
|
|
continue;
|
|
}
|
|
seenIds.add(id);
|
|
|
|
const epPath = sanitizeText(ep.path || '');
|
|
let baseUrl = fullUrl;
|
|
if (epPath && fullUrl.endsWith(epPath)) {
|
|
baseUrl = fullUrl.slice(0, fullUrl.length - epPath.length);
|
|
} else {
|
|
try {
|
|
const parsedUrl = new URL(fullUrl);
|
|
baseUrl = `${parsedUrl.protocol}//${parsedUrl.host}`;
|
|
} catch {
|
|
baseUrl = fullUrl;
|
|
}
|
|
}
|
|
|
|
// Overlay overrides for this endpoint
|
|
const epOverlay = overlayEndpoints[id] || {};
|
|
const isEnabled = epOverlay.enabled !== false;
|
|
const isPublicTry = epOverlay.publicTry !== false;
|
|
|
|
// Normalise params
|
|
const paramSources = epOverlay.paramSources || {};
|
|
const rawParams = Array.isArray(ep.params) ? ep.params : [];
|
|
const params: ParamSpec[] = [];
|
|
|
|
for (const p of rawParams) {
|
|
const name = sanitizeText(p.n);
|
|
if (!name) continue;
|
|
|
|
const rawTy = sanitizeText(p.ty);
|
|
const kind = parseKind(rawTy);
|
|
const location = parseLocation(p.lc);
|
|
const rawDesc = sanitizeText(p.ds);
|
|
const isSecret = isSecretParam(name, rawDesc);
|
|
const required = Boolean(p.rq);
|
|
const defVal = p.df && p.df !== 'None' && p.df !== 'null' ? sanitizeText(p.df) : undefined;
|
|
|
|
let source: ParamSpec['source'] = 'user.file';
|
|
if (paramSources[name]) {
|
|
const mapped = paramSources[name];
|
|
if (mapped === 'user.file') source = 'user.file';
|
|
else if (mapped === 'secret') source = 'secret';
|
|
else if (mapped.startsWith('option:')) source = { option: mapped.split(':')[1] };
|
|
else if (mapped.startsWith('constant:')) source = { constant: mapped.split(':')[1] };
|
|
} else {
|
|
// Inference
|
|
if (kind === 'file') {
|
|
source = 'user.file';
|
|
} else if (isSecret) {
|
|
source = 'secret';
|
|
} else if (defVal) {
|
|
source = { constant: defVal };
|
|
} else if (required) {
|
|
misconfigured.push(`${id}.${name}`);
|
|
}
|
|
}
|
|
|
|
params.push({
|
|
name,
|
|
alias: p.al ? sanitizeText(p.al) : undefined,
|
|
description: isSecret ? undefined : rawDesc, // Redact secret hints per PRD §7.11
|
|
required,
|
|
kind,
|
|
rawType: rawTy,
|
|
default: defVal,
|
|
location,
|
|
isSecret,
|
|
source,
|
|
});
|
|
}
|
|
|
|
// Normalise variants from subs
|
|
const rawSubs = Array.isArray(ep.subs) ? ep.subs : [];
|
|
const variants: VariantSpec[] = [];
|
|
const overlayVariants: any[] = epOverlay.variants || [];
|
|
|
|
for (const sub of rawSubs) {
|
|
const subPath = sanitizeText(sub.p || '');
|
|
const segments = subPath.replace(/\/+$/, '').split('/');
|
|
const variantId = segments[segments.length - 1] || 'default';
|
|
const url = baseUrl ? `${baseUrl.replace(/\/+$/, '')}/${subPath.replace(/^\/+/, '')}` : subPath;
|
|
|
|
const matchedOverlayVar = overlayVariants.find((v: any) => v.id === variantId);
|
|
const label: Localised = matchedOverlayVar?.label || {
|
|
en: sanitizeText(sub.d || variantId),
|
|
};
|
|
|
|
variants.push({
|
|
id: variantId,
|
|
method: sanitizeText(sub.m || 'POST'),
|
|
path: subPath,
|
|
url,
|
|
description: sanitizeText(sub.d),
|
|
label,
|
|
});
|
|
}
|
|
|
|
const limits = {
|
|
timeoutMs: epOverlay.limits?.timeoutMs ?? defaults.timeoutMs ?? 60000,
|
|
retries: epOverlay.limits?.retries ?? defaults.retries ?? 0,
|
|
maxUploadMB: epOverlay.limits?.maxUploadMB ?? defaults.maxUploadMB ?? 12,
|
|
allowedMime: epOverlay.limits?.allowedMime ?? defaults.allowedMime ?? ['image/jpeg', 'image/png', 'image/webp'],
|
|
rateLimit: epOverlay.limits?.rateLimit ?? defaults.rateLimit ?? 'tryit-default',
|
|
};
|
|
|
|
const responseKind = epOverlay.response?.kind ?? defaults.response?.kind ?? 'binary-image';
|
|
const responseDelivery = epOverlay.response?.delivery ?? defaults.response?.delivery ?? 'stream';
|
|
|
|
const hash = crypto
|
|
.createHash('md5')
|
|
.update(JSON.stringify({ id, fullUrl, params, variants, limits }))
|
|
.digest('hex');
|
|
|
|
normalised.push({
|
|
id,
|
|
groupKey,
|
|
groupTitle,
|
|
title: sanitizeText(ep.title || id),
|
|
summary: sanitizeText(ep.summary),
|
|
description: sanitizeText(ep.desc),
|
|
method: (ep.method ? sanitizeText(ep.method).toUpperCase() : 'POST') as any,
|
|
url: fullUrl,
|
|
baseUrl,
|
|
path: epPath,
|
|
responseType: sanitizeText(ep.resp || 'image/jpeg'),
|
|
params,
|
|
variants,
|
|
enabled: isEnabled,
|
|
publicTry: isPublicTry,
|
|
limits,
|
|
response: {
|
|
kind: responseKind,
|
|
delivery: responseDelivery,
|
|
},
|
|
hash,
|
|
});
|
|
}
|
|
}
|
|
|
|
// Compute Diff against previous snapshot
|
|
const diff = computeDiff(lastGoodSnapshot || [], normalised, misconfigured);
|
|
lastGoodSnapshot = normalised;
|
|
cachedEndpoints = normalised;
|
|
lastDiffReport = diff;
|
|
|
|
return { endpoints: normalised, diff };
|
|
} catch (err: any) {
|
|
console.error('[Catalog] Error normalising catalogue:', err);
|
|
if (lastGoodSnapshot) {
|
|
console.warn('[Catalog] Falling back to last good snapshot');
|
|
return { endpoints: lastGoodSnapshot, diff: lastDiffReport || createEmptyDiff() };
|
|
}
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
function createEmptyDiff(): DiffReport {
|
|
return {
|
|
timestamp: new Date().toISOString(),
|
|
addedEndpointIds: [],
|
|
removedEndpointIds: [],
|
|
changedEndpointIds: [],
|
|
affectedTools: [],
|
|
misconfiguredEndpoints: [],
|
|
totalEndpoints: 0,
|
|
};
|
|
}
|
|
|
|
function computeDiff(oldList: EndpointSpec[], newList: EndpointSpec[], misconfigured: string[]): DiffReport {
|
|
const oldMap = new Map(oldList.map((e) => [e.id, e]));
|
|
const newMap = new Map(newList.map((e) => [e.id, e]));
|
|
|
|
const added: string[] = [];
|
|
const removed: string[] = [];
|
|
const changed: string[] = [];
|
|
|
|
for (const [id, ep] of newMap.entries()) {
|
|
if (!oldMap.has(id)) {
|
|
added.push(id);
|
|
} else if (oldMap.get(id)!.hash !== ep.hash) {
|
|
changed.push(id);
|
|
}
|
|
}
|
|
|
|
for (const id of oldMap.keys()) {
|
|
if (!newMap.has(id)) {
|
|
removed.push(id);
|
|
}
|
|
}
|
|
|
|
return {
|
|
timestamp: new Date().toISOString(),
|
|
addedEndpointIds: added,
|
|
removedEndpointIds: removed,
|
|
changedEndpointIds: changed,
|
|
affectedTools: [],
|
|
misconfiguredEndpoints: misconfigured,
|
|
totalEndpoints: newList.length,
|
|
};
|
|
}
|
|
|
|
export function getEndpoints(): EndpointSpec[] {
|
|
if (!cachedEndpoints) {
|
|
normaliseCatalog();
|
|
}
|
|
return cachedEndpoints || [];
|
|
}
|
|
|
|
export function getEndpointById(id: string): EndpointSpec | undefined {
|
|
return getEndpoints().find((e) => e.id === id);
|
|
}
|
|
|
|
export function getLastDiffReport(): DiffReport | null {
|
|
return lastDiffReport;
|
|
}
|