Files
webb/src/lib/catalog/loader.ts
T

334 lines
11 KiB
TypeScript

import fs from 'fs';
import path from 'path';
import crypto from 'crypto';
import { EndpointSpec, ParamSpec, VariantSpec, DiffReport, Localised } from '@/types';
let cachedEndpoints: EndpointSpec[] | null = null;
let lastGoodSnapshot: EndpointSpec[] | null = null;
let lastDiffReport: DiffReport | null = null;
function sanitizeText(str: any): string {
if (typeof str !== 'string') return '';
// Strip non-printable control characters, trim, collapse whitespace
return str.replace(/[\u0000-\u0008\u000B-\u001F\u007F]/g, '').trim();
}
function parseKind(rawType: string): ParamSpec['kind'] {
const t = (rawType || '').toLowerCase();
if (t.includes('file')) return 'file';
if (t.includes('int')) return 'integer';
if (t.includes('float') || t.includes('number') || t.includes('double')) return 'number';
if (t.includes('bool')) return 'boolean';
if (t.includes('json') || t.includes('object') || t.includes('array')) return 'json';
return 'string';
}
function parseLocation(lc: string): ParamSpec['location'] {
const l = (lc || '').toLowerCase();
if (l.includes('multipart') || l.includes('form-data')) return 'multipart';
if (l.includes('form') || l.includes('x-www-form-urlencoded')) return 'form';
if (l.includes('json')) return 'json';
if (l.includes('query')) return 'query';
if (l.includes('header')) return 'header';
if (l.includes('path')) return 'path';
return 'multipart';
}
function isSecretParam(name: string, desc: string): boolean {
const n = (name || '').toLowerCase();
const d = (desc || '').toLowerCase();
if (/secret|token|password|auth|key/.test(n)) return true;
if (/authentication|api[-_ ]?key|bearer/.test(d)) return true;
return false;
}
export function loadRawCatalog(): any {
const catalogPath = process.env.CATALOG_PATH || path.join(process.cwd(), 'content', 'api.catalog.json');
if (!fs.existsSync(catalogPath)) {
throw new Error(`Catalogue file not found at ${catalogPath}`);
}
const rawText = fs.readFileSync(catalogPath, 'utf8');
return JSON.parse(rawText);
}
export function loadOverlay(): any {
const overlayPath = path.join(process.cwd(), 'content', 'api.overlay.json');
if (!fs.existsSync(overlayPath)) {
return { schemaVersion: '1.0', defaults: {}, secrets: {}, rateLimits: {}, endpoints: {} };
}
const rawText = fs.readFileSync(overlayPath, 'utf8');
return JSON.parse(rawText);
}
export function extractGroups(raw: any): any[] {
if (!raw) return [];
// (a) Array of groups
if (Array.isArray(raw)) {
// Check if it is a bare list of endpoints
if (raw.length > 0 && raw[0].full_url && !raw[0].endpoints) {
return [{ title: 'Default', endpoints: raw }];
}
return raw;
}
// (b) Object with list under any key (apis, groups, data, etc.)
for (const key of ['apis', 'groups', 'data', 'catalog', 'endpoints']) {
if (Array.isArray(raw[key])) {
if (key === 'endpoints') {
return [raw];
}
return raw[key];
}
}
// (c) Single group object
if (raw && Array.isArray(raw.endpoints)) {
return [raw];
}
return [];
}
export function normaliseCatalog(rawInput?: any, overlayInput?: any): { endpoints: EndpointSpec[]; diff: DiffReport } {
try {
const raw = rawInput || loadRawCatalog();
const overlay = overlayInput || loadOverlay();
const defaults = overlay.defaults || {};
const overlayEndpoints = overlay.endpoints || {};
const groups = extractGroups(raw);
const seenIds = new Set<string>();
const normalised: EndpointSpec[] = [];
const misconfigured: string[] = [];
for (const group of groups) {
const groupKey = sanitizeText(group.api_key || group.title || 'default');
const groupTitle = sanitizeText(group.title || groupKey);
const rawEndpoints = Array.isArray(group.endpoints) ? group.endpoints : [];
for (const ep of rawEndpoints) {
const id = sanitizeText(ep.id);
const fullUrl = sanitizeText(ep.full_url);
if (!id || !fullUrl) {
console.warn(`[Catalog] Skipping endpoint without id or full_url`, ep);
continue;
}
if (seenIds.has(id)) {
console.warn(`[Catalog] Duplicate endpoint ID detected: ${id}. Using first occurrence.`);
continue;
}
seenIds.add(id);
const epPath = sanitizeText(ep.path || '');
let baseUrl = fullUrl;
if (epPath && fullUrl.endsWith(epPath)) {
baseUrl = fullUrl.slice(0, fullUrl.length - epPath.length);
} else {
try {
const parsedUrl = new URL(fullUrl);
baseUrl = `${parsedUrl.protocol}//${parsedUrl.host}`;
} catch {
baseUrl = fullUrl;
}
}
// Overlay overrides for this endpoint
const epOverlay = overlayEndpoints[id] || {};
const isEnabled = epOverlay.enabled !== false;
const isPublicTry = epOverlay.publicTry !== false;
// Normalise params
const paramSources = epOverlay.paramSources || {};
const rawParams = Array.isArray(ep.params) ? ep.params : [];
const params: ParamSpec[] = [];
for (const p of rawParams) {
const name = sanitizeText(p.n);
if (!name) continue;
const rawTy = sanitizeText(p.ty);
const kind = parseKind(rawTy);
const location = parseLocation(p.lc);
const rawDesc = sanitizeText(p.ds);
const isSecret = isSecretParam(name, rawDesc);
const required = Boolean(p.rq);
const defVal = p.df && p.df !== 'None' && p.df !== 'null' ? sanitizeText(p.df) : undefined;
let source: ParamSpec['source'] = 'user.file';
if (paramSources[name]) {
const mapped = paramSources[name];
if (mapped === 'user.file') source = 'user.file';
else if (mapped === 'secret') source = 'secret';
else if (mapped.startsWith('option:')) source = { option: mapped.split(':')[1] };
else if (mapped.startsWith('constant:')) source = { constant: mapped.split(':')[1] };
} else {
// Inference
if (kind === 'file') {
source = 'user.file';
} else if (isSecret) {
source = 'secret';
} else if (defVal) {
source = { constant: defVal };
} else if (required) {
misconfigured.push(`${id}.${name}`);
}
}
params.push({
name,
alias: p.al ? sanitizeText(p.al) : undefined,
description: isSecret ? undefined : rawDesc, // Redact secret hints per PRD §7.11
required,
kind,
rawType: rawTy,
default: defVal,
location,
isSecret,
source,
});
}
// Normalise variants from subs
const rawSubs = Array.isArray(ep.subs) ? ep.subs : [];
const variants: VariantSpec[] = [];
const overlayVariants: any[] = epOverlay.variants || [];
for (const sub of rawSubs) {
const subPath = sanitizeText(sub.p || '');
const segments = subPath.replace(/\/+$/, '').split('/');
const variantId = segments[segments.length - 1] || 'default';
const url = baseUrl ? `${baseUrl.replace(/\/+$/, '')}/${subPath.replace(/^\/+/, '')}` : subPath;
const matchedOverlayVar = overlayVariants.find((v: any) => v.id === variantId);
const label: Localised = matchedOverlayVar?.label || {
en: sanitizeText(sub.d || variantId),
};
variants.push({
id: variantId,
method: sanitizeText(sub.m || 'POST'),
path: subPath,
url,
description: sanitizeText(sub.d),
label,
});
}
const limits = {
timeoutMs: epOverlay.limits?.timeoutMs ?? defaults.timeoutMs ?? 60000,
retries: epOverlay.limits?.retries ?? defaults.retries ?? 0,
maxUploadMB: epOverlay.limits?.maxUploadMB ?? defaults.maxUploadMB ?? 12,
allowedMime: epOverlay.limits?.allowedMime ?? defaults.allowedMime ?? ['image/jpeg', 'image/png', 'image/webp'],
rateLimit: epOverlay.limits?.rateLimit ?? defaults.rateLimit ?? 'tryit-default',
};
const responseKind = epOverlay.response?.kind ?? defaults.response?.kind ?? 'binary-image';
const responseDelivery = epOverlay.response?.delivery ?? defaults.response?.delivery ?? 'stream';
const hash = crypto
.createHash('md5')
.update(JSON.stringify({ id, fullUrl, params, variants, limits }))
.digest('hex');
normalised.push({
id,
groupKey,
groupTitle,
title: sanitizeText(ep.title || id),
summary: sanitizeText(ep.summary),
description: sanitizeText(ep.desc),
method: (ep.method ? sanitizeText(ep.method).toUpperCase() : 'POST') as any,
url: fullUrl,
baseUrl,
path: epPath,
responseType: sanitizeText(ep.resp || 'image/jpeg'),
params,
variants,
enabled: isEnabled,
publicTry: isPublicTry,
limits,
response: {
kind: responseKind,
delivery: responseDelivery,
},
hash,
});
}
}
// Compute Diff against previous snapshot
const diff = computeDiff(lastGoodSnapshot || [], normalised, misconfigured);
lastGoodSnapshot = normalised;
cachedEndpoints = normalised;
lastDiffReport = diff;
return { endpoints: normalised, diff };
} catch (err: any) {
console.error('[Catalog] Error normalising catalogue:', err);
if (lastGoodSnapshot) {
console.warn('[Catalog] Falling back to last good snapshot');
return { endpoints: lastGoodSnapshot, diff: lastDiffReport || createEmptyDiff() };
}
throw err;
}
}
function createEmptyDiff(): DiffReport {
return {
timestamp: new Date().toISOString(),
addedEndpointIds: [],
removedEndpointIds: [],
changedEndpointIds: [],
affectedTools: [],
misconfiguredEndpoints: [],
totalEndpoints: 0,
};
}
function computeDiff(oldList: EndpointSpec[], newList: EndpointSpec[], misconfigured: string[]): DiffReport {
const oldMap = new Map(oldList.map((e) => [e.id, e]));
const newMap = new Map(newList.map((e) => [e.id, e]));
const added: string[] = [];
const removed: string[] = [];
const changed: string[] = [];
for (const [id, ep] of newMap.entries()) {
if (!oldMap.has(id)) {
added.push(id);
} else if (oldMap.get(id)!.hash !== ep.hash) {
changed.push(id);
}
}
for (const id of oldMap.keys()) {
if (!newMap.has(id)) {
removed.push(id);
}
}
return {
timestamp: new Date().toISOString(),
addedEndpointIds: added,
removedEndpointIds: removed,
changedEndpointIds: changed,
affectedTools: [],
misconfiguredEndpoints: misconfigured,
totalEndpoints: newList.length,
};
}
export function getEndpoints(): EndpointSpec[] {
if (!cachedEndpoints) {
normaliseCatalog();
}
return cachedEndpoints || [];
}
export function getEndpointById(id: string): EndpointSpec | undefined {
return getEndpoints().find((e) => e.id === id);
}
export function getLastDiffReport(): DiffReport | null {
return lastDiffReport;
}