Files
webb/src/app/robots.ts
T

27 lines
722 B
TypeScript
Raw Normal View History

import { MetadataRoute } from 'next';
import { getSiteConfig } from '@/lib/config/loader';
export default function robots(): MetadataRoute.Robots {
const baseUrl = (process.env.SITE_URL || getSiteConfig().seo.siteUrl || 'http://localhost:3000').replace(/\/$/, '');
return {
rules: [
{
userAgent: '*',
allow: ['/', '/tools', '/tools/', '/apps', '/apps/', '/trust', '/privacy', '/terms'],
disallow: ['/api/', '/go/', '/catalog-manager/', '/_next/', '/static/'],
},
{
userAgent: 'GPTBot',
disallow: ['/'],
},
{
userAgent: 'CCBot',
disallow: ['/'],
},
],
sitemap: `${baseUrl}/sitemap.xml`,
host: baseUrl,
};
}