♻️ refactor: refactor the sitemap implement (#4012)

* ✨ feat: Add new sitemap

* 🐛 fix: Fix url

* ✅ test: Add test

* 🐛 fix: Fix host

* ✅ test: Fix test

* ✅ test: Fix test

* 🐛 fix: Fix alternative

* 🐛 fix: Try to fix

* 🐛 fix: Fix build

* 🐛 fix: Fix build

* 🔧 chore: Update git ignore

* 🐛 fix: Fix review problem

.gitignore

@@ -55,6 +55,8 @@ next-env.d.ts
 .next
 .env
 public/*.js
+public/sitemap.xml
+public/sitemap-index.xml
 bun.lockb
 sitemap*.xml
 robots.txt

next-sitemap.config.mjs

@@ -1,53 +0,0 @@
-import { glob } from 'glob';
-
-const isVercelPreview = process.env.VERCEL === '1' && process.env.VERCEL_ENV !== 'production';
-
-const vercelPreviewUrl = `https://${process.env.VERCEL_URL}`;
-
-const siteUrl = isVercelPreview ? vercelPreviewUrl : 'https://lobechat.com';
-
-/** @type {import('next-sitemap').IConfig} */
-const config = {
-  // next-sitemap does not work with app dir inside the /src dir (and have other problems e.g. with route groups)
-  // https://github.com/iamvishnusankar/next-sitemap/issues/700#issuecomment-1759458127
-  // https://github.com/iamvishnusankar/next-sitemap/issues/701
-  // additionalPaths is a workaround for this (once the issues are fixed, we can remove it)
-  additionalPaths: async () => {
-    const routes = await glob('src/app/**/page.{md,mdx,ts,tsx}', {
-      cwd: new URL('.', import.meta.url).pathname,
-    });
-
-    // https://nextjs.org/docs/app/building-your-application/routing/colocation#private-folders
-    const publicRoutes = routes.filter(
-      (page) => !page.split('/').some((folder) => folder.startsWith('_')),
-    );
-
-    // https://nextjs.org/docs/app/building-your-application/routing/colocation#route-groups
-    const publicRoutesWithoutRouteGroups = publicRoutes.map((page) =>
-      page
-        .split('/')
-        .filter((folder) => !folder.startsWith('(') && !folder.endsWith(')'))
-        .join('/'),
-    );
-
-    const locs = publicRoutesWithoutRouteGroups.map((route) => {
-      const path = route.replace(/^src\/app/, '').replace(/\/[^/]+$/, '');
-      const loc = path === '' ? siteUrl : `${siteUrl}/${path}`;
-
-      return loc;
-    });
-
-    const paths = locs.map((loc) => ({
-      changefreq: 'daily',
-      lastmod: new Date().toISOString(),
-      loc,
-      priority: 0.7,
-    }));
-
-    return paths;
-  },
-  generateRobotsTxt: true,
-  siteUrl,
-};
-
-export default config;

next.config.mjs

@@ -107,6 +107,16 @@ const nextConfig = {
   output: buildWithDocker ? 'standalone' : undefined,
   reactStrictMode: true,
   redirects: async () => [
+    {
+      destination: '/sitemap-index.xml',
+      permanent: true,
+      source: '/sitemap.xml',
+    },
+    {
+      destination: '/discover',
+      permanent: true,
+      source: '/market',
+    },
     {
       destination: '/settings/common',
       permanent: true,

package.json

@@ -29,11 +29,11 @@
     "build": "next build",
     "postbuild": "npm run build-sitemap && npm run build-migrate-db",
     "build-migrate-db": "bun run db:migrate",
-    "build-sitemap": "next-sitemap --config next-sitemap.config.mjs",
+    "build-sitemap": "tsx ./scripts/buildSitemapIndex/index.ts",
     "build:analyze": "ANALYZE=true next build",
     "build:docker": "DOCKER=true next build && npm run build-sitemap",
     "db:generate": "drizzle-kit generate",
-    "db:migrate": "MIGRATION_DB=1 tsx scripts/migrateServerDB/index.ts",
+    "db:migrate": "MIGRATION_DB=1 tsx ./scripts/migrateServerDB/index.ts",
     "db:push": "drizzle-kit push",
     "db:push-test": "NODE_ENV=test drizzle-kit push",
     "db:studio": "drizzle-kit studio",
@@ -65,11 +65,11 @@
     "test:update": "vitest -u",
     "type-check": "tsc --noEmit",
     "webhook:ngrok": "ngrok http http://localhost:3011",
-    "workflow:docs": "tsx scripts/docsWorkflow/index.ts",
-    "workflow:i18n": "tsx scripts/i18nWorkflow/index.ts",
+    "workflow:docs": "tsx ./scripts/docsWorkflow/index.ts",
+    "workflow:i18n": "tsx ./scripts/i18nWorkflow/index.ts",
     "workflow:mdx": "tsx ./scripts/mdxWorkflow/index.ts",
     "workflow:mdx-with-lint": "tsx ./scripts/mdxWorkflow/index.ts && eslint \"docs/**/*.mdx\" --quiet --fix",
-    "workflow:readme": "tsx scripts/readmeWorkflow/index.ts"
+    "workflow:readme": "tsx ./scripts/readmeWorkflow/index.ts"
   },
   "lint-staged": {
     "*.md": [
@@ -172,7 +172,6 @@
     "next": "14.2.8",
     "next-auth": "beta",
     "next-mdx-remote": "^4.4.1",
-    "next-sitemap": "^4.2.3",
     "nextjs-toploader": "^3.6.15",
     "numeral": "^2.0.6",
     "nuqs": "^1.17.8",

scripts/buildSitemapIndex/index.ts

@@ -0,0 +1,12 @@
+import { writeFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+
+import { sitemapModule } from '@/server/sitemap';
+
+const genSitemap = () => {
+  const sitemapIndexXML = sitemapModule.getIndex();
+  const filename = resolve(__dirname, '../../', 'public', 'sitemap-index.xml');
+  writeFileSync(filename, sitemapIndexXML);
+};
+
+genSitemap();

src/app/(main)/discover/(detail)/assistant/[slug]/page.tsx

@@ -105,6 +105,7 @@ const Page = async ({ params, searchParams }: Props) => {
           />
         }
         /* ↓ cloud slot ↓ */
+
         /* ↑ cloud slot ↑ */
       >
         <Temp data={data} identifier={identifier} />

src/app/(main)/discover/(detail)/model/[...slugs]/features/ProviderList/ProviderItem.tsx

@@ -67,6 +67,7 @@ const ProviderItem = memo<ProviderItemProps>(({ mobile, modelId, identifier }) =
         : '--',
     },
     /* ↓ cloud slot ↓ */
+
     /* ↑ cloud slot ↑ */
   ];
 

src/app/(main)/discover/(detail)/model/[...slugs]/page.tsx

@@ -100,6 +100,7 @@ const Page = async ({ params, searchParams }: Props) => {
         mobile={mobile}
         sidebar={<InfoSidebar data={data} identifier={identifier} mobile={mobile} />}
         /* ↓ cloud slot ↓ */
+
         /* ↑ cloud slot ↑ */
       >
         <ProviderList data={providerData} identifier={identifier} mobile={mobile} />

src/app/(main)/discover/(detail)/plugin/[slug]/page.tsx

@@ -89,6 +89,7 @@ const Page = async ({ params, searchParams }: Props) => {
         mobile={mobile}
         sidebar={<InfoSidebar data={data} identifier={identifier} mobile={mobile} />}
         /* ↓ cloud slot ↓ */
+
         /* ↑ cloud slot ↑ */
       >
         <ParameterList data={data} />

src/app/(main)/discover/(detail)/provider/[slug]/features/ModelList/ModelItem.tsx

@@ -79,6 +79,7 @@ const ModelItem = memo<SuggestionItemProps>(({ mobile, meta, identifier }) => {
         : '--',
     },
     /* ↓ cloud slot ↓ */
+
     /* ↑ cloud slot ↑ */
   ];
 

src/app/(main)/discover/(detail)/provider/[slug]/page.tsx

@@ -102,6 +102,7 @@ const Page = async ({ params, searchParams }: Props) => {
         mobile={mobile}
         sidebar={<InfoSidebar data={data} identifier={identifier} />}
         /* ↓ cloud slot ↓ */
+
         /* ↑ cloud slot ↑ */
       >
         <ModelList identifier={identifier} mobile={mobile} modelData={modelData} />

src/app/(main)/discover/(list)/_layout/Desktop/Nav.tsx

@@ -108,6 +108,7 @@ const Nav = memo(() => {
       {!isHome && !isProviders && (
         <Flexbox align={'center'} gap={4} horizontal>
           {/* ↓ cloud slot ↓ */}
+
           {/* ↑ cloud slot ↑ */}
         </Flexbox>
       )}

src/app/(main)/discover/_layout/Desktop/index.tsx

@@ -14,6 +14,7 @@ const Layout = ({ children }: PropsWithChildren) => {
         {children}
       </Flexbox>
       {/* ↓ cloud slot ↓ */}
+
       {/* ↑ cloud slot ↑ */}
     </>
   );

src/app/page.tsx

@@ -1,6 +1,6 @@
 import { Metadata } from 'next';
 
-import { getCanonicalUrl } from '@/const/url';
+import { getCanonicalUrl } from '@/server/utils/url';
 
 import Client from './(loading)/Client';
 import Redirect from './(loading)/Redirect';

src/app/robots.tsx

@@ -0,0 +1,16 @@
+import { MetadataRoute } from 'next';
+
+import { sitemapModule } from '@/server/sitemap';
+import { getCanonicalUrl } from '@/server/utils/url';
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    host: getCanonicalUrl(),
+    rules: {
+      allow: ['/'],
+      disallow: ['/api/*'],
+      userAgent: '*',
+    },
+    sitemap: sitemapModule.getRobots(),
+  };
+}

src/app/sitemap.tsx

@@ -0,0 +1,30 @@
+import { MetadataRoute } from 'next';
+
+import { SitemapType, sitemapModule } from '@/server/sitemap';
+
+export const generateSitemaps = async () => {
+  // Fetch the total number of products and calculate the number of sitemaps needed
+  return sitemapModule.sitemapIndexs;
+};
+
+const Sitemap = async ({ id }: { id: SitemapType }): Promise<MetadataRoute.Sitemap> => {
+  switch (id) {
+    case SitemapType.Pages: {
+      return sitemapModule.getPage();
+    }
+    case SitemapType.Assistants: {
+      return sitemapModule.getAssistants();
+    }
+    case SitemapType.Plugins: {
+      return sitemapModule.getPlugins();
+    }
+    case SitemapType.Models: {
+      return sitemapModule.getModels();
+    }
+    case SitemapType.Providers: {
+      return sitemapModule.getProviders();
+    }
+  }
+};
+
+export default Sitemap;

src/const/url.ts

@@ -2,6 +2,7 @@ import qs from 'query-string';
 import urlJoin from 'url-join';
 
 import { withBasePath } from '@/utils/basePath';
+import { isDev } from '@/utils/env';
 
 import pkg from '../../package.json';
 import { INBOX_SESSION_ID } from './session';
@@ -12,8 +13,6 @@ export const OFFICIAL_URL = 'https://lobechat.com/';
 export const OFFICIAL_PREVIEW_URL = 'https://chat-preview.lobehub.com/';
 export const OFFICIAL_SITE = 'https://lobehub.com/';
 
-export const getCanonicalUrl = (path: string) => urlJoin(OFFICIAL_URL, path);
-
 export const OG_URL = '/og/cover.png?v=1';
 
 export const GITHUB = pkg.homepage;
@@ -73,3 +72,4 @@ export const mailTo = (email: string) => `mailto:${email}`;
 
 export const AES_GCM_URL = 'https://datatracker.ietf.org/doc/html/draft-ietf-avt-srtp-aes-gcm-01';
 export const BASE_PROVIDER_DOC_URL = 'https://lobehub.com/docs/usage/providers';
+export const SITEMAP_BASE_URL = isDev ? '/sitemap.xml/' : 'sitemap';

src/server/ld.test.ts

@@ -0,0 +1,102 @@
+// @vitest-environment node
+import { describe, expect, it } from 'vitest';
+
+import { DEFAULT_LANG } from '@/const/locale';
+
+import { AUTHOR_LIST, Ld } from './ld';
+
+describe('Ld', () => {
+  const ld = new Ld();
+
+  describe('generate', () => {
+    it('should generate correct LD+JSON structure', () => {
+      const result = ld.generate({
+        title: 'Test Title',
+        description: 'Test Description',
+        url: 'https://example.com/test',
+        locale: DEFAULT_LANG,
+      });
+
+      expect(result['@context']).toBe('https://schema.org');
+      expect(Array.isArray(result['@graph'])).toBe(true);
+      expect(result['@graph'].length).toBeGreaterThan(0);
+    });
+  });
+
+  describe('genOrganization', () => {
+    it('should generate correct organization structure', () => {
+      const org = ld.genOrganization();
+
+      expect(org['@type']).toBe('Organization');
+      expect(org.name).toBe('LobeHub');
+      expect(org.url).toBe('https://lobehub.com/');
+    });
+  });
+
+  describe('getAuthors', () => {
+    it('should return default author when no ids provided', () => {
+      const author = ld.getAuthors();
+      expect(author['@type']).toBe('Organization');
+    });
+
+    it('should return person when valid id provided', () => {
+      const author = ld.getAuthors(['arvinxx']);
+      expect(author['@type']).toBe('Person');
+      // @ts-ignore
+      expect(author.name).toBe(AUTHOR_LIST.arvinxx.name);
+    });
+  });
+
+  describe('genWebPage', () => {
+    it('should generate correct webpage structure', () => {
+      const webpage = ld.genWebPage({
+        title: 'Test Page',
+        description: 'Test Description',
+        url: 'https://example.com/test',
+        locale: DEFAULT_LANG,
+      });
+
+      expect(webpage['@type']).toBe('WebPage');
+      expect(webpage.name).toBe('Test Page · LobeChat');
+      expect(webpage.description).toBe('Test Description');
+    });
+  });
+
+  describe('genImageObject', () => {
+    it('should generate correct image object', () => {
+      const image = ld.genImageObject({
+        image: 'https://example.com/image.jpg',
+        url: 'https://example.com/test',
+      });
+
+      expect(image['@type']).toBe('ImageObject');
+      expect(image.url).toBe('https://example.com/image.jpg');
+    });
+  });
+
+  describe('genWebSite', () => {
+    it('should generate correct website structure', () => {
+      const website = ld.genWebSite();
+
+      expect(website['@type']).toBe('WebSite');
+      expect(website.name).toBe('LobeChat');
+    });
+  });
+
+  describe('genArticle', () => {
+    it('should generate correct article structure', () => {
+      const article = ld.genArticle({
+        title: 'Test Article',
+        description: 'Test Description',
+        url: 'https://example.com/test',
+        author: ['arvinxx'],
+        identifier: 'test-id',
+        locale: DEFAULT_LANG,
+      });
+
+      expect(article['@type']).toBe('Article');
+      expect(article.headline).toBe('Test Article · LobeChat');
+      expect(article.author['@type']).toBe('Person');
+    });
+  });
+});

src/server/ld.ts

@@ -3,15 +3,9 @@ import urlJoin from 'url-join';
 
 import { BRANDING_NAME } from '@/const/branding';
 import { DEFAULT_LANG } from '@/const/locale';
-import {
-  EMAIL_BUSINESS,
-  EMAIL_SUPPORT,
-  OFFICIAL_SITE,
-  OFFICIAL_URL,
-  X,
-  getCanonicalUrl,
-} from '@/const/url';
+import { EMAIL_BUSINESS, EMAIL_SUPPORT, OFFICIAL_SITE, OFFICIAL_URL, X } from '@/const/url';
 import { Locales } from '@/locales/resources';
+import { getCanonicalUrl } from '@/server/utils/url';
 
 import pkg from '../../package.json';
 
@@ -37,7 +31,7 @@ export const AUTHOR_LIST = {
   },
 };
 
-class Ld {
+export class Ld {
   generate({
     image = '/og/cover.png',
     article,

src/server/metadata.test.ts

@@ -0,0 +1,138 @@
+// @vitest-environment node
+import { describe, expect, it } from 'vitest';
+
+import { BRANDING_NAME } from '@/const/branding';
+import { OG_URL } from '@/const/url';
+
+import { Meta } from './metadata';
+
+describe('Metadata', () => {
+  const meta = new Meta();
+
+  describe('generate', () => {
+    it('should generate metadata with default values', () => {
+      const result = meta.generate({
+        title: 'Test Title',
+        url: 'https://example.com',
+      });
+
+      expect(result).toMatchObject({
+        title: 'Test Title',
+        description: expect.any(String),
+        openGraph: expect.objectContaining({
+          title: `Test Title · ${BRANDING_NAME}`,
+          description: expect.any(String),
+          images: [{ url: OG_URL, alt: `Test Title · ${BRANDING_NAME}` }],
+        }),
+        twitter: expect.objectContaining({
+          title: `Test Title · ${BRANDING_NAME}`,
+          description: expect.any(String),
+          images: [OG_URL],
+        }),
+      });
+    });
+
+    it('should generate metadata with custom values', () => {
+      const result = meta.generate({
+        title: 'Custom Title',
+        description: 'Custom description',
+        image: 'https://custom-image.com',
+        url: 'https://example.com/custom',
+        type: 'article',
+        tags: ['tag1', 'tag2'],
+        locale: 'fr-FR',
+        alternate: true,
+      });
+
+      expect(result).toMatchObject({
+        title: 'Custom Title',
+        description: expect.stringContaining('Custom description'),
+        openGraph: expect.objectContaining({
+          title: `Custom Title · ${BRANDING_NAME}`,
+          description: 'Custom description',
+          images: [{ url: 'https://custom-image.com', alt: `Custom Title · ${BRANDING_NAME}` }],
+          type: 'article',
+          locale: 'fr-FR',
+        }),
+        twitter: expect.objectContaining({
+          title: `Custom Title · ${BRANDING_NAME}`,
+          description: 'Custom description',
+          images: ['https://custom-image.com'],
+        }),
+        alternates: expect.objectContaining({
+          languages: expect.any(Object),
+        }),
+      });
+    });
+  });
+
+  describe('genAlternateLocales', () => {
+    it('should generate alternate locales correctly', () => {
+      const result = (meta as any).genAlternateLocales('en', '/test');
+
+      expect(result).toHaveProperty('x-default', expect.stringContaining('/test'));
+      expect(result).toHaveProperty('zh-CN', expect.stringContaining('hl=zh-CN'));
+      expect(result).not.toHaveProperty('en');
+    });
+  });
+
+  describe('genTwitter', () => {
+    it('should generate Twitter metadata correctly', () => {
+      const result = (meta as any).genTwitter({
+        title: 'Twitter Title',
+        description: 'Twitter description',
+        image: 'https://twitter-image.com',
+        url: 'https://example.com/twitter',
+      });
+
+      expect(result).toEqual({
+        card: 'summary_large_image',
+        title: 'Twitter Title',
+        description: 'Twitter description',
+        images: ['https://twitter-image.com'],
+        site: '@lobehub',
+        url: 'https://example.com/twitter',
+      });
+    });
+  });
+
+  describe('genOpenGraph', () => {
+    it('should generate OpenGraph metadata correctly', () => {
+      const result = (meta as any).genOpenGraph({
+        title: 'OG Title',
+        description: 'OG description',
+        image: 'https://og-image.com',
+        url: 'https://example.com/og',
+        locale: 'es-ES',
+        type: 'article',
+        alternate: true,
+      });
+
+      expect(result).toMatchObject({
+        title: 'OG Title',
+        description: 'OG description',
+        images: [{ url: 'https://og-image.com', alt: 'OG Title' }],
+        locale: 'es-ES',
+        type: 'article',
+        url: 'https://example.com/og',
+        siteName: 'LobeChat',
+        alternateLocale: expect.arrayContaining([
+          'ar',
+          'bg-BG',
+          'de-DE',
+          'en-US',
+          'es-ES',
+          'fr-FR',
+          'ja-JP',
+          'ko-KR',
+          'pt-BR',
+          'ru-RU',
+          'tr-TR',
+          'zh-CN',
+          'zh-TW',
+          'vi-VN',
+        ]),
+      });
+    });
+  });
+});

src/server/metadata.ts

@@ -3,8 +3,9 @@ import qs from 'query-string';
 
 import { BRANDING_NAME } from '@/const/branding';
 import { DEFAULT_LANG } from '@/const/locale';
-import { OG_URL, getCanonicalUrl } from '@/const/url';
+import { OG_URL } from '@/const/url';
 import { Locales, locales } from '@/locales/resources';
+import { getCanonicalUrl } from '@/server/utils/url';
 import { formatDescLength, formatTitleLength } from '@/utils/genOG';
 
 export class Meta {
@@ -59,7 +60,6 @@ export class Meta {
     let links: any = {};
     const defaultLink = getCanonicalUrl(path);
     for (const alterLocales of locales) {
-      if (locale === alterLocales) continue;
       links[alterLocales] = qs.stringifyUrl({
         query: { hl: alterLocales },
         url: defaultLink,
@@ -125,7 +125,7 @@ export class Meta {
     };
 
     if (alternate) {
-      data['alternateLocale'] = locales.filter((l) => l !== locale);
+      data['alternateLocale'] = locales;
     }
 
     return data;

src/server/modules/AssistantStore/index.test.ts

@@ -13,7 +13,7 @@ describe('AssistantStore', () => {
 
   it('should return the index URL for a not supported language', () => {
     const agentMarket = new AssistantStore();
-    const url = agentMarket.getAgentIndexUrl('ko-KR');
+    const url = agentMarket.getAgentIndexUrl('xxx' as any);
     expect(url).toBe('https://chat-agents.lobehub.com');
   });
 

src/server/modules/AssistantStore/index.ts

@@ -4,10 +4,6 @@ import { appEnv } from '@/config/app';
 import { DEFAULT_LANG, isLocaleNotSupport } from '@/const/locale';
 import { Locales, normalizeLocale } from '@/locales/resources';
 
-const checkSupportLocale = (lang: Locales) => {
-  return isLocaleNotSupport(lang) || normalizeLocale(lang) !== 'zh-CN';
-};
-
 export class AssistantStore {
   private readonly baseUrl: string;
 
@@ -16,13 +12,13 @@ export class AssistantStore {
   }
 
   getAgentIndexUrl = (lang: Locales = DEFAULT_LANG) => {
-    if (checkSupportLocale(lang)) return this.baseUrl;
+    if (isLocaleNotSupport(lang)) return this.baseUrl;
 
     return urlJoin(this.baseUrl, `index.${normalizeLocale(lang)}.json`);
   };
 
   getAgentUrl = (identifier: string, lang: Locales = DEFAULT_LANG) => {
-    if (checkSupportLocale(lang)) return urlJoin(this.baseUrl, `${identifier}.json`);
+    if (isLocaleNotSupport(lang)) return urlJoin(this.baseUrl, `${identifier}.json`);
 
     return urlJoin(this.baseUrl, `${identifier}.${normalizeLocale(lang)}.json`);
   };

src/server/sitemap.test.ts

@@ -0,0 +1,179 @@
+// @vitest-environment node
+import { describe, expect, it, vi } from 'vitest';
+
+import { getCanonicalUrl } from '@/server/utils/url';
+import { AssistantCategory, PluginCategory } from '@/types/discover';
+
+import { LAST_MODIFIED, Sitemap, SitemapType } from './sitemap';
+
+describe('Sitemap', () => {
+  const sitemap = new Sitemap();
+
+  describe('getIndex', () => {
+    it('should return a valid sitemap index', () => {
+      const index = sitemap.getIndex();
+      expect(index).toContain('<?xml version="1.0" encoding="UTF-8"?>');
+      expect(index).toContain('<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">');
+      [
+        SitemapType.Pages,
+        SitemapType.Assistants,
+        SitemapType.Plugins,
+        SitemapType.Models,
+        SitemapType.Providers,
+      ].forEach((type) => {
+        expect(index).toContain(`<loc>${getCanonicalUrl(`/sitemap/${type}.xml`)}</loc>`);
+      });
+      expect(index).toContain(`<lastmod>${LAST_MODIFIED}</lastmod>`);
+    });
+  });
+
+  describe('getPage', () => {
+    it('should return a valid page sitemap', async () => {
+      const pageSitemap = await sitemap.getPage();
+      expect(pageSitemap).toContainEqual(
+        expect.objectContaining({
+          url: getCanonicalUrl('/'),
+          changeFrequency: 'monthly',
+          priority: 0.4,
+        }),
+      );
+      expect(pageSitemap).toContainEqual(
+        expect.objectContaining({
+          url: getCanonicalUrl('/discover'),
+          changeFrequency: 'daily',
+          priority: 0.7,
+        }),
+      );
+      Object.values(AssistantCategory).forEach((category) => {
+        expect(pageSitemap).toContainEqual(
+          expect.objectContaining({
+            url: getCanonicalUrl(`/discover/assistants/${category}`),
+            changeFrequency: 'daily',
+            priority: 0.7,
+          }),
+        );
+      });
+      Object.values(PluginCategory).forEach((category) => {
+        expect(pageSitemap).toContainEqual(
+          expect.objectContaining({
+            url: getCanonicalUrl(`/discover/plugins/${category}`),
+            changeFrequency: 'daily',
+            priority: 0.7,
+          }),
+        );
+      });
+    });
+  });
+
+  describe('getAssistants', () => {
+    it('should return a valid assistants sitemap', async () => {
+      vi.spyOn(sitemap['discoverService'], 'getAssistantList').mockResolvedValue([
+        // @ts-ignore
+        { identifier: 'test-assistant', createdAt: '2023-01-01' },
+      ]);
+
+      const assistantsSitemap = await sitemap.getAssistants();
+      expect(assistantsSitemap.length).toBe(14);
+      expect(assistantsSitemap).toContainEqual(
+        expect.objectContaining({
+          url: getCanonicalUrl('/discover/assistant/test-assistant'),
+          lastModified: '2023-01-01T00:00:00.000Z',
+        }),
+      );
+      expect(assistantsSitemap).toContainEqual(
+        expect.objectContaining({
+          url: getCanonicalUrl('/discover/assistant/test-assistant?hl=zh-CN'),
+          lastModified: '2023-01-01T00:00:00.000Z',
+        }),
+      );
+    });
+  });
+
+  describe('getPlugins', () => {
+    it('should return a valid plugins sitemap', async () => {
+      vi.spyOn(sitemap['discoverService'], 'getPluginList').mockResolvedValue([
+        // @ts-ignore
+        { identifier: 'test-plugin', createdAt: '2023-01-01' },
+      ]);
+
+      const pluginsSitemap = await sitemap.getPlugins();
+      expect(pluginsSitemap.length).toBe(14);
+      expect(pluginsSitemap).toContainEqual(
+        expect.objectContaining({
+          url: getCanonicalUrl('/discover/plugin/test-plugin'),
+          lastModified: '2023-01-01T00:00:00.000Z',
+        }),
+      );
+      expect(pluginsSitemap).toContainEqual(
+        expect.objectContaining({
+          url: getCanonicalUrl('/discover/plugin/test-plugin?hl=ja-JP'),
+          lastModified: '2023-01-01T00:00:00.000Z',
+        }),
+      );
+    });
+  });
+
+  describe('getModels', () => {
+    it('should return a valid models sitemap', async () => {
+      vi.spyOn(sitemap['discoverService'], 'getModelList').mockResolvedValue([
+        // @ts-ignore
+        { identifier: 'test:model', createdAt: '2023-01-01' },
+      ]);
+
+      const modelsSitemap = await sitemap.getModels();
+      expect(modelsSitemap.length).toBe(14);
+      expect(modelsSitemap).toContainEqual(
+        expect.objectContaining({
+          url: getCanonicalUrl('/discover/model/test:model'),
+          lastModified: '2023-01-01T00:00:00.000Z',
+        }),
+      );
+      expect(modelsSitemap).toContainEqual(
+        expect.objectContaining({
+          url: getCanonicalUrl('/discover/model/test:model?hl=ko-KR'),
+          lastModified: '2023-01-01T00:00:00.000Z',
+        }),
+      );
+    });
+  });
+
+  describe('getProviders', () => {
+    it('should return a valid providers sitemap', async () => {
+      vi.spyOn(sitemap['discoverService'], 'getProviderList').mockResolvedValue([
+        // @ts-ignore
+        { identifier: 'test-provider', createdAt: '2023-01-01' },
+      ]);
+
+      const providersSitemap = await sitemap.getProviders();
+      expect(providersSitemap.length).toBe(14);
+      expect(providersSitemap).toContainEqual(
+        expect.objectContaining({
+          url: getCanonicalUrl('/discover/provider/test-provider'),
+          lastModified: '2023-01-01T00:00:00.000Z',
+        }),
+      );
+      expect(providersSitemap).toContainEqual(
+        expect.objectContaining({
+          url: getCanonicalUrl('/discover/provider/test-provider?hl=ar'),
+          lastModified: '2023-01-01T00:00:00.000Z',
+        }),
+      );
+    });
+  });
+
+  describe('getRobots', () => {
+    it('should return correct robots.txt entries', () => {
+      const robots = sitemap.getRobots();
+      expect(robots).toContain(getCanonicalUrl('/sitemap-index.xml'));
+      [
+        SitemapType.Pages,
+        SitemapType.Assistants,
+        SitemapType.Plugins,
+        SitemapType.Models,
+        SitemapType.Providers,
+      ].forEach((type) => {
+        expect(robots).toContain(getCanonicalUrl(`/sitemap/${type}.xml`));
+      });
+    });
+  });
+});

src/server/sitemap.ts

@@ -0,0 +1,243 @@
+import { flatten } from 'lodash-es';
+import { MetadataRoute } from 'next';
+import qs from 'query-string';
+import urlJoin from 'url-join';
+
+import { DEFAULT_LANG } from '@/const/locale';
+import { SITEMAP_BASE_URL } from '@/const/url';
+import { Locales, locales as allLocales } from '@/locales/resources';
+import { DiscoverService } from '@/server/services/discover';
+import { getCanonicalUrl } from '@/server/utils/url';
+import { AssistantCategory, PluginCategory } from '@/types/discover';
+import { isDev } from '@/utils/env';
+
+export interface SitemapItem {
+  alternates?: {
+    languages?: string;
+  };
+  changeFrequency?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
+  lastModified?: string | Date;
+  priority?: number;
+  url: string;
+}
+
+export enum SitemapType {
+  Assistants = 'assistants',
+  Models = 'models',
+  Pages = 'pages',
+  Plugins = 'plugins',
+  Providers = 'providers',
+}
+
+export const LAST_MODIFIED = new Date().toISOString();
+
+export class Sitemap {
+  sitemapIndexs = [
+    { id: SitemapType.Pages },
+    { id: SitemapType.Assistants },
+    { id: SitemapType.Plugins },
+    { id: SitemapType.Models },
+    { id: SitemapType.Providers },
+  ];
+
+  private discoverService = new DiscoverService();
+
+  private _generateSitemapLink(url: string) {
+    return [
+      '<sitemap>',
+      `<loc>${url}</loc>`,
+      `<lastmod>${LAST_MODIFIED}</lastmod>`,
+      '</sitemap>',
+    ].join('\n');
+  }
+
+  private _formatTime(time?: string) {
+    try {
+      if (!time) return LAST_MODIFIED;
+      return new Date(time).toISOString() || LAST_MODIFIED;
+    } catch {
+      return LAST_MODIFIED;
+    }
+  }
+
+  private _genSitemapItem = (
+    lang: Locales,
+    url: string,
+    {
+      lastModified,
+      changeFrequency = 'monthly',
+      priority = 0.4,
+      noLocales,
+      locales = allLocales,
+    }: {
+      changeFrequency?: SitemapItem['changeFrequency'];
+      lastModified?: string;
+      locales?: typeof allLocales;
+      noLocales?: boolean;
+      priority?: number;
+    } = {},
+  ) => {
+    const sitemap = {
+      changeFrequency,
+      lastModified: this._formatTime(lastModified),
+      priority,
+      url:
+        lang === DEFAULT_LANG
+          ? getCanonicalUrl(url)
+          : qs.stringifyUrl({ query: { hl: lang }, url: getCanonicalUrl(url) }),
+    };
+    if (noLocales) return sitemap;
+
+    const languages: any = {};
+    for (const locale of locales) {
+      if (locale === lang) continue;
+      languages[locale] = qs.stringifyUrl({
+        query: { hl: locale },
+        url: getCanonicalUrl(url),
+      });
+    }
+    return {
+      alternates: {
+        languages,
+      },
+      ...sitemap,
+    };
+  };
+
+  private _genSitemap(
+    url: string,
+    {
+      lastModified,
+      changeFrequency = 'monthly',
+      priority = 0.4,
+      noLocales,
+      locales = allLocales,
+    }: {
+      changeFrequency?: SitemapItem['changeFrequency'];
+      lastModified?: string;
+      locales?: typeof allLocales;
+      noLocales?: boolean;
+      priority?: number;
+    } = {},
+  ) {
+    if (noLocales)
+      return [
+        this._genSitemapItem(DEFAULT_LANG, url, {
+          changeFrequency,
+          lastModified,
+          locales,
+          noLocales,
+          priority,
+        }),
+      ];
+    return locales.map((lang) =>
+      this._genSitemapItem(lang, url, {
+        changeFrequency,
+        lastModified,
+        locales,
+        noLocales,
+        priority,
+      }),
+    );
+  }
+
+  getIndex(): string {
+    return [
+      '<?xml version="1.0" encoding="UTF-8"?>',
+      '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+      ...this.sitemapIndexs.map((item) =>
+        this._generateSitemapLink(
+          getCanonicalUrl(SITEMAP_BASE_URL, isDev ? item.id : `${item.id}.xml`),
+        ),
+      ),
+      '</sitemapindex>',
+    ].join('\n');
+  }
+
+  async getAssistants(): Promise<MetadataRoute.Sitemap> {
+    const list = await this.discoverService.getAssistantList(DEFAULT_LANG);
+    const sitmap = list.map((item) =>
+      this._genSitemap(urlJoin('/discover/assistant', item.identifier), {
+        lastModified: item?.createdAt || LAST_MODIFIED,
+      }),
+    );
+    return flatten(sitmap);
+  }
+
+  async getPlugins(): Promise<MetadataRoute.Sitemap> {
+    const list = await this.discoverService.getPluginList(DEFAULT_LANG);
+    const sitmap = list.map((item) =>
+      this._genSitemap(urlJoin('/discover/plugin', item.identifier), {
+        lastModified: item?.createdAt || LAST_MODIFIED,
+      }),
+    );
+    return flatten(sitmap);
+  }
+
+  async getModels(): Promise<MetadataRoute.Sitemap> {
+    const list = await this.discoverService.getModelList(DEFAULT_LANG);
+    const sitmap = list.map((item) =>
+      this._genSitemap(urlJoin('/discover/model', item.identifier), {
+        lastModified: item?.createdAt || LAST_MODIFIED,
+      }),
+    );
+    return flatten(sitmap);
+  }
+
+  async getProviders(): Promise<MetadataRoute.Sitemap> {
+    const list = await this.discoverService.getProviderList(DEFAULT_LANG);
+    const sitmap = list.map((item) =>
+      this._genSitemap(urlJoin('/discover/provider', item.identifier), {
+        lastModified: item?.createdAt || LAST_MODIFIED,
+      }),
+    );
+    return flatten(sitmap);
+  }
+
+  async getPage(): Promise<MetadataRoute.Sitemap> {
+    const assistantsCategory = Object.values(AssistantCategory);
+    const pluginCategory = Object.values(PluginCategory);
+    const modelCategory = await this.discoverService.getProviderList(DEFAULT_LANG);
+    return [
+      ...this._genSitemap('/', { noLocales: true }),
+      ...this._genSitemap('/chat', { noLocales: true }),
+      ...this._genSitemap('/welcome', { noLocales: true }),
+      /* ↓ cloud slot ↓ */
+
+      /* ↑ cloud slot ↑ */
+      ...this._genSitemap('/discover', { changeFrequency: 'daily', priority: 0.7 }),
+      ...this._genSitemap('/discover/assistants', { changeFrequency: 'daily', priority: 0.7 }),
+      ...assistantsCategory.flatMap((slug) =>
+        this._genSitemap(`/discover/assistants/${slug}`, {
+          changeFrequency: 'daily',
+          priority: 0.7,
+        }),
+      ),
+      ...this._genSitemap('/discover/plugins', { changeFrequency: 'daily', priority: 0.7 }),
+      ...pluginCategory.flatMap((slug) =>
+        this._genSitemap(`/discover/plugins/${slug}`, {
+          changeFrequency: 'daily',
+          priority: 0.7,
+        }),
+      ),
+      ...this._genSitemap('/discover/models', { changeFrequency: 'daily', priority: 0.7 }),
+      ...modelCategory.flatMap((slug) =>
+        this._genSitemap(`/discover/models/${slug}`, {
+          changeFrequency: 'daily',
+          priority: 0.7,
+        }),
+      ),
+      ...this._genSitemap('/discover/providers', { changeFrequency: 'daily', priority: 0.7 }),
+    ];
+  }
+  getRobots() {
+    return [
+      getCanonicalUrl('/sitemap-index.xml'),
+      ...this.sitemapIndexs.map((index) =>
+        getCanonicalUrl(SITEMAP_BASE_URL, isDev ? index.id : `${index.id}.xml`),
+      ),
+    ];
+  }
+}
+
+export const sitemapModule = new Sitemap();

src/server/translation.test.ts

@@ -0,0 +1,137 @@
+// @vitest-environment node
+import { cookies } from 'next/headers';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { DEFAULT_LANG, LOBE_LOCALE_COOKIE } from '@/const/locale';
+import { normalizeLocale } from '@/locales/resources';
+import * as env from '@/utils/env';
+
+import { getLocale, translation } from './translation';
+
+// Mock external dependencies
+vi.mock('next/headers', () => ({
+  cookies: vi.fn(),
+}));
+
+vi.mock('node:fs', () => ({
+  existsSync: vi.fn(),
+  readFileSync: vi.fn(),
+}));
+
+vi.mock('node:path', () => ({
+  join: vi.fn(),
+}));
+
+vi.mock('@/const/locale', () => ({
+  DEFAULT_LANG: 'en-US',
+  LOBE_LOCALE_COOKIE: 'LOBE_LOCALE',
+}));
+
+vi.mock('@/locales/resources', () => ({
+  normalizeLocale: vi.fn((locale) => locale),
+}));
+
+vi.mock('@/utils/env', () => ({
+  isDev: false,
+}));
+
+describe('getLocale', () => {
+  const mockCookieStore = {
+    get: vi.fn(),
+  };
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    (cookies as any).mockReturnValue(mockCookieStore);
+  });
+
+  it('should return the provided locale if hl is specified', async () => {
+    const result = await getLocale('fr-FR');
+    expect(result).toBe('fr-FR');
+    expect(normalizeLocale).toHaveBeenCalledWith('fr-FR');
+  });
+
+  it('should return the locale from cookie if available', async () => {
+    mockCookieStore.get.mockReturnValue({ value: 'de-DE' });
+    const result = await getLocale();
+    expect(result).toBe('de-DE');
+    expect(mockCookieStore.get).toHaveBeenCalledWith(LOBE_LOCALE_COOKIE);
+  });
+
+  it('should return DEFAULT_LANG if no cookie is set', async () => {
+    mockCookieStore.get.mockReturnValue(undefined);
+    const result = await getLocale();
+    expect(result).toBe(DEFAULT_LANG);
+  });
+});
+
+describe('translation', () => {
+  const mockTranslations = {
+    key1: 'Value 1',
+    key2: 'Value 2 with {{param}}',
+    nested: { key: 'Nested value' },
+  };
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    (fs.existsSync as any).mockReturnValue(true);
+    (fs.readFileSync as any).mockReturnValue(JSON.stringify(mockTranslations));
+    (path.join as any).mockImplementation((...args: any) => args.join('/'));
+  });
+
+  it('should return correct translation object', async () => {
+    const result = await translation('common', 'en-US');
+    expect(result).toHaveProperty('locale', 'en-US');
+    expect(result).toHaveProperty('t');
+    expect(typeof result.t).toBe('function');
+  });
+
+  it('should translate keys correctly', async () => {
+    const { t } = await translation('common', 'en-US');
+    expect(t('key1')).toBe('Value 1');
+    expect(t('key2', { param: 'test' })).toBe('Value 2 with test');
+    expect(t('nested.key')).toBe('Nested value');
+  });
+
+  it('should return key if translation is not found', async () => {
+    const { t } = await translation('common', 'en-US');
+    expect(t('nonexistent.key')).toBe('nonexistent.key');
+  });
+
+  it('should use fallback language if specified locale file does not exist', async () => {
+    (fs.existsSync as any).mockReturnValueOnce(false);
+    await translation('common', 'nonexistent-LANG');
+    expect(fs.readFileSync).toHaveBeenCalledWith(
+      expect.stringContaining(`/${DEFAULT_LANG}/common.json`),
+      'utf8',
+    );
+  });
+
+  it('should use zh-CN in dev mode when fallback is needed', async () => {
+    (fs.existsSync as any).mockReturnValueOnce(false);
+    (env.isDev as unknown as boolean) = true;
+    await translation('common', 'nonexistent-LANG');
+    expect(fs.readFileSync).toHaveBeenCalledWith(
+      expect.stringContaining('/zh-CN/common.json'),
+      'utf8',
+    );
+  });
+
+  it('should handle file reading errors', async () => {
+    const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+    (fs.readFileSync as any).mockImplementation(() => {
+      throw new Error('File read error');
+    });
+
+    const result = await translation('common', 'en-US');
+    expect(result.t('any.key')).toBe('any.key');
+    expect(consoleErrorSpy).toHaveBeenCalledWith(
+      'Error while reading translation file',
+      expect.any(Error),
+    );
+
+    consoleErrorSpy.mockRestore();
+  });
+});

src/server/utils/url.test.ts

@@ -0,0 +1,61 @@
+// @vitest-environment node
+import urlJoin from 'url-join';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+// 模拟 urlJoin 函数
+vi.mock('url-join', () => ({
+  default: vi.fn((...args) => args.join('/')),
+}));
+
+describe('getCanonicalUrl', () => {
+  const originalEnv = process.env;
+
+  beforeEach(() => {
+    // 在每个测试前重置 process.env
+    vi.resetModules();
+    process.env = { ...originalEnv };
+  });
+
+  afterEach(() => {
+    // 在每个测试后恢复原始的 process.env
+    process.env = originalEnv;
+  });
+
+  it('should return correct URL for production environment', async () => {
+    process.env.VERCEL = undefined;
+    process.env.VERCEL_ENV = undefined;
+
+    const { getCanonicalUrl } = await import('./url'); // 动态导入以获取最新的环境变量状态
+    const result = getCanonicalUrl('path', 'to', 'page');
+    expect(result).toBe('https://lobechat.com/path/to/page');
+    expect(urlJoin).toHaveBeenCalledWith('https://lobechat.com', 'path', 'to', 'page');
+  });
+
+  it('should return correct URL for Vercel preview environment', async () => {
+    process.env.VERCEL = '1';
+    process.env.VERCEL_ENV = 'preview';
+    process.env.VERCEL_URL = 'preview-url.vercel.app';
+
+    const { getCanonicalUrl } = await import('./url'); // 动态导入
+    const result = getCanonicalUrl('path', 'to', 'page');
+    expect(result).toBe('https://preview-url.vercel.app/path/to/page');
+    expect(urlJoin).toHaveBeenCalledWith('https://preview-url.vercel.app', 'path', 'to', 'page');
+  });
+
+  it('should return production URL when VERCEL is set but VERCEL_ENV is production', async () => {
+    process.env.VERCEL = '1';
+    process.env.VERCEL_ENV = 'production';
+
+    const { getCanonicalUrl } = await import('./url'); // 动态导入
+    const result = getCanonicalUrl('path', 'to', 'page');
+    expect(result).toBe('https://lobechat.com/path/to/page');
+    expect(urlJoin).toHaveBeenCalledWith('https://lobechat.com', 'path', 'to', 'page');
+  });
+
+  it('should work correctly without additional path arguments', async () => {
+    const { getCanonicalUrl } = await import('./url'); // 动态导入
+    const result = getCanonicalUrl();
+    expect(result).toBe('https://lobechat.com');
+    expect(urlJoin).toHaveBeenCalledWith('https://lobechat.com');
+  });
+});

src/server/utils/url.ts

@@ -0,0 +1,9 @@
+import urlJoin from 'url-join';
+
+const isVercelPreview = process.env.VERCEL === '1' && process.env.VERCEL_ENV !== 'production';
+
+const vercelPreviewUrl = `https://${process.env.VERCEL_URL}`;
+
+const siteUrl = isVercelPreview ? vercelPreviewUrl : 'https://lobechat.com';
+
+export const getCanonicalUrl = (...paths: string[]) => urlJoin(siteUrl, ...paths);