💄 style: add imagen model to vertex ai (#9699)

* ♻️ refactor: rename isLocalUrl to isDesktopLocalStaticServerUrl Rename the function to better reflect its specific purpose of checking desktop local static server URLs (127.0.0.1 only). Update all usages across the codebase including imports, function calls, and test cases. * ✨ feat(model-bank): add Vertex AI image generation models - Add Nano Banana (Gemini 2.5 Flash Image) models - Add Imagen 4 series (Standard, Ultra, Fast, Preview variants) - Export shared parameters for reuse across providers * ✅ test(context-engine): fix mock after function rename Update test mock from isLocalUrl to isDesktopLocalStaticServerUrl * ♻️ refactor: use submodule imports for @lobechat/utils - Change from barrel imports to direct submodule imports - Update test to mock only necessary functions (imageUrlToBase64) - Fix test URL from localhost to 127.0.0.1 for isDesktopLocalStaticServerUrl - Update package.json exports for utils submodules * ✅ test: update mocks after function rename Update test mocks from isLocalUrl to isDesktopLocalStaticServerUrl * ✅ test(chat): fix mocks to use submodule imports
2026-03-27 13:29:15 +07:00 · 2025-10-14 18:15:58 +08:00
parent c07d900648
commit 3b2a2c1c54
9 changed files with 154 additions and 76 deletions
--- a/packages/context-engine/src/processors/MessageContent.ts
+++ b/packages/context-engine/src/processors/MessageContent.ts
@@ -1,5 +1,7 @@
 import { filesPrompts } from '@lobechat/prompts';
-import { imageUrlToBase64, isLocalUrl, parseDataUri } from '@lobechat/utils';
+import { imageUrlToBase64 } from '@lobechat/utils/imageToBase64';
+import { parseDataUri } from '@lobechat/utils/uriParser';
+import { isDesktopLocalStaticServerUrl } from '@lobechat/utils/url';
 import debug from 'debug';

 import { BaseProcessor } from '../base/BaseProcessor';
@@ -277,7 +279,7 @@ export class MessageContentProcessor extends BaseProcessor {
        const { type } = parseDataUri(image.url);

        let processedUrl = image.url;
-        if (type === 'url' && isLocalUrl(image.url)) {
+        if (type === 'url' && isDesktopLocalStaticServerUrl(image.url)) {
          const { base64, mimeType } = await imageUrlToBase64(image.url);
          processedUrl = `data:${mimeType};base64,${base64}`;
        }
--- a/packages/context-engine/src/processors/tests/MessageContent.test.ts
+++ b/packages/context-engine/src/processors/tests/MessageContent.test.ts
@@ -4,19 +4,16 @@ import { describe, expect, it, vi } from 'vitest';
 import type { PipelineContext } from '../../types';
 import { MessageContentProcessor } from '../MessageContent';

-vi.mock('@lobechat/utils', () => ({
-  imageUrlToBase64: vi.fn().mockResolvedValue({
-    base64: 'base64-data',
-    mimeType: 'image/png',
-  }),
-  isLocalUrl: vi.fn((url: string) => url.includes('localhost') || url.includes('127.0.0.1')),
-  parseDataUri: vi.fn((url: string) => {
-    if (url.startsWith('data:')) {
-      return { type: 'data' };
-    }
-    return { type: 'url' };
-  }),
-}));
+vi.mock('@lobechat/utils/imageToBase64', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('@lobechat/utils/imageToBase64')>();
+  return {
+    ...actual,
+    imageUrlToBase64: vi.fn().mockResolvedValue({
+      base64: 'base64-data',
+      mimeType: 'image/png',
+    }),
+  };
+});

 const createContext = (messages: ChatMessage[]): PipelineContext => ({
  initialState: { messages: [] } as any,
@@ -138,7 +135,7 @@ describe('MessageContentProcessor', () => {
          role: 'user',
          content: 'Hello',
          imageList: [
-            { url: 'http://localhost:3000/image.jpg', alt: '', id: 'test' } as ChatImageItem,
+            { url: 'http://127.0.0.1:3000/image.jpg', alt: '', id: 'test' } as ChatImageItem,
          ],
          createdAt: Date.now(),
          updatedAt: Date.now(),
--- a/packages/model-bank/src/aiModels/google.ts
+++ b/packages/model-bank/src/aiModels/google.ts
@@ -820,7 +820,7 @@ const googleChatModels: AIChatModelCard[] = [
 ];

 // Common parameters for Imagen models
-const imagenBaseParameters: ModelParamsSchema = {
+export const imagenGenParameters: ModelParamsSchema = {
  aspectRatio: {
    default: '1:1',
    enum: ['1:1', '16:9', '9:16', '3:4', '4:3'],
@@ -841,7 +841,7 @@ const NANO_BANANA_ASPECT_RATIOS = [
  '21:9', // 1536x672
 ];

-const nanoBananaParameters: ModelParamsSchema = {
+export const nanoBananaParameters: ModelParamsSchema = {
  aspectRatio: {
    default: '1:1',
    enum: NANO_BANANA_ASPECT_RATIOS,
@@ -895,7 +895,7 @@ const googleImageModels: AIImageModelCard[] = [
    description: 'Imagen 4th generation text-to-image model series',
    organization: 'Deepmind',
    releasedAt: '2025-08-15',
-    parameters: imagenBaseParameters,
+    parameters: imagenGenParameters,
    pricing: {
      units: [{ name: 'imageGeneration', rate: 0.04, strategy: 'fixed', unit: 'image' }],
    },
@@ -908,7 +908,7 @@ const googleImageModels: AIImageModelCard[] = [
    description: 'Imagen 4th generation text-to-image model series Ultra version',
    organization: 'Deepmind',
    releasedAt: '2025-08-15',
-    parameters: imagenBaseParameters,
+    parameters: imagenGenParameters,
    pricing: {
      units: [{ name: 'imageGeneration', rate: 0.06, strategy: 'fixed', unit: 'image' }],
    },
@@ -921,7 +921,7 @@ const googleImageModels: AIImageModelCard[] = [
    description: 'Imagen 4th generation text-to-image model series Fast version',
    organization: 'Deepmind',
    releasedAt: '2025-08-15',
-    parameters: imagenBaseParameters,
+    parameters: imagenGenParameters,
    pricing: {
      units: [{ name: 'imageGeneration', rate: 0.02, strategy: 'fixed', unit: 'image' }],
    },
@@ -933,7 +933,7 @@ const googleImageModels: AIImageModelCard[] = [
    description: 'Imagen 4th generation text-to-image model series',
    organization: 'Deepmind',
    releasedAt: '2024-06-06',
-    parameters: imagenBaseParameters,
+    parameters: imagenGenParameters,
    pricing: {
      units: [{ name: 'imageGeneration', rate: 0.04, strategy: 'fixed', unit: 'image' }],
    },
@@ -945,7 +945,7 @@ const googleImageModels: AIImageModelCard[] = [
    description: 'Imagen 4th generation text-to-image model series Ultra version',
    organization: 'Deepmind',
    releasedAt: '2025-06-11',
-    parameters: imagenBaseParameters,
+    parameters: imagenGenParameters,
    pricing: {
      units: [{ name: 'imageGeneration', rate: 0.06, strategy: 'fixed', unit: 'image' }],
    },
--- a/packages/model-bank/src/aiModels/vertexai.ts
+++ b/packages/model-bank/src/aiModels/vertexai.ts
@@ -1,4 +1,5 @@
-import { AIChatModelCard } from '../types/aiModel';
+import { AIChatModelCard, AIImageModelCard } from '../types/aiModel';
+import { imagenGenParameters, nanoBananaParameters } from './google';

 // ref: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models
 const vertexaiChatModels: AIChatModelCard[] = [
@@ -278,6 +279,66 @@ const vertexaiChatModels: AIChatModelCard[] = [
  },
 ];

-export const allModels = [...vertexaiChatModels];
+/* eslint-disable sort-keys-fix/sort-keys-fix */
+const vertexaiImageModels: AIImageModelCard[] = [
+  {
+    displayName: 'Nano Banana',
+    id: 'gemini-2.5-flash-image:image',
+    enabled: true,
+    type: 'image',
+    description:
+      'Nano Banana 是 Google 最新、最快、最高效的原生多模态模型，它允许您通过对话生成和编辑图像。',
+    releasedAt: '2025-08-26',
+    parameters: nanoBananaParameters,
+    pricing: {
+      units: [
+        { name: 'textInput', rate: 0.3, strategy: 'fixed', unit: 'millionTokens' },
+        { name: 'textOutput', rate: 2.5, strategy: 'fixed', unit: 'millionTokens' },
+        { name: 'imageOutput', rate: 30, strategy: 'fixed', unit: 'millionTokens' },
+      ],
+    },
+  },
+  {
+    displayName: 'Imagen 4',
+    id: 'imagen-4.0-generate-001',
+    enabled: true,
+    type: 'image',
+    description: 'Imagen 4th generation text-to-image model series',
+    organization: 'Deepmind',
+    releasedAt: '2025-08-15',
+    parameters: imagenGenParameters,
+    pricing: {
+      units: [{ name: 'imageGeneration', rate: 0.04, strategy: 'fixed', unit: 'image' }],
+    },
+  },
+  {
+    displayName: 'Imagen 4 Ultra',
+    id: 'imagen-4.0-ultra-generate-001',
+    enabled: true,
+    type: 'image',
+    description: 'Imagen 4th generation text-to-image model series Ultra version',
+    organization: 'Deepmind',
+    releasedAt: '2025-08-15',
+    parameters: imagenGenParameters,
+    pricing: {
+      units: [{ name: 'imageGeneration', rate: 0.06, strategy: 'fixed', unit: 'image' }],
+    },
+  },
+  {
+    displayName: 'Imagen 4 Fast',
+    id: 'imagen-4.0-fast-generate-001',
+    enabled: true,
+    type: 'image',
+    description: 'Imagen 4th generation text-to-image model series Fast version',
+    organization: 'Deepmind',
+    releasedAt: '2025-08-15',
+    parameters: imagenGenParameters,
+    pricing: {
+      units: [{ name: 'imageGeneration', rate: 0.02, strategy: 'fixed', unit: 'image' }],
+    },
+  },
+];
+
+export const allModels = [...vertexaiChatModels, ...vertexaiImageModels];

 export default allModels;
--- a/packages/utils/package.json
+++ b/packages/utils/package.json
@@ -6,7 +6,8 @@
    ".": "./src/index.ts",
    "./server": "./src/server/index.ts",
    "./client": "./src/client/index.ts",
-    "./object": "./src/object.ts"
+    "./object": "./src/object.ts",
+    "./*": "./src/*.ts"
  },
  "scripts": {
    "test": "vitest",
--- a/packages/utils/src/url.test.ts
+++ b/packages/utils/src/url.test.ts
@@ -1,11 +1,11 @@
 import { describe, expect, it } from 'vitest';

-import { pathString } from './url';
 import {
  inferContentTypeFromImageUrl,
  inferFileExtensionFromImageUrl,
+  isDesktopLocalStaticServerUrl,
  isLocalOrPrivateUrl,
-  isLocalUrl,
+  pathString,
 } from './url';

 describe('pathString', () => {
@@ -404,36 +404,36 @@ describe('inferFileExtensionFromImageUrl', () => {
  });
 });

-describe('isLocalUrl', () => {
+describe('isDesktopLocalStaticServerUrl', () => {
  it('should return true for 127.0.0.1', () => {
-    expect(isLocalUrl('http://127.0.0.1')).toBe(true);
-    expect(isLocalUrl('https://127.0.0.1')).toBe(true);
-    expect(isLocalUrl('http://127.0.0.1:8080')).toBe(true);
-    expect(isLocalUrl('http://127.0.0.1/path/to/resource')).toBe(true);
-    expect(isLocalUrl('https://127.0.0.1/path?query=1#hash')).toBe(true);
+    expect(isDesktopLocalStaticServerUrl('http://127.0.0.1')).toBe(true);
+    expect(isDesktopLocalStaticServerUrl('https://127.0.0.1')).toBe(true);
+    expect(isDesktopLocalStaticServerUrl('http://127.0.0.1:8080')).toBe(true);
+    expect(isDesktopLocalStaticServerUrl('http://127.0.0.1/path/to/resource')).toBe(true);
+    expect(isDesktopLocalStaticServerUrl('https://127.0.0.1/path?query=1#hash')).toBe(true);
  });

  it('should return false for other 127.x.x.x addresses', () => {
-    expect(isLocalUrl('http://127.0.0.2')).toBe(false);
-    expect(isLocalUrl('http://127.1.1.1')).toBe(false);
-    expect(isLocalUrl('http://127.255.255.255')).toBe(false);
+    expect(isDesktopLocalStaticServerUrl('http://127.0.0.2')).toBe(false);
+    expect(isDesktopLocalStaticServerUrl('http://127.1.1.1')).toBe(false);
+    expect(isDesktopLocalStaticServerUrl('http://127.255.255.255')).toBe(false);
  });

  it('should return false for localhost', () => {
-    expect(isLocalUrl('http://localhost')).toBe(false);
-    expect(isLocalUrl('http://localhost:3000')).toBe(false);
-    expect(isLocalUrl('https://localhost/api')).toBe(false);
+    expect(isDesktopLocalStaticServerUrl('http://localhost')).toBe(false);
+    expect(isDesktopLocalStaticServerUrl('http://localhost:3000')).toBe(false);
+    expect(isDesktopLocalStaticServerUrl('https://localhost/api')).toBe(false);
  });

  it('should return false for domain names', () => {
-    expect(isLocalUrl('https://example.com')).toBe(false);
-    expect(isLocalUrl('http://www.google.com')).toBe(false);
+    expect(isDesktopLocalStaticServerUrl('https://example.com')).toBe(false);
+    expect(isDesktopLocalStaticServerUrl('http://www.google.com')).toBe(false);
  });

  it('should return false for malformed URLs', () => {
-    expect(isLocalUrl('invalid-url')).toBe(false);
-    expect(isLocalUrl('http://')).toBe(false);
-    expect(isLocalUrl('')).toBe(false);
+    expect(isDesktopLocalStaticServerUrl('invalid-url')).toBe(false);
+    expect(isDesktopLocalStaticServerUrl('http://')).toBe(false);
+    expect(isDesktopLocalStaticServerUrl('')).toBe(false);
  });
 });

--- a/packages/utils/src/url.ts
+++ b/packages/utils/src/url.ts
@@ -125,25 +125,21 @@ export function inferContentTypeFromImageUrl(url: string) {
 }

 /**
- * Check if a URL points to localhost (127.0.0.1)
 *
- * This function safely determines if the provided URL's hostname is '127.0.0.1'.
- * It handles malformed URLs gracefully by returning false instead of throwing errors.
- *
- * @param url - The URL string to check
- * @returns true if the URL's hostname is '127.0.0.1', false otherwise (including for malformed URLs)
+ * Check if a URL points to desktop local static server
 *
 * @example
 * ```typescript
- * isLocalUrl('http://127.0.0.1:8080/path') // true
- * isLocalUrl('https://example.com') // false
- * isLocalUrl('invalid-url') // false (instead of throwing)
- * isLocalUrl('') // false (instead of throwing)
+ * isDesktopLocalStaticServerUrl('http://127.0.0.1:8080/path') // true
+ * isDesktopLocalStaticServerUrl('http://localhost:8080/path') // false
+ * isDesktopLocalStaticServerUrl('https://example.com') // false
+ * isDesktopLocalStaticServerUrl('invalid-url') // false (instead of throwing)
+ * isDesktopLocalStaticServerUrl('') // false (instead of throwing)
 * ```
 *
 * check: apps/desktop/src/main/core/StaticFileServerManager.ts
 */
-export function isLocalUrl(url: string) {
+export function isDesktopLocalStaticServerUrl(url: string) {
  try {
    return new URL(url).hostname === '127.0.0.1';
  } catch {
--- a/src/services/chat/chat.test.ts
+++ b/src/services/chat/chat.test.ts
@@ -37,9 +37,13 @@ vi.mock('@/utils/fetch', async (importOriginal) => {

  return { ...(module as any), getMessageError: vi.fn() };
 });
-vi.mock('@lobechat/utils', () => ({
-  isLocalUrl: vi.fn(),
+vi.mock('@lobechat/utils/url', () => ({
+  isDesktopLocalStaticServerUrl: vi.fn(),
+}));
+vi.mock('@lobechat/utils/imageToBase64', () => ({
  imageUrlToBase64: vi.fn(),
+}));
+vi.mock('@lobechat/utils/uriParser', () => ({
  parseDataUri: vi.fn(),
 }));

@@ -280,9 +284,10 @@ describe('ChatService', () => {
    describe('should handle content correctly for vision models', () => {
      it('should include image content when with vision model', async () => {
        // Mock utility functions used in processImageList
-        const { parseDataUri, isLocalUrl } = await import('@lobechat/utils');
+        const { parseDataUri } = await import('@lobechat/utils/uriParser');
+        const { isDesktopLocalStaticServerUrl } = await import('@lobechat/utils/url');
        vi.mocked(parseDataUri).mockReturnValue({ type: 'url', base64: null, mimeType: null });
-        vi.mocked(isLocalUrl).mockReturnValue(false); // Not a local URL
+        vi.mocked(isDesktopLocalStaticServerUrl).mockReturnValue(false); // Not a local URL

        const messages = [
          {
@@ -357,11 +362,13 @@ describe('ChatService', () => {

    describe('local image URL conversion', () => {
      it('should convert local image URLs to base64 and call processImageList', async () => {
-        const { imageUrlToBase64, parseDataUri, isLocalUrl } = await import('@lobechat/utils');
+        const { imageUrlToBase64 } = await import('@lobechat/utils/imageToBase64');
+        const { parseDataUri } = await import('@lobechat/utils/uriParser');
+        const { isDesktopLocalStaticServerUrl } = await import('@lobechat/utils/url');

        // Mock for local URL
        vi.mocked(parseDataUri).mockReturnValue({ type: 'url', base64: null, mimeType: null });
-        vi.mocked(isLocalUrl).mockReturnValue(true); // This is a local URL
+        vi.mocked(isDesktopLocalStaticServerUrl).mockReturnValue(true); // This is a local URL
        vi.mocked(imageUrlToBase64).mockResolvedValue({
          base64: 'converted-base64-content',
          mimeType: 'image/png',
@@ -397,7 +404,9 @@ describe('ChatService', () => {

        // Verify the utility functions were called
        expect(parseDataUri).toHaveBeenCalledWith('http://127.0.0.1:3000/uploads/image.png');
-        expect(isLocalUrl).toHaveBeenCalledWith('http://127.0.0.1:3000/uploads/image.png');
+        expect(isDesktopLocalStaticServerUrl).toHaveBeenCalledWith(
+          'http://127.0.0.1:3000/uploads/image.png',
+        );
        expect(imageUrlToBase64).toHaveBeenCalledWith('http://127.0.0.1:3000/uploads/image.png');

        // Verify the final result contains base64 converted URL
@@ -428,11 +437,13 @@ describe('ChatService', () => {
      });

      it('should not convert remote URLs to base64 and call processImageList', async () => {
-        const { imageUrlToBase64, parseDataUri, isLocalUrl } = await import('@lobechat/utils');
+        const { imageUrlToBase64 } = await import('@lobechat/utils/imageToBase64');
+        const { parseDataUri } = await import('@lobechat/utils/uriParser');
+        const { isDesktopLocalStaticServerUrl } = await import('@lobechat/utils/url');

        // Mock for remote URL
        vi.mocked(parseDataUri).mockReturnValue({ type: 'url', base64: null, mimeType: null });
-        vi.mocked(isLocalUrl).mockReturnValue(false); // This is NOT a local URL
+        vi.mocked(isDesktopLocalStaticServerUrl).mockReturnValue(false); // This is NOT a local URL
        vi.mocked(imageUrlToBase64).mockClear(); // Clear to ensure it's not called

        const messages = [
@@ -464,7 +475,9 @@ describe('ChatService', () => {

        // Verify the utility functions were called
        expect(parseDataUri).toHaveBeenCalledWith('https://example.com/remote-image.jpg');
-        expect(isLocalUrl).toHaveBeenCalledWith('https://example.com/remote-image.jpg');
+        expect(isDesktopLocalStaticServerUrl).toHaveBeenCalledWith(
+          'https://example.com/remote-image.jpg',
+        );
        expect(imageUrlToBase64).not.toHaveBeenCalled(); // Should NOT be called for remote URLs

        // Verify the final result preserves original URL
@@ -492,13 +505,15 @@ describe('ChatService', () => {
      });

      it('should handle mixed local and remote URLs correctly', async () => {
-        const { imageUrlToBase64, parseDataUri, isLocalUrl } = await import('@lobechat/utils');
+        const { imageUrlToBase64 } = await import('@lobechat/utils/imageToBase64');
+        const { parseDataUri } = await import('@lobechat/utils/uriParser');
+        const { isDesktopLocalStaticServerUrl } = await import('@lobechat/utils/url');

        // Mock parseDataUri to always return url type
        vi.mocked(parseDataUri).mockReturnValue({ type: 'url', base64: null, mimeType: null });

-        // Mock isLocalUrl to return true only for 127.0.0.1 URLs
-        vi.mocked(isLocalUrl).mockImplementation((url: string) => {
+        // Mock isDesktopLocalStaticServerUrl to return true only for 127.0.0.1 URLs
+        vi.mocked(isDesktopLocalStaticServerUrl).mockImplementation((url: string) => {
          return new URL(url).hostname === '127.0.0.1';
        });

@@ -544,10 +559,16 @@ describe('ChatService', () => {
          model: 'gpt-4-vision-preview',
        });

-        // Verify isLocalUrl was called for each image
-        expect(isLocalUrl).toHaveBeenCalledWith('http://127.0.0.1:3000/local1.jpg');
-        expect(isLocalUrl).toHaveBeenCalledWith('https://example.com/remote1.png');
-        expect(isLocalUrl).toHaveBeenCalledWith('http://127.0.0.1:8080/local2.gif');
+        // Verify isDesktopLocalStaticServerUrl was called for each image
+        expect(isDesktopLocalStaticServerUrl).toHaveBeenCalledWith(
+          'http://127.0.0.1:3000/local1.jpg',
+        );
+        expect(isDesktopLocalStaticServerUrl).toHaveBeenCalledWith(
+          'https://example.com/remote1.png',
+        );
+        expect(isDesktopLocalStaticServerUrl).toHaveBeenCalledWith(
+          'http://127.0.0.1:8080/local2.gif',
+        );

        // Verify imageUrlToBase64 was called only for local URLs
        expect(imageUrlToBase64).toHaveBeenCalledWith('http://127.0.0.1:3000/local1.jpg');
--- a/src/services/chat/clientModelRuntime.test.ts
+++ b/src/services/chat/clientModelRuntime.test.ts
@@ -46,7 +46,7 @@ vi.mock('@/utils/fetch', async (importOriginal) => {

 // Mock image processing utilities
 vi.mock('@/utils/url', () => ({
-  isLocalUrl: vi.fn(),
+  isDesktopLocalStaticServerUrl: vi.fn(),
 }));

 vi.mock('@/utils/imageToBase64', () => ({
@@ -81,12 +81,12 @@ beforeEach(async () => {
  vi.clearAllMocks();

  // Set default mock return values for image processing utilities
-  const { isLocalUrl } = await import('@/utils/url');
+  const { isDesktopLocalStaticServerUrl } = await import('@/utils/url');
  const { imageUrlToBase64 } = await import('@/utils/imageToBase64');
  const { parseDataUri } = await import('@lobechat/model-runtime');

  vi.mocked(parseDataUri).mockReturnValue({ type: 'url', base64: null, mimeType: null });
-  vi.mocked(isLocalUrl).mockReturnValue(false);
+  vi.mocked(isDesktopLocalStaticServerUrl).mockReturnValue(false);
  vi.mocked(imageUrlToBase64).mockResolvedValue({
    base64: 'mock-base64',
    mimeType: 'image/jpeg',