diff --git a/packages/model-runtime/src/core/contextBuilders/anthropic.test.ts b/packages/model-runtime/src/core/contextBuilders/anthropic.test.ts index 728784e305..06f74c1a46 100644 --- a/packages/model-runtime/src/core/contextBuilders/anthropic.test.ts +++ b/packages/model-runtime/src/core/contextBuilders/anthropic.test.ts @@ -125,6 +125,44 @@ describe('anthropicHelpers', () => { await expect(buildAnthropicBlock(content)).rejects.toThrow('Invalid image URL: invalid-url'); }); + + it('should return undefined for unsupported SVG image (base64)', async () => { + vi.mocked(parseDataUri).mockReturnValueOnce({ + mimeType: 'image/svg+xml', + base64: 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==', + type: 'base64', + }); + + const content = { + type: 'image_url', + image_url: { + url: 'data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==', + }, + } as const; + + const result = await buildAnthropicBlock(content); + expect(result).toBeUndefined(); + }); + + it('should return undefined for unsupported SVG image (URL)', async () => { + vi.mocked(parseDataUri).mockReturnValueOnce({ + mimeType: null, + base64: null, + type: 'url', + }); + vi.mocked(imageUrlToBase64).mockResolvedValueOnce({ + base64: 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==', + mimeType: 'image/svg+xml', + }); + + const content = { + type: 'image_url', + image_url: { url: 'https://example.com/image.svg' }, + } as const; + + const result = await buildAnthropicBlock(content); + expect(result).toBeUndefined(); + }); }); describe('buildAnthropicMessage', () => { diff --git a/packages/model-runtime/src/core/contextBuilders/anthropic.ts b/packages/model-runtime/src/core/contextBuilders/anthropic.ts index e6cd478889..a40ee58e86 100644 --- a/packages/model-runtime/src/core/contextBuilders/anthropic.ts +++ b/packages/model-runtime/src/core/contextBuilders/anthropic.ts @@ -5,6 +5,19 @@ import OpenAI from 'openai'; import { OpenAIChatMessage, UserMessageContentPart } from '../../types'; import { parseDataUri } from '../../utils/uriParser'; +const ANTHROPIC_SUPPORTED_IMAGE_TYPES = new Set([ + 'image/jpeg', + 'image/jpg', + 'image/png', + 'image/gif', + 'image/webp', +]); + +const isImageTypeSupported = (mimeType: string | null): boolean => { + if (!mimeType) return true; + return ANTHROPIC_SUPPORTED_IMAGE_TYPES.has(mimeType.toLowerCase()); +}; + export const buildAnthropicBlock = async ( content: UserMessageContentPart, ): Promise => { @@ -23,7 +36,9 @@ export const buildAnthropicBlock = async ( case 'image_url': { const { mimeType, base64, type } = parseDataUri(content.image_url.url); - if (type === 'base64') + if (type === 'base64') { + if (!isImageTypeSupported(mimeType)) return undefined; + return { source: { data: base64 as string, @@ -32,9 +47,13 @@ export const buildAnthropicBlock = async ( }, type: 'image', }; + } if (type === 'url') { const { base64, mimeType } = await imageUrlToBase64(content.image_url.url); + + if (!isImageTypeSupported(mimeType)) return undefined; + return { source: { data: base64 as string, diff --git a/packages/model-runtime/src/core/contextBuilders/google.test.ts b/packages/model-runtime/src/core/contextBuilders/google.test.ts index 5a39a32c84..7d2a80dd9a 100644 --- a/packages/model-runtime/src/core/contextBuilders/google.test.ts +++ b/packages/model-runtime/src/core/contextBuilders/google.test.ts @@ -149,6 +149,48 @@ describe('google contextBuilders', () => { thoughtSignature: GEMINI_MAGIC_THOUGHT_SIGNATURE, }); }); + + it('should return undefined for unsupported SVG image (base64)', async () => { + const svgBase64 = + 'data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg=='; + + vi.mocked(parseDataUri).mockReturnValueOnce({ + base64: 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==', + mimeType: 'image/svg+xml', + type: 'base64', + }); + + const content: UserMessageContentPart = { + image_url: { url: svgBase64 }, + type: 'image_url', + }; + + const result = await buildGooglePart(content); + expect(result).toBeUndefined(); + }); + + it('should return undefined for unsupported SVG image (URL)', async () => { + const svgUrl = 'https://example.com/image.svg'; + + vi.mocked(parseDataUri).mockReturnValueOnce({ + base64: null, + mimeType: null, + type: 'url', + }); + + vi.spyOn(imageToBase64Module, 'imageUrlToBase64').mockResolvedValueOnce({ + base64: 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==', + mimeType: 'image/svg+xml', + }); + + const content: UserMessageContentPart = { + image_url: { url: svgUrl }, + type: 'image_url', + }; + + const result = await buildGooglePart(content); + expect(result).toBeUndefined(); + }); }); describe('buildGoogleMessage', () => { diff --git a/packages/model-runtime/src/core/contextBuilders/google.ts b/packages/model-runtime/src/core/contextBuilders/google.ts index e11f981bc6..8e174a9cda 100644 --- a/packages/model-runtime/src/core/contextBuilders/google.ts +++ b/packages/model-runtime/src/core/contextBuilders/google.ts @@ -11,6 +11,19 @@ import { ChatCompletionTool, OpenAIChatMessage, UserMessageContentPart } from '. import { safeParseJSON } from '../../utils/safeParseJSON'; import { parseDataUri } from '../../utils/uriParser'; +const GOOGLE_SUPPORTED_IMAGE_TYPES = new Set([ + 'image/jpeg', + 'image/jpg', + 'image/png', + 'image/gif', + 'image/webp', +]); + +const isImageTypeSupported = (mimeType: string | null): boolean => { + if (!mimeType) return true; + return GOOGLE_SUPPORTED_IMAGE_TYPES.has(mimeType.toLowerCase()); +}; + /** * Magic thoughtSignature * @see https://ai.google.dev/gemini-api/docs/thought-signatures#model-behavior:~:text=context_engineering_is_the_way_to_go @@ -43,6 +56,8 @@ export const buildGooglePart = async ( throw new TypeError("Image URL doesn't contain base64 data"); } + if (!isImageTypeSupported(mimeType)) return undefined; + return { inlineData: { data: base64, mimeType: mimeType || 'image/png' }, thoughtSignature: GEMINI_MAGIC_THOUGHT_SIGNATURE, @@ -52,6 +67,8 @@ export const buildGooglePart = async ( if (type === 'url') { const { base64, mimeType } = await imageUrlToBase64(content.image_url.url); + if (!isImageTypeSupported(mimeType)) return undefined; + return { inlineData: { data: base64, mimeType }, thoughtSignature: GEMINI_MAGIC_THOUGHT_SIGNATURE,