🐛 fix(model-runtime): filter unsupported image types (SVG) before sending to vision models (#11698)

Vision models like Claude and Gemini don't support SVG images (image/svg+xml).
Previously, SVG images were passed through unchanged, causing runtime errors.

Changes:
- Add supported image types check in Anthropic context builder
- Add supported image types check in Google context builder
- Filter out unsupported formats (like SVG) by returning undefined
- Add 4 test cases for SVG filtering (base64 and URL scenarios)

Supported formats: image/jpeg, image/jpg, image/png, image/gif, image/webp

Closes: LOBE-4125

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Arvin Xu
2026-01-22 17:39:38 +08:00
committed by GitHub
parent 6df77315b9
commit c0c99a7ede
4 changed files with 117 additions and 1 deletions

View File

@@ -125,6 +125,44 @@ describe('anthropicHelpers', () => {
await expect(buildAnthropicBlock(content)).rejects.toThrow('Invalid image URL: invalid-url');
});
it('should return undefined for unsupported SVG image (base64)', async () => {
vi.mocked(parseDataUri).mockReturnValueOnce({
mimeType: 'image/svg+xml',
base64: 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==',
type: 'base64',
});
const content = {
type: 'image_url',
image_url: {
url: 'data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==',
},
} as const;
const result = await buildAnthropicBlock(content);
expect(result).toBeUndefined();
});
it('should return undefined for unsupported SVG image (URL)', async () => {
vi.mocked(parseDataUri).mockReturnValueOnce({
mimeType: null,
base64: null,
type: 'url',
});
vi.mocked(imageUrlToBase64).mockResolvedValueOnce({
base64: 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==',
mimeType: 'image/svg+xml',
});
const content = {
type: 'image_url',
image_url: { url: 'https://example.com/image.svg' },
} as const;
const result = await buildAnthropicBlock(content);
expect(result).toBeUndefined();
});
});
describe('buildAnthropicMessage', () => {

View File

@@ -5,6 +5,19 @@ import OpenAI from 'openai';
import { OpenAIChatMessage, UserMessageContentPart } from '../../types';
import { parseDataUri } from '../../utils/uriParser';
const ANTHROPIC_SUPPORTED_IMAGE_TYPES = new Set([
'image/jpeg',
'image/jpg',
'image/png',
'image/gif',
'image/webp',
]);
const isImageTypeSupported = (mimeType: string | null): boolean => {
if (!mimeType) return true;
return ANTHROPIC_SUPPORTED_IMAGE_TYPES.has(mimeType.toLowerCase());
};
export const buildAnthropicBlock = async (
content: UserMessageContentPart,
): Promise<Anthropic.ContentBlock | Anthropic.ImageBlockParam | undefined> => {
@@ -23,7 +36,9 @@ export const buildAnthropicBlock = async (
case 'image_url': {
const { mimeType, base64, type } = parseDataUri(content.image_url.url);
if (type === 'base64')
if (type === 'base64') {
if (!isImageTypeSupported(mimeType)) return undefined;
return {
source: {
data: base64 as string,
@@ -32,9 +47,13 @@ export const buildAnthropicBlock = async (
},
type: 'image',
};
}
if (type === 'url') {
const { base64, mimeType } = await imageUrlToBase64(content.image_url.url);
if (!isImageTypeSupported(mimeType)) return undefined;
return {
source: {
data: base64 as string,

View File

@@ -149,6 +149,48 @@ describe('google contextBuilders', () => {
thoughtSignature: GEMINI_MAGIC_THOUGHT_SIGNATURE,
});
});
it('should return undefined for unsupported SVG image (base64)', async () => {
const svgBase64 =
'data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==';
vi.mocked(parseDataUri).mockReturnValueOnce({
base64: 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==',
mimeType: 'image/svg+xml',
type: 'base64',
});
const content: UserMessageContentPart = {
image_url: { url: svgBase64 },
type: 'image_url',
};
const result = await buildGooglePart(content);
expect(result).toBeUndefined();
});
it('should return undefined for unsupported SVG image (URL)', async () => {
const svgUrl = 'https://example.com/image.svg';
vi.mocked(parseDataUri).mockReturnValueOnce({
base64: null,
mimeType: null,
type: 'url',
});
vi.spyOn(imageToBase64Module, 'imageUrlToBase64').mockResolvedValueOnce({
base64: 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==',
mimeType: 'image/svg+xml',
});
const content: UserMessageContentPart = {
image_url: { url: svgUrl },
type: 'image_url',
};
const result = await buildGooglePart(content);
expect(result).toBeUndefined();
});
});
describe('buildGoogleMessage', () => {

View File

@@ -11,6 +11,19 @@ import { ChatCompletionTool, OpenAIChatMessage, UserMessageContentPart } from '.
import { safeParseJSON } from '../../utils/safeParseJSON';
import { parseDataUri } from '../../utils/uriParser';
const GOOGLE_SUPPORTED_IMAGE_TYPES = new Set([
'image/jpeg',
'image/jpg',
'image/png',
'image/gif',
'image/webp',
]);
const isImageTypeSupported = (mimeType: string | null): boolean => {
if (!mimeType) return true;
return GOOGLE_SUPPORTED_IMAGE_TYPES.has(mimeType.toLowerCase());
};
/**
* Magic thoughtSignature
* @see https://ai.google.dev/gemini-api/docs/thought-signatures#model-behavior:~:text=context_engineering_is_the_way_to_go
@@ -43,6 +56,8 @@ export const buildGooglePart = async (
throw new TypeError("Image URL doesn't contain base64 data");
}
if (!isImageTypeSupported(mimeType)) return undefined;
return {
inlineData: { data: base64, mimeType: mimeType || 'image/png' },
thoughtSignature: GEMINI_MAGIC_THOUGHT_SIGNATURE,
@@ -52,6 +67,8 @@ export const buildGooglePart = async (
if (type === 'url') {
const { base64, mimeType } = await imageUrlToBase64(content.image_url.url);
if (!isImageTypeSupported(mimeType)) return undefined;
return {
inlineData: { data: base64, mimeType },
thoughtSignature: GEMINI_MAGIC_THOUGHT_SIGNATURE,