mirror of
https://github.com/lobehub/lobehub.git
synced 2026-03-26 13:19:34 +07:00
🐛 fix(model-runtime): filter unsupported image types (SVG) before sending to vision models (#11698)
Vision models like Claude and Gemini don't support SVG images (image/svg+xml). Previously, SVG images were passed through unchanged, causing runtime errors. Changes: - Add supported image types check in Anthropic context builder - Add supported image types check in Google context builder - Filter out unsupported formats (like SVG) by returning undefined - Add 4 test cases for SVG filtering (base64 and URL scenarios) Supported formats: image/jpeg, image/jpg, image/png, image/gif, image/webp Closes: LOBE-4125 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -125,6 +125,44 @@ describe('anthropicHelpers', () => {
|
||||
|
||||
await expect(buildAnthropicBlock(content)).rejects.toThrow('Invalid image URL: invalid-url');
|
||||
});
|
||||
|
||||
it('should return undefined for unsupported SVG image (base64)', async () => {
|
||||
vi.mocked(parseDataUri).mockReturnValueOnce({
|
||||
mimeType: 'image/svg+xml',
|
||||
base64: 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==',
|
||||
type: 'base64',
|
||||
});
|
||||
|
||||
const content = {
|
||||
type: 'image_url',
|
||||
image_url: {
|
||||
url: 'data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==',
|
||||
},
|
||||
} as const;
|
||||
|
||||
const result = await buildAnthropicBlock(content);
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should return undefined for unsupported SVG image (URL)', async () => {
|
||||
vi.mocked(parseDataUri).mockReturnValueOnce({
|
||||
mimeType: null,
|
||||
base64: null,
|
||||
type: 'url',
|
||||
});
|
||||
vi.mocked(imageUrlToBase64).mockResolvedValueOnce({
|
||||
base64: 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==',
|
||||
mimeType: 'image/svg+xml',
|
||||
});
|
||||
|
||||
const content = {
|
||||
type: 'image_url',
|
||||
image_url: { url: 'https://example.com/image.svg' },
|
||||
} as const;
|
||||
|
||||
const result = await buildAnthropicBlock(content);
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildAnthropicMessage', () => {
|
||||
|
||||
@@ -5,6 +5,19 @@ import OpenAI from 'openai';
|
||||
import { OpenAIChatMessage, UserMessageContentPart } from '../../types';
|
||||
import { parseDataUri } from '../../utils/uriParser';
|
||||
|
||||
const ANTHROPIC_SUPPORTED_IMAGE_TYPES = new Set([
|
||||
'image/jpeg',
|
||||
'image/jpg',
|
||||
'image/png',
|
||||
'image/gif',
|
||||
'image/webp',
|
||||
]);
|
||||
|
||||
const isImageTypeSupported = (mimeType: string | null): boolean => {
|
||||
if (!mimeType) return true;
|
||||
return ANTHROPIC_SUPPORTED_IMAGE_TYPES.has(mimeType.toLowerCase());
|
||||
};
|
||||
|
||||
export const buildAnthropicBlock = async (
|
||||
content: UserMessageContentPart,
|
||||
): Promise<Anthropic.ContentBlock | Anthropic.ImageBlockParam | undefined> => {
|
||||
@@ -23,7 +36,9 @@ export const buildAnthropicBlock = async (
|
||||
case 'image_url': {
|
||||
const { mimeType, base64, type } = parseDataUri(content.image_url.url);
|
||||
|
||||
if (type === 'base64')
|
||||
if (type === 'base64') {
|
||||
if (!isImageTypeSupported(mimeType)) return undefined;
|
||||
|
||||
return {
|
||||
source: {
|
||||
data: base64 as string,
|
||||
@@ -32,9 +47,13 @@ export const buildAnthropicBlock = async (
|
||||
},
|
||||
type: 'image',
|
||||
};
|
||||
}
|
||||
|
||||
if (type === 'url') {
|
||||
const { base64, mimeType } = await imageUrlToBase64(content.image_url.url);
|
||||
|
||||
if (!isImageTypeSupported(mimeType)) return undefined;
|
||||
|
||||
return {
|
||||
source: {
|
||||
data: base64 as string,
|
||||
|
||||
@@ -149,6 +149,48 @@ describe('google contextBuilders', () => {
|
||||
thoughtSignature: GEMINI_MAGIC_THOUGHT_SIGNATURE,
|
||||
});
|
||||
});
|
||||
|
||||
it('should return undefined for unsupported SVG image (base64)', async () => {
|
||||
const svgBase64 =
|
||||
'data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==';
|
||||
|
||||
vi.mocked(parseDataUri).mockReturnValueOnce({
|
||||
base64: 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==',
|
||||
mimeType: 'image/svg+xml',
|
||||
type: 'base64',
|
||||
});
|
||||
|
||||
const content: UserMessageContentPart = {
|
||||
image_url: { url: svgBase64 },
|
||||
type: 'image_url',
|
||||
};
|
||||
|
||||
const result = await buildGooglePart(content);
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should return undefined for unsupported SVG image (URL)', async () => {
|
||||
const svgUrl = 'https://example.com/image.svg';
|
||||
|
||||
vi.mocked(parseDataUri).mockReturnValueOnce({
|
||||
base64: null,
|
||||
mimeType: null,
|
||||
type: 'url',
|
||||
});
|
||||
|
||||
vi.spyOn(imageToBase64Module, 'imageUrlToBase64').mockResolvedValueOnce({
|
||||
base64: 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==',
|
||||
mimeType: 'image/svg+xml',
|
||||
});
|
||||
|
||||
const content: UserMessageContentPart = {
|
||||
image_url: { url: svgUrl },
|
||||
type: 'image_url',
|
||||
};
|
||||
|
||||
const result = await buildGooglePart(content);
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildGoogleMessage', () => {
|
||||
|
||||
@@ -11,6 +11,19 @@ import { ChatCompletionTool, OpenAIChatMessage, UserMessageContentPart } from '.
|
||||
import { safeParseJSON } from '../../utils/safeParseJSON';
|
||||
import { parseDataUri } from '../../utils/uriParser';
|
||||
|
||||
const GOOGLE_SUPPORTED_IMAGE_TYPES = new Set([
|
||||
'image/jpeg',
|
||||
'image/jpg',
|
||||
'image/png',
|
||||
'image/gif',
|
||||
'image/webp',
|
||||
]);
|
||||
|
||||
const isImageTypeSupported = (mimeType: string | null): boolean => {
|
||||
if (!mimeType) return true;
|
||||
return GOOGLE_SUPPORTED_IMAGE_TYPES.has(mimeType.toLowerCase());
|
||||
};
|
||||
|
||||
/**
|
||||
* Magic thoughtSignature
|
||||
* @see https://ai.google.dev/gemini-api/docs/thought-signatures#model-behavior:~:text=context_engineering_is_the_way_to_go
|
||||
@@ -43,6 +56,8 @@ export const buildGooglePart = async (
|
||||
throw new TypeError("Image URL doesn't contain base64 data");
|
||||
}
|
||||
|
||||
if (!isImageTypeSupported(mimeType)) return undefined;
|
||||
|
||||
return {
|
||||
inlineData: { data: base64, mimeType: mimeType || 'image/png' },
|
||||
thoughtSignature: GEMINI_MAGIC_THOUGHT_SIGNATURE,
|
||||
@@ -52,6 +67,8 @@ export const buildGooglePart = async (
|
||||
if (type === 'url') {
|
||||
const { base64, mimeType } = await imageUrlToBase64(content.image_url.url);
|
||||
|
||||
if (!isImageTypeSupported(mimeType)) return undefined;
|
||||
|
||||
return {
|
||||
inlineData: { data: base64, mimeType },
|
||||
thoughtSignature: GEMINI_MAGIC_THOUGHT_SIGNATURE,
|
||||
|
||||
Reference in New Issue
Block a user