mirror of
https://github.com/lobehub/lobehub.git
synced 2026-03-27 13:29:15 +07:00
🐛 fix(model-runtime): filter unsupported image types (SVG) before sending to vision models (#11698)
Vision models like Claude and Gemini don't support SVG images (image/svg+xml). Previously, SVG images were passed through unchanged, causing runtime errors. Changes: - Add supported image types check in Anthropic context builder - Add supported image types check in Google context builder - Filter out unsupported formats (like SVG) by returning undefined - Add 4 test cases for SVG filtering (base64 and URL scenarios) Supported formats: image/jpeg, image/jpg, image/png, image/gif, image/webp Closes: LOBE-4125 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -125,6 +125,44 @@ describe('anthropicHelpers', () => {
|
|||||||
|
|
||||||
await expect(buildAnthropicBlock(content)).rejects.toThrow('Invalid image URL: invalid-url');
|
await expect(buildAnthropicBlock(content)).rejects.toThrow('Invalid image URL: invalid-url');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('should return undefined for unsupported SVG image (base64)', async () => {
|
||||||
|
vi.mocked(parseDataUri).mockReturnValueOnce({
|
||||||
|
mimeType: 'image/svg+xml',
|
||||||
|
base64: 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==',
|
||||||
|
type: 'base64',
|
||||||
|
});
|
||||||
|
|
||||||
|
const content = {
|
||||||
|
type: 'image_url',
|
||||||
|
image_url: {
|
||||||
|
url: 'data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==',
|
||||||
|
},
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
const result = await buildAnthropicBlock(content);
|
||||||
|
expect(result).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return undefined for unsupported SVG image (URL)', async () => {
|
||||||
|
vi.mocked(parseDataUri).mockReturnValueOnce({
|
||||||
|
mimeType: null,
|
||||||
|
base64: null,
|
||||||
|
type: 'url',
|
||||||
|
});
|
||||||
|
vi.mocked(imageUrlToBase64).mockResolvedValueOnce({
|
||||||
|
base64: 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==',
|
||||||
|
mimeType: 'image/svg+xml',
|
||||||
|
});
|
||||||
|
|
||||||
|
const content = {
|
||||||
|
type: 'image_url',
|
||||||
|
image_url: { url: 'https://example.com/image.svg' },
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
const result = await buildAnthropicBlock(content);
|
||||||
|
expect(result).toBeUndefined();
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('buildAnthropicMessage', () => {
|
describe('buildAnthropicMessage', () => {
|
||||||
|
|||||||
@@ -5,6 +5,19 @@ import OpenAI from 'openai';
|
|||||||
import { OpenAIChatMessage, UserMessageContentPart } from '../../types';
|
import { OpenAIChatMessage, UserMessageContentPart } from '../../types';
|
||||||
import { parseDataUri } from '../../utils/uriParser';
|
import { parseDataUri } from '../../utils/uriParser';
|
||||||
|
|
||||||
|
const ANTHROPIC_SUPPORTED_IMAGE_TYPES = new Set([
|
||||||
|
'image/jpeg',
|
||||||
|
'image/jpg',
|
||||||
|
'image/png',
|
||||||
|
'image/gif',
|
||||||
|
'image/webp',
|
||||||
|
]);
|
||||||
|
|
||||||
|
const isImageTypeSupported = (mimeType: string | null): boolean => {
|
||||||
|
if (!mimeType) return true;
|
||||||
|
return ANTHROPIC_SUPPORTED_IMAGE_TYPES.has(mimeType.toLowerCase());
|
||||||
|
};
|
||||||
|
|
||||||
export const buildAnthropicBlock = async (
|
export const buildAnthropicBlock = async (
|
||||||
content: UserMessageContentPart,
|
content: UserMessageContentPart,
|
||||||
): Promise<Anthropic.ContentBlock | Anthropic.ImageBlockParam | undefined> => {
|
): Promise<Anthropic.ContentBlock | Anthropic.ImageBlockParam | undefined> => {
|
||||||
@@ -23,7 +36,9 @@ export const buildAnthropicBlock = async (
|
|||||||
case 'image_url': {
|
case 'image_url': {
|
||||||
const { mimeType, base64, type } = parseDataUri(content.image_url.url);
|
const { mimeType, base64, type } = parseDataUri(content.image_url.url);
|
||||||
|
|
||||||
if (type === 'base64')
|
if (type === 'base64') {
|
||||||
|
if (!isImageTypeSupported(mimeType)) return undefined;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
source: {
|
source: {
|
||||||
data: base64 as string,
|
data: base64 as string,
|
||||||
@@ -32,9 +47,13 @@ export const buildAnthropicBlock = async (
|
|||||||
},
|
},
|
||||||
type: 'image',
|
type: 'image',
|
||||||
};
|
};
|
||||||
|
}
|
||||||
|
|
||||||
if (type === 'url') {
|
if (type === 'url') {
|
||||||
const { base64, mimeType } = await imageUrlToBase64(content.image_url.url);
|
const { base64, mimeType } = await imageUrlToBase64(content.image_url.url);
|
||||||
|
|
||||||
|
if (!isImageTypeSupported(mimeType)) return undefined;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
source: {
|
source: {
|
||||||
data: base64 as string,
|
data: base64 as string,
|
||||||
|
|||||||
@@ -149,6 +149,48 @@ describe('google contextBuilders', () => {
|
|||||||
thoughtSignature: GEMINI_MAGIC_THOUGHT_SIGNATURE,
|
thoughtSignature: GEMINI_MAGIC_THOUGHT_SIGNATURE,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('should return undefined for unsupported SVG image (base64)', async () => {
|
||||||
|
const svgBase64 =
|
||||||
|
'data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==';
|
||||||
|
|
||||||
|
vi.mocked(parseDataUri).mockReturnValueOnce({
|
||||||
|
base64: 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==',
|
||||||
|
mimeType: 'image/svg+xml',
|
||||||
|
type: 'base64',
|
||||||
|
});
|
||||||
|
|
||||||
|
const content: UserMessageContentPart = {
|
||||||
|
image_url: { url: svgBase64 },
|
||||||
|
type: 'image_url',
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await buildGooglePart(content);
|
||||||
|
expect(result).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return undefined for unsupported SVG image (URL)', async () => {
|
||||||
|
const svgUrl = 'https://example.com/image.svg';
|
||||||
|
|
||||||
|
vi.mocked(parseDataUri).mockReturnValueOnce({
|
||||||
|
base64: null,
|
||||||
|
mimeType: null,
|
||||||
|
type: 'url',
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.spyOn(imageToBase64Module, 'imageUrlToBase64').mockResolvedValueOnce({
|
||||||
|
base64: 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==',
|
||||||
|
mimeType: 'image/svg+xml',
|
||||||
|
});
|
||||||
|
|
||||||
|
const content: UserMessageContentPart = {
|
||||||
|
image_url: { url: svgUrl },
|
||||||
|
type: 'image_url',
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await buildGooglePart(content);
|
||||||
|
expect(result).toBeUndefined();
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('buildGoogleMessage', () => {
|
describe('buildGoogleMessage', () => {
|
||||||
|
|||||||
@@ -11,6 +11,19 @@ import { ChatCompletionTool, OpenAIChatMessage, UserMessageContentPart } from '.
|
|||||||
import { safeParseJSON } from '../../utils/safeParseJSON';
|
import { safeParseJSON } from '../../utils/safeParseJSON';
|
||||||
import { parseDataUri } from '../../utils/uriParser';
|
import { parseDataUri } from '../../utils/uriParser';
|
||||||
|
|
||||||
|
const GOOGLE_SUPPORTED_IMAGE_TYPES = new Set([
|
||||||
|
'image/jpeg',
|
||||||
|
'image/jpg',
|
||||||
|
'image/png',
|
||||||
|
'image/gif',
|
||||||
|
'image/webp',
|
||||||
|
]);
|
||||||
|
|
||||||
|
const isImageTypeSupported = (mimeType: string | null): boolean => {
|
||||||
|
if (!mimeType) return true;
|
||||||
|
return GOOGLE_SUPPORTED_IMAGE_TYPES.has(mimeType.toLowerCase());
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Magic thoughtSignature
|
* Magic thoughtSignature
|
||||||
* @see https://ai.google.dev/gemini-api/docs/thought-signatures#model-behavior:~:text=context_engineering_is_the_way_to_go
|
* @see https://ai.google.dev/gemini-api/docs/thought-signatures#model-behavior:~:text=context_engineering_is_the_way_to_go
|
||||||
@@ -43,6 +56,8 @@ export const buildGooglePart = async (
|
|||||||
throw new TypeError("Image URL doesn't contain base64 data");
|
throw new TypeError("Image URL doesn't contain base64 data");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!isImageTypeSupported(mimeType)) return undefined;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
inlineData: { data: base64, mimeType: mimeType || 'image/png' },
|
inlineData: { data: base64, mimeType: mimeType || 'image/png' },
|
||||||
thoughtSignature: GEMINI_MAGIC_THOUGHT_SIGNATURE,
|
thoughtSignature: GEMINI_MAGIC_THOUGHT_SIGNATURE,
|
||||||
@@ -52,6 +67,8 @@ export const buildGooglePart = async (
|
|||||||
if (type === 'url') {
|
if (type === 'url') {
|
||||||
const { base64, mimeType } = await imageUrlToBase64(content.image_url.url);
|
const { base64, mimeType } = await imageUrlToBase64(content.image_url.url);
|
||||||
|
|
||||||
|
if (!isImageTypeSupported(mimeType)) return undefined;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
inlineData: { data: base64, mimeType },
|
inlineData: { data: base64, mimeType },
|
||||||
thoughtSignature: GEMINI_MAGIC_THOUGHT_SIGNATURE,
|
thoughtSignature: GEMINI_MAGIC_THOUGHT_SIGNATURE,
|
||||||
|
|||||||
Reference in New Issue
Block a user