feat: update the sandbox export files & save files way (#11249)

feat: update the sandbox export files & save files way
This commit is contained in:
Shinji-Li
2026-01-05 18:03:25 +08:00
committed by GitHub
parent 995e8cf89a
commit 039b0a1064
6 changed files with 177 additions and 250 deletions

View File

@@ -425,51 +425,38 @@ export class CloudSandboxExecutionRuntime {
/**
* Export a file from the sandbox to cloud storage
* 1. Get a pre-signed upload URL from our server
* 2. Call the sandbox to upload the file to that URL
* 3. Return the download URL to the user
* Uses a single tRPC call that handles:
* 1. Generate pre-signed upload URL
* 2. Call sandbox to upload file
* 3. Create persistent file record
* 4. Return permanent /f/:id URL
*/
async exportFile(args: ExportFileParams): Promise<BuiltinServerRuntimeOutput> {
try {
// Extract filename from path
const filename = args.path.split('/').pop() || 'exported_file';
// Step 1: Get pre-signed upload URL from our server
const uploadUrlResult = await codeInterpreterService.getExportFileUploadUrl(
// Single call that handles everything: upload URL generation, sandbox upload, and file record creation
const result = await codeInterpreterService.exportAndUploadFile(
args.path,
filename,
this.context.topicId,
);
if (!uploadUrlResult.success) {
throw new Error(uploadUrlResult.error?.message || 'Failed to get upload URL');
}
// Step 2: Call the sandbox's exportFile tool with the upload URL
// The sandbox will read the file and upload it to the pre-signed URL
const result = await this.callTool('exportFile', {
path: args.path,
uploadUrl: uploadUrlResult.uploadUrl,
});
// Check if the sandbox upload was successful
const uploadSuccess = result.success && result.result?.success !== false;
const fileSize = result.result?.size;
const mimeType = result.result?.mimeType;
const fileContent = result.result?.content;
const state: ExportFileState = {
content: fileContent,
downloadUrl: uploadSuccess ? uploadUrlResult.downloadUrl : '',
filename,
mimeType,
downloadUrl: result.success && result.url ? result.url : '',
fileId: result.fileId,
filename: result.filename,
mimeType: result.mimeType,
path: args.path,
size: fileSize,
success: uploadSuccess,
size: result.size,
success: result.success,
};
if (!uploadSuccess) {
if (!result.success) {
return {
content: JSON.stringify({
error: result.result?.error || 'Failed to upload file from sandbox',
error: result.error?.message || 'Failed to export file from sandbox',
filename,
success: false,
}),
@@ -479,7 +466,7 @@ export class CloudSandboxExecutionRuntime {
}
return {
content: `File exported successfully.\n\nFilename: ${filename}\nDownload URL: ${uploadUrlResult.downloadUrl}`,
content: `File exported successfully.\n\nFilename: ${filename}\nDownload URL: ${result.url}`,
state,
success: true,
};

View File

@@ -90,10 +90,10 @@ export interface GlobFilesState {
}
export interface ExportFileState {
/** File content for text files (only when mimeType is text-like and size <= 1MB) */
content?: string;
/** The download URL for the exported file */
/** The download URL for the exported file (permanent /f/:id URL) */
downloadUrl: string;
/** The file ID in database (returned from server) */
fileId?: string;
/** The exported file name */
filename: string;
/** The MIME type of the file */

View File

@@ -1,14 +1,13 @@
import { type CodeInterpreterToolName, MarketSDK } from '@lobehub/market-sdk';
import { TRPCError } from '@trpc/server';
import debug from 'debug';
import { sha256 } from 'js-sha256';
import { z } from 'zod';
import { DocumentModel } from '@/database/models/document';
import { FileModel } from '@/database/models/file';
import { type ToolCallContent } from '@/libs/mcp';
import { authedProcedure, router } from '@/libs/trpc/lambda';
import { marketUserInfo, serverDatabase, telemetry } from '@/libs/trpc/lambda/middleware';
import { generateTrustedClientToken } from '@/libs/trusted-client';
import { generateTrustedClientToken, isTrustedClientEnabled } from '@/libs/trusted-client';
import { FileS3 } from '@/server/modules/S3';
import { DiscoverService } from '@/server/services/discover';
import { FileService } from '@/server/services/file';
@@ -69,21 +68,13 @@ const callCodeInterpreterToolSchema = z.object({
userId: z.string(),
});
// Schema for getting export file upload URL
const getExportFileUploadUrlSchema = z.object({
// Schema for export and upload file (combined operation)
const exportAndUploadFileSchema = z.object({
filename: z.string(),
path: z.string(),
topicId: z.string(),
});
// Schema for saving exported file content to document
const saveExportedFileContentSchema = z.object({
content: z.string(),
fileId: z.string(),
fileType: z.string(),
filename: z.string(),
url: z.string(),
});
// Schema for cloud MCP endpoint call
const callCloudMcpEndpointSchema = z.object({
apiParams: z.record(z.any()),
@@ -94,8 +85,7 @@ const callCloudMcpEndpointSchema = z.object({
// ============================== Type Exports ==============================
export type CallCodeInterpreterToolInput = z.infer<typeof callCodeInterpreterToolSchema>;
export type GetExportFileUploadUrlInput = z.infer<typeof getExportFileUploadUrlSchema>;
export type SaveExportedFileContentInput = z.infer<typeof saveExportedFileContentSchema>;
export type ExportAndUploadFileInput = z.infer<typeof exportAndUploadFileSchema>;
export interface CallToolResult {
error?: {
@@ -107,22 +97,16 @@ export interface CallToolResult {
success: boolean;
}
export interface GetExportFileUploadUrlResult {
downloadUrl: string;
error?: {
message: string;
};
key: string;
success: boolean;
uploadUrl: string;
}
export interface SaveExportedFileContentResult {
documentId?: string;
export interface ExportAndUploadFileResult {
error?: {
message: string;
};
fileId?: string;
filename: string;
mimeType?: string;
size?: number;
success: boolean;
url?: string;
}
// ============================== Router ==============================
@@ -140,17 +124,26 @@ export const marketRouter = router({
let result: { content: string; state: any; success: boolean } | undefined;
try {
// Query user_settings to get market.accessToken
const userState = await ctx.userModel.getUserState(async () => ({}));
const userAccessToken = userState.settings?.market?.accessToken;
// Check if trusted client is enabled - if so, we don't need user's accessToken
const trustedClientEnabled = isTrustedClientEnabled();
log('callCloudMcpEndpoint: userAccessToken exists=%s', !!userAccessToken);
let userAccessToken: string | undefined;
if (!userAccessToken) {
throw new TRPCError({
code: 'UNAUTHORIZED',
message: 'User access token not found. Please sign in to Market first.',
});
if (!trustedClientEnabled) {
// Query user_settings to get market.accessToken only if trusted client is not enabled
const userState = await ctx.userModel.getUserState(async () => ({}));
userAccessToken = userState.settings?.market?.accessToken;
log('callCloudMcpEndpoint: userAccessToken exists=%s', !!userAccessToken);
if (!userAccessToken) {
throw new TRPCError({
code: 'UNAUTHORIZED',
message: 'User access token not found. Please sign in to Market first.',
});
}
} else {
log('callCloudMcpEndpoint: using trusted client authentication');
}
const cloudResult = await ctx.discoverService.callCloudMcpEndpoint({
@@ -277,99 +270,122 @@ export const marketRouter = router({
}),
/**
* Generate a pre-signed upload URL for exporting files from sandbox
* Export a file from sandbox and upload to S3, then create a persistent file record
* This combines the previous getExportFileUploadUrl + callCodeInterpreterTool + createFileRecord flow
* Returns a permanent /f/:id URL instead of a temporary pre-signed URL
*/
getExportFileUploadUrl: marketToolProcedure
.input(getExportFileUploadUrlSchema)
.mutation(async ({ input }) => {
const { filename, topicId } = input;
exportAndUploadFile: marketToolProcedure
.input(exportAndUploadFileSchema)
.mutation(async ({ input, ctx }) => {
const { path, filename, topicId } = input;
log('Generating export file upload URL for: %s in topic: %s', filename, topicId);
log('Exporting and uploading file: %s from path: %s in topic: %s', filename, path, topicId);
try {
const s3 = new FileS3();
// Use date-based sharding for privacy compliance (GDPR, CCPA)
const today = new Date().toISOString().split('T')[0];
// Generate a unique key for the exported file
const key = `code-interpreter-exports/${topicId}/${filename}`;
const key = `code-interpreter-exports/${today}/${topicId}/${filename}`;
// Generate pre-signed upload URL
// Step 1: Generate pre-signed upload URL
const uploadUrl = await s3.createPreSignedUrl(key);
// Generate download URL (pre-signed for preview)
const downloadUrl = await s3.createPreSignedUrlForPreview(key);
log('Generated upload URL for key: %s', key);
return {
downloadUrl,
key,
success: true,
uploadUrl,
} as GetExportFileUploadUrlResult;
} catch (error) {
log('Error generating export file upload URL: %O', error);
// Step 2: Generate trusted client token if user info is available
const trustedClientToken = ctx.marketUserInfo
? generateTrustedClientToken(ctx.marketUserInfo)
: undefined;
return {
downloadUrl: '',
error: {
message: (error as Error).message,
},
key: '',
success: false,
uploadUrl: '',
} as GetExportFileUploadUrlResult;
}
}),
// Only require user accessToken if trusted client is not available
let userAccessToken: string | undefined;
if (!trustedClientToken) {
const userState = await ctx.userModel.getUserState(async () => ({}));
userAccessToken = userState.settings?.market?.accessToken;
/**
* Save exported file content to documents table
*/
saveExportedFileContent: marketToolProcedure
.input(saveExportedFileContentSchema)
.mutation(async ({ ctx, input }) => {
const { content, fileId, fileType, filename, url } = input;
log('Saving exported file content: fileId=%s, filename=%s', fileId, filename);
try {
const documentModel = new DocumentModel(ctx.serverDB, ctx.userId);
const fileModel = new FileModel(ctx.serverDB, ctx.userId);
// Verify the file exists
const file = await fileModel.findById(fileId);
if (!file) {
return {
error: { message: 'File not found' },
success: false,
} as SaveExportedFileContentResult;
if (!userAccessToken) {
return {
error: { message: 'User access token not found. Please sign in to Market first.' },
filename,
success: false,
} as ExportAndUploadFileResult;
}
} else {
log('Using trusted client authentication for exportAndUploadFile');
}
// Create document record with the file content
const document = await documentModel.create({
content,
fileId,
fileType,
filename,
source: url,
sourceType: 'file',
title: filename,
totalCharCount: content.length,
totalLineCount: content.split('\n').length,
// Initialize MarketSDK
const market = new MarketSDK({
accessToken: userAccessToken,
baseURL: process.env.NEXT_PUBLIC_MARKET_BASE_URL,
trustedClientToken,
});
log('Created document for exported file: documentId=%s, fileId=%s', document.id, fileId);
// Step 3: Call sandbox's exportFile tool with the upload URL
const response = await market.plugins.runBuildInTool(
'exportFile',
{ path, uploadUrl },
{ topicId, userId: ctx.userId },
);
log('Sandbox exportFile response: %O', response);
if (!response.success) {
return {
error: { message: response.error?.message || 'Failed to export file from sandbox' },
filename,
success: false,
} as ExportAndUploadFileResult;
}
const result = response.data?.result;
const uploadSuccess = result?.success !== false;
if (!uploadSuccess) {
return {
error: { message: result?.error || 'Failed to upload file from sandbox' },
filename,
success: false,
} as ExportAndUploadFileResult;
}
// Step 4: Get file metadata from S3 to verify upload and get actual size
const metadata = await s3.getFileMetadata(key);
const fileSize = metadata.contentLength;
const mimeType = metadata.contentType || result?.mimeType || 'application/octet-stream';
// Step 5: Create persistent file record using FileService
// Generate a simple hash from the key (since we don't have the actual file content)
const fileHash = sha256(key + Date.now().toString());
const { fileId, url } = await ctx.fileService.createFileRecord({
fileHash,
fileType: mimeType,
name: filename,
size: fileSize,
url: key, // Store S3 key
});
log('Created file record: fileId=%s, url=%s', fileId, url);
return {
documentId: document.id,
fileId,
filename,
mimeType,
size: fileSize,
success: true,
} as SaveExportedFileContentResult;
url, // This is the permanent /f/:id URL
} as ExportAndUploadFileResult;
} catch (error) {
log('Error saving exported file content: %O', error);
log('Error in exportAndUploadFile: %O', error);
return {
error: { message: (error as Error).message },
filename,
success: false,
} as SaveExportedFileContentResult;
} as ExportAndUploadFileResult;
}
}),
});

View File

@@ -149,7 +149,7 @@ export class DiscoverService {
apiParams: Record<string, any>;
identifier: string;
toolName: string;
userAccessToken: string;
userAccessToken?: string;
}) {
log('callCloudMcpEndpoint: params=%O', {
apiParams: params.apiParams,
@@ -159,7 +159,14 @@ export class DiscoverService {
});
try {
// Call cloud gateway with user access token in Authorization header
// Build headers - only include Authorization if userAccessToken is provided
// When userAccessToken is not provided, MarketSDK will use trustedClientToken for authentication
const headers: Record<string, string> = {};
if (params.userAccessToken) {
headers.Authorization = `Bearer ${params.userAccessToken}`;
}
// Call cloud gateway with optional user access token in Authorization header
const result = await this.market.plugins.callCloudGateway(
{
apiParams: params.apiParams,
@@ -167,9 +174,7 @@ export class DiscoverService {
toolName: params.toolName,
},
{
headers: {
Authorization: `Bearer ${params.userAccessToken}`,
},
headers,
},
);

View File

@@ -2,10 +2,8 @@ import { toolsClient } from '@/libs/trpc/client';
import type {
CallCodeInterpreterToolInput,
CallToolResult,
GetExportFileUploadUrlInput,
GetExportFileUploadUrlResult,
SaveExportedFileContentInput,
SaveExportedFileContentResult,
ExportAndUploadFileInput,
ExportAndUploadFileResult,
} from '@/server/routers/tools/market';
import { useUserStore } from '@/store/user';
import { settingsSelectors } from '@/store/user/slices/settings/selectors/settings';
@@ -44,31 +42,25 @@ class CodeInterpreterService {
}
/**
* Get a pre-signed upload URL for exporting a file from the sandbox
* Export a file from sandbox and upload to S3, then create a persistent file record
* This is a single call that combines: getUploadUrl + callTool(exportFile) + createFileRecord
* Returns a permanent /f/:id URL instead of a temporary pre-signed URL
* @param path - The file path in the sandbox
* @param filename - The name of the file to export
* @param topicId - The topic ID for organizing files
*/
async getExportFileUploadUrl(
async exportAndUploadFile(
path: string,
filename: string,
topicId: string,
): Promise<GetExportFileUploadUrlResult> {
const input: GetExportFileUploadUrlInput = {
): Promise<ExportAndUploadFileResult> {
const input: ExportAndUploadFileInput = {
filename,
path,
topicId,
};
return toolsClient.market.getExportFileUploadUrl.mutate(input);
}
/**
* Save exported file content to documents table
* This creates a document record linked to the file for content retrieval
* @param params - File content and metadata
*/
async saveExportedFileContent(
params: SaveExportedFileContentInput,
): Promise<SaveExportedFileContentResult> {
return toolsClient.market.saveExportedFileContent.mutate(params);
return toolsClient.market.exportAndUploadFile.mutate(input);
}
}

View File

@@ -9,63 +9,20 @@ import { type StateCreator } from 'zustand/vanilla';
import { type MCPToolCallResult } from '@/libs/mcp';
import { chatService } from '@/services/chat';
import { codeInterpreterService } from '@/services/codeInterpreter';
import { fileService } from '@/services/file';
import { mcpService } from '@/services/mcp';
import { messageService } from '@/services/message';
import { AI_RUNTIME_OPERATION_TYPES } from '@/store/chat/slices/operation';
import { type ChatStore } from '@/store/chat/store';
import { useToolStore } from '@/store/tool';
import { hasExecutor } from '@/store/tool/slices/builtin/executors';
import { useUserStore } from '@/store/user';
import { userProfileSelectors } from '@/store/user/slices/auth/selectors';
import { safeParseJSON } from '@/utils/safeParseJSON';
import { dbMessageSelectors } from '../../message/selectors';
const log = debug('lobe-store:plugin-types');
/**
* Get MIME type from filename extension
*/
const getMimeTypeFromFilename = (filename: string): string => {
const ext = filename.split('.').pop()?.toLowerCase() || '';
const mimeTypes: Record<string, string> = {
// Images
bmp: 'image/bmp',
gif: 'image/gif',
jpeg: 'image/jpeg',
jpg: 'image/jpeg',
png: 'image/png',
svg: 'image/svg+xml',
webp: 'image/webp',
// Videos
mp4: 'video/mp4',
webm: 'video/webm',
mov: 'video/quicktime',
avi: 'video/x-msvideo',
// Documents
csv: 'text/csv',
doc: 'application/msword',
docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
html: 'text/html',
json: 'application/json',
md: 'text/markdown',
pdf: 'application/pdf',
ppt: 'application/vnd.ms-powerpoint',
pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
rtf: 'application/rtf',
txt: 'text/plain',
xls: 'application/vnd.ms-excel',
xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
xml: 'application/xml',
// Code
css: 'text/css',
js: 'text/javascript',
py: 'text/x-python',
ts: 'text/typescript',
};
return mimeTypes[ext] || 'application/octet-stream';
};
/**
* Plugin type-specific implementations
* Each method handles a specific type of plugin invocation
@@ -284,10 +241,13 @@ export const pluginTypes: StateCreator<
const { CloudSandboxExecutionRuntime } =
await import('@lobechat/builtin-tool-cloud-sandbox/executionRuntime');
// Get userId from user store
const userId = userProfileSelectors.userId(useUserStore.getState()) || 'anonymous';
// Create runtime with context
const runtime = new CloudSandboxExecutionRuntime({
topicId: message?.topicId || 'default',
userId: 'current-user', // TODO: Get actual userId from auth context
userId,
});
// Parse arguments
@@ -341,58 +301,25 @@ export const pluginTypes: StateCreator<
context,
);
// Handle exportFile: save exported file and associate with assistant message (parent)
// Handle exportFile: associate the file (already created by server) with assistant message (parent)
if (payload.apiName === 'exportFile' && data.success && data.state) {
const exportState = data.state as ExportFileState;
if (exportState.downloadUrl && exportState.filename) {
// Server now creates the file record and returns fileId in the response
if (exportState.fileId && exportState.filename) {
try {
// Generate a hash from the URL path (without query params) for deduplication
// Extract the path before query params: .../code-interpreter-exports/tpc_xxx/filename.ext
const urlPath = exportState.downloadUrl.split('?')[0];
const hash = `ci-export-${btoa(urlPath).slice(0, 32)}`;
// Use mimeType from state if available, otherwise infer from filename
const mimeType = exportState.mimeType || getMimeTypeFromFilename(exportState.filename);
// 1. Create file record in database
const fileResult = await fileService.createFile({
fileType: mimeType,
hash,
name: exportState.filename,
size: exportState.size || 0,
source: 'code-interpreter',
url: exportState.downloadUrl,
});
// 2. If there's text content, save it to documents table for retrieval
if (exportState.content) {
await codeInterpreterService.saveExportedFileContent({
content: exportState.content,
fileId: fileResult.id,
fileType: mimeType,
filename: exportState.filename,
url: exportState.downloadUrl,
});
log(
'[invokeCloudCodeInterpreterTool] Saved file content to document: fileId=%s',
fileResult.id,
);
}
// 3. Associate file with the assistant message (parent of tool message)
// Associate file with the assistant message (parent of tool message)
// The current message (id) is the tool message, we need to attach to its parent
const targetMessageId = message?.parentId || id;
await messageService.addFilesToMessage(targetMessageId, [fileResult.id], {
await messageService.addFilesToMessage(targetMessageId, [exportState.fileId], {
agentId: message?.agentId,
topicId: message?.topicId,
});
log(
'[invokeCloudCodeInterpreterTool] Saved exported file: targetMessageId=%s, fileId=%s, filename=%s',
'[invokeCloudCodeInterpreterTool] Associated exported file with message: targetMessageId=%s, fileId=%s, filename=%s',
targetMessageId,
fileResult.id,
exportState.fileId,
exportState.filename,
);
} catch (error) {