🐛 fix: fix page selection not display correctly (#11765)

* fix page selection

* fix page selection

* fix page selection

* fix page selection

* fix page context engine

* fix page context engine
This commit is contained in:
Arvin Xu
2026-01-24 15:23:15 +08:00
committed by GitHub
parent 0755965836
commit 7ae5f687f7
27 changed files with 1586 additions and 71 deletions

View File

@@ -208,7 +208,9 @@
"operation.sendMessage": "Sending message",
"owner": "Group owner",
"pageCopilot.title": "Page Agent",
"pageCopilot.welcome": "**Clearer, sharper writing**\n\nDraft, rewrite, or polish—tell me your intent and Ill refine the rest.",
"pageCopilot.welcome": "**Clearer, sharper writing**\n\nDraft, rewrite, or polish—tell me your intent and I'll refine the rest.",
"pageSelection.lines": "Lines {{start}}-{{end}}",
"pageSelection.reference": "Selected Text",
"pin": "Pin",
"pinOff": "Unpin",
"prompts.summaryExpert": "As a summary expert, please summarize the following content based on the system prompts above:",

View File

@@ -209,6 +209,8 @@
"owner": "群主",
"pageCopilot.title": "文稿助理",
"pageCopilot.welcome": "**让文字更清晰、更到位**\n\n起草、改写、润色都可以。你把意图说清楚其余交给我打磨",
"pageSelection.lines": "第 {{start}}-{{end}} 行",
"pageSelection.reference": "选中文本",
"pin": "置顶",
"pinOff": "取消置顶",
"prompts.summaryExpert": "作为一名总结专家,请结合以上系统提示词,将以下内容进行总结:",

View File

@@ -0,0 +1,204 @@
import type { Message, PipelineContext, ProcessorOptions } from '../types';
import { BaseProcessor } from './BaseProcessor';
import { CONTEXT_INSTRUCTION, SYSTEM_CONTEXT_END, SYSTEM_CONTEXT_START } from './constants';
/**
* Base Provider for appending content to every user message
* Used for injecting context that should be attached to each user message individually
* (e.g., page selections that are specific to each message)
*
* Features:
* - Iterates through all user messages
* - For each message, calls buildContentForMessage to get content to inject
* - Wraps content with SYSTEM CONTEXT markers (or reuses existing wrapper)
* - Runs BEFORE BaseLastUserContentProvider so that the last user message
* can reuse the SYSTEM CONTEXT wrapper created here
*/
export abstract class BaseEveryUserContentProvider extends BaseProcessor {
constructor(options: ProcessorOptions = {}) {
super(options);
}
/**
* Build the content to inject for a specific user message
* Subclasses must implement this method
* @param message - The user message to build content for
* @param index - The index of the message in the messages array
* @param isLastUser - Whether this is the last user message
* @returns Object with content and contextType, or null to skip injection for this message
*/
protected abstract buildContentForMessage(
message: Message,
index: number,
isLastUser: boolean,
): { content: string; contextType: string } | null;
/**
* Get the text content from a message (handles both string and array content)
*/
private getTextContent(content: string | any[]): string {
if (typeof content === 'string') {
return content;
}
if (Array.isArray(content)) {
const lastTextPart = content.findLast((part: any) => part.type === 'text');
return lastTextPart?.text || '';
}
return '';
}
/**
* Check if the content already has a system context wrapper
*/
protected hasSystemContextWrapper(content: string | any[]): boolean {
const textContent = this.getTextContent(content);
return textContent.includes(SYSTEM_CONTEXT_START) && textContent.includes(SYSTEM_CONTEXT_END);
}
/**
* Wrap content with system context markers
*/
protected wrapWithSystemContext(content: string, contextType: string): string {
return `${SYSTEM_CONTEXT_START}
${CONTEXT_INSTRUCTION}
<${contextType}>
${content}
</${contextType}>
${SYSTEM_CONTEXT_END}`;
}
/**
* Insert content into existing system context wrapper (before the END marker)
*/
private insertIntoExistingWrapper(existingContent: string, newContextBlock: string): string {
const endMarkerIndex = existingContent.lastIndexOf(SYSTEM_CONTEXT_END);
if (endMarkerIndex === -1) {
return existingContent + '\n\n' + newContextBlock;
}
const beforeEnd = existingContent.slice(0, endMarkerIndex);
const afterEnd = existingContent.slice(endMarkerIndex);
return beforeEnd + newContextBlock + '\n' + afterEnd;
}
/**
* Create a context block without the full wrapper (for inserting into existing wrapper)
*/
protected createContextBlock(content: string, contextType: string): string {
return `<${contextType}>
${content}
</${contextType}>`;
}
/**
* Append content to a message with SYSTEM CONTEXT wrapper
*/
protected appendToMessage(message: Message, content: string, contextType: string): Message {
const currentContent = message.content;
// Handle string content
if (typeof currentContent === 'string') {
let newContent: string;
if (this.hasSystemContextWrapper(currentContent)) {
// Insert into existing wrapper
const contextBlock = this.createContextBlock(content, contextType);
newContent = this.insertIntoExistingWrapper(currentContent, contextBlock);
} else {
// Create new wrapper
newContent = currentContent + '\n\n' + this.wrapWithSystemContext(content, contextType);
}
return {
...message,
content: newContent,
};
}
// Handle array content (multimodal messages)
if (Array.isArray(currentContent)) {
const lastTextIndex = currentContent.findLastIndex((part: any) => part.type === 'text');
if (lastTextIndex !== -1) {
const newContent = [...currentContent];
const existingText = newContent[lastTextIndex].text;
let updatedText: string;
if (this.hasSystemContextWrapper(existingText)) {
// Insert into existing wrapper
const contextBlock = this.createContextBlock(content, contextType);
updatedText = this.insertIntoExistingWrapper(existingText, contextBlock);
} else {
// Create new wrapper
updatedText = existingText + '\n\n' + this.wrapWithSystemContext(content, contextType);
}
newContent[lastTextIndex] = {
...newContent[lastTextIndex],
text: updatedText,
};
return {
...message,
content: newContent,
};
} else {
// No text part found, add a new one with wrapper
return {
...message,
content: [
...currentContent,
{ text: this.wrapWithSystemContext(content, contextType), type: 'text' },
],
};
}
}
return message;
}
/**
* Find the index of the last user message
*/
protected findLastUserMessageIndex(messages: Message[]): number {
for (let i = messages.length - 1; i >= 0; i--) {
if (messages[i].role === 'user') {
return i;
}
}
return -1;
}
/**
* Process the context by injecting content to every user message
*/
protected async doProcess(context: PipelineContext): Promise<PipelineContext> {
const clonedContext = this.cloneContext(context);
const lastUserIndex = this.findLastUserMessageIndex(clonedContext.messages);
let injectCount = 0;
// Iterate through all messages
for (let i = 0; i < clonedContext.messages.length; i++) {
const message = clonedContext.messages[i];
// Only process user messages
if (message.role !== 'user') continue;
const isLastUser = i === lastUserIndex;
const result = this.buildContentForMessage(message, i, isLastUser);
if (!result) continue;
// Append to this user message with SYSTEM CONTEXT wrapper
clonedContext.messages[i] = this.appendToMessage(message, result.content, result.contextType);
injectCount++;
}
// Update metadata with injection count
if (injectCount > 0) {
clonedContext.metadata[`${this.name}InjectedCount`] = injectCount;
}
return this.markAsExecuted(clonedContext);
}
}

View File

@@ -1,13 +1,6 @@
import type { Message, PipelineContext, ProcessorOptions } from '../types';
import { BaseProcessor } from './BaseProcessor';
const SYSTEM_CONTEXT_START = '<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->';
const SYSTEM_CONTEXT_END = '<!-- END SYSTEM CONTEXT -->';
const CONTEXT_INSTRUCTION = `<context.instruction>following part contains context information injected by the system. Please follow these instructions:
1. Always prioritize handling user-visible content.
2. the context is only required when user's queries rely on it.
</context.instruction>`;
import { CONTEXT_INSTRUCTION, SYSTEM_CONTEXT_END, SYSTEM_CONTEXT_START } from './constants';
/**
* Base Provider for appending content to the last user message

View File

@@ -0,0 +1,354 @@
import { describe, expect, it } from 'vitest';
import type { Message, PipelineContext } from '../../types';
import { BaseEveryUserContentProvider } from '../BaseEveryUserContentProvider';
class TestEveryUserContentProvider extends BaseEveryUserContentProvider {
readonly name = 'TestEveryUserContentProvider';
constructor(
private contentBuilder?: (
message: Message,
index: number,
isLastUser: boolean,
) => { content: string; contextType: string } | null,
) {
super();
}
protected buildContentForMessage(
message: Message,
index: number,
isLastUser: boolean,
): { content: string; contextType: string } | null {
if (this.contentBuilder) {
return this.contentBuilder(message, index, isLastUser);
}
// Default: inject content for every user message
return {
content: `Content for message ${index}`,
contextType: 'test_context',
};
}
// Expose protected methods for testing
testHasSystemContextWrapper(content: string | any[]) {
return this.hasSystemContextWrapper(content);
}
testWrapWithSystemContext(content: string, contextType: string) {
return this.wrapWithSystemContext(content, contextType);
}
testCreateContextBlock(content: string, contextType: string) {
return this.createContextBlock(content, contextType);
}
testAppendToMessage(message: Message, content: string, contextType: string) {
return this.appendToMessage(message, content, contextType);
}
testFindLastUserMessageIndex(messages: Message[]) {
return this.findLastUserMessageIndex(messages);
}
}
describe('BaseEveryUserContentProvider', () => {
const createContext = (messages: any[] = []): PipelineContext => ({
initialState: {
messages: [],
model: 'test-model',
provider: 'test-provider',
},
isAborted: false,
messages,
metadata: {
maxTokens: 4000,
model: 'test-model',
},
});
describe('findLastUserMessageIndex', () => {
it('should find the last user message', () => {
const provider = new TestEveryUserContentProvider();
const messages = [
{ content: 'Hello', role: 'user' },
{ content: 'Hi', role: 'assistant' },
{ content: 'Question', role: 'user' },
{ content: 'Answer', role: 'assistant' },
];
expect(provider.testFindLastUserMessageIndex(messages)).toBe(2);
});
it('should return -1 when no user messages exist', () => {
const provider = new TestEveryUserContentProvider();
const messages = [{ content: 'System', role: 'system' }];
expect(provider.testFindLastUserMessageIndex(messages)).toBe(-1);
});
});
describe('hasSystemContextWrapper', () => {
it('should detect existing system context wrapper in string content', () => {
const provider = new TestEveryUserContentProvider();
const withWrapper = `Question
<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->
<test>content</test>
<!-- END SYSTEM CONTEXT -->`;
const withoutWrapper = 'Simple question';
expect(provider.testHasSystemContextWrapper(withWrapper)).toBe(true);
expect(provider.testHasSystemContextWrapper(withoutWrapper)).toBe(false);
});
it('should detect existing system context wrapper in array content', () => {
const provider = new TestEveryUserContentProvider();
const withWrapper = [
{
text: `Question
<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->
<test>content</test>
<!-- END SYSTEM CONTEXT -->`,
type: 'text',
},
];
const withoutWrapper = [{ text: 'Simple question', type: 'text' }];
expect(provider.testHasSystemContextWrapper(withWrapper)).toBe(true);
expect(provider.testHasSystemContextWrapper(withoutWrapper)).toBe(false);
});
});
describe('wrapWithSystemContext', () => {
it('should wrap content with system context markers', () => {
const provider = new TestEveryUserContentProvider();
const result = provider.testWrapWithSystemContext('Test content', 'test_type');
expect(result).toContain('<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->');
expect(result).toContain('<context.instruction>');
expect(result).toContain('<test_type>');
expect(result).toContain('Test content');
expect(result).toContain('</test_type>');
expect(result).toContain('<!-- END SYSTEM CONTEXT -->');
});
});
describe('createContextBlock', () => {
it('should create context block without wrapper', () => {
const provider = new TestEveryUserContentProvider();
const result = provider.testCreateContextBlock('Block content', 'block_type');
expect(result).toBe(`<block_type>
Block content
</block_type>`);
});
});
describe('appendToMessage', () => {
it('should append with new wrapper to string content without existing wrapper', () => {
const provider = new TestEveryUserContentProvider();
const message: Message = { content: 'Original question', role: 'user' };
const result = provider.testAppendToMessage(message, 'New content', 'new_type');
expect(result.content).toContain('Original question');
expect(result.content).toContain('<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->');
expect(result.content).toContain('<new_type>');
expect(result.content).toContain('New content');
expect(result.content).toContain('<!-- END SYSTEM CONTEXT -->');
});
it('should insert into existing wrapper in string content', () => {
const provider = new TestEveryUserContentProvider();
const message: Message = {
content: `Original question
<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->
<context.instruction>...</context.instruction>
<existing_type>
Existing content
</existing_type>
<!-- END SYSTEM CONTEXT -->`,
role: 'user',
};
const result = provider.testAppendToMessage(message, 'New content', 'new_type');
// Should have only one SYSTEM CONTEXT wrapper
const content = result.content as string;
const startCount = (content.match(/<!-- SYSTEM CONTEXT/g) || []).length;
const endCount = (content.match(/<!-- END SYSTEM CONTEXT/g) || []).length;
expect(startCount).toBe(1);
expect(endCount).toBe(1);
expect(content).toContain('<existing_type>');
expect(content).toContain('<new_type>');
expect(content).toContain('New content');
});
it('should handle array content without existing wrapper', () => {
const provider = new TestEveryUserContentProvider();
const message: Message = {
content: [
{ text: 'Original question', type: 'text' },
{ image_url: { url: 'http://example.com/img.png' }, type: 'image_url' },
],
role: 'user',
};
const result = provider.testAppendToMessage(message, 'New content', 'new_type');
expect(result.content[0].text).toContain('Original question');
expect(result.content[0].text).toContain('<!-- SYSTEM CONTEXT');
expect(result.content[0].text).toContain('<new_type>');
expect(result.content[1].type).toBe('image_url');
});
it('should add new text part when array content has no text part', () => {
const provider = new TestEveryUserContentProvider();
const message: Message = {
content: [{ image_url: { url: 'http://example.com/img.png' }, type: 'image_url' }],
role: 'user',
};
const result = provider.testAppendToMessage(message, 'New content', 'new_type');
expect(result.content).toHaveLength(2);
expect(result.content[1].type).toBe('text');
expect(result.content[1].text).toContain('<!-- SYSTEM CONTEXT');
expect(result.content[1].text).toContain('New content');
});
});
describe('process integration', () => {
it('should inject content to all user messages', async () => {
const provider = new TestEveryUserContentProvider();
const context = createContext([
{ content: 'First question', role: 'user' },
{ content: 'First answer', role: 'assistant' },
{ content: 'Second question', role: 'user' },
{ content: 'Second answer', role: 'assistant' },
{ content: 'Third question', role: 'user' },
]);
const result = await provider.process(context);
// All user messages should have content injected
expect(result.messages[0].content).toContain('First question');
expect(result.messages[0].content).toContain('<test_context>');
expect(result.messages[0].content).toContain('Content for message 0');
expect(result.messages[2].content).toContain('Second question');
expect(result.messages[2].content).toContain('<test_context>');
expect(result.messages[2].content).toContain('Content for message 2');
expect(result.messages[4].content).toContain('Third question');
expect(result.messages[4].content).toContain('<test_context>');
expect(result.messages[4].content).toContain('Content for message 4');
// Assistant messages should be unchanged
expect(result.messages[1].content).toBe('First answer');
expect(result.messages[3].content).toBe('Second answer');
});
it('should correctly identify isLastUser parameter', async () => {
const isLastUserCalls: boolean[] = [];
const provider = new TestEveryUserContentProvider((message, index, isLastUser) => {
isLastUserCalls.push(isLastUser);
return { content: `Content ${index}`, contextType: 'test' };
});
const context = createContext([
{ content: 'First', role: 'user' },
{ content: 'Answer', role: 'assistant' },
{ content: 'Second', role: 'user' },
{ content: 'Answer', role: 'assistant' },
{ content: 'Third (last)', role: 'user' },
]);
await provider.process(context);
expect(isLastUserCalls).toEqual([false, false, true]);
});
it('should skip injection when buildContentForMessage returns null', async () => {
const provider = new TestEveryUserContentProvider((message, index) => {
// Only inject for first user message
if (index === 0) {
return { content: 'First only', contextType: 'test' };
}
return null;
});
const context = createContext([
{ content: 'First question', role: 'user' },
{ content: 'Answer', role: 'assistant' },
{ content: 'Second question', role: 'user' },
]);
const result = await provider.process(context);
expect(result.messages[0].content).toContain('<test>');
expect(result.messages[0].content).toContain('First only');
expect(result.messages[2].content).toBe('Second question');
});
it('should update metadata with injection count', async () => {
const provider = new TestEveryUserContentProvider();
const context = createContext([
{ content: 'First', role: 'user' },
{ content: 'Second', role: 'user' },
{ content: 'Third', role: 'user' },
]);
const result = await provider.process(context);
expect(result.metadata.TestEveryUserContentProviderInjectedCount).toBe(3);
});
it('should not set metadata when no injections made', async () => {
const provider = new TestEveryUserContentProvider(() => null);
const context = createContext([{ content: 'Question', role: 'user' }]);
const result = await provider.process(context);
expect(result.metadata.TestEveryUserContentProviderInjectedCount).toBeUndefined();
});
});
describe('integration with BaseLastUserContentProvider', () => {
it('should allow BaseLastUserContentProvider to reuse wrapper created by BaseEveryUserContentProvider', async () => {
// First: BaseEveryUserContentProvider injects to last user message
const everyProvider = new TestEveryUserContentProvider((message, index, isLastUser) => {
if (isLastUser) {
return { content: 'Selection content', contextType: 'user_selections' };
}
return null;
});
const context = createContext([
{ content: 'First question', role: 'user' },
{ content: 'Answer', role: 'assistant' },
{ content: 'Last question', role: 'user' },
]);
const result = await everyProvider.process(context);
// The last user message should have a SYSTEM CONTEXT wrapper
const lastUserContent = result.messages[2].content as string;
expect(lastUserContent).toContain('<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->');
expect(lastUserContent).toContain('<user_selections>');
expect(lastUserContent).toContain('Selection content');
expect(lastUserContent).toContain('<!-- END SYSTEM CONTEXT -->');
// Now BaseLastUserContentProvider can detect and reuse this wrapper
expect(everyProvider.testHasSystemContextWrapper(lastUserContent)).toBe(true);
});
});
});

View File

@@ -0,0 +1,20 @@
/**
* Shared constants for context injection
*/
/**
* System context wrapper markers
* Used to wrap injected context content so models can distinguish it from user content
*/
export const SYSTEM_CONTEXT_START = '<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->';
export const SYSTEM_CONTEXT_END = '<!-- END SYSTEM CONTEXT -->';
/**
* Context instruction text
* Provides guidance to the model on how to handle injected context
*/
export const CONTEXT_INSTRUCTION = `<context.instruction>following part contains context information injected by the system. Please follow these instructions:
1. Always prioritize handling user-visible content.
2. the context is only required when user's queries rely on it.
</context.instruction>`;

View File

@@ -28,6 +28,7 @@ import {
HistorySummaryProvider,
KnowledgeInjector,
PageEditorContextInjector,
PageSelectionsInjector,
SystemRoleInjector,
ToolSystemRoleProvider,
UserMemoryInjector,
@@ -143,19 +144,19 @@ export class MessagesEngine {
return [
// =============================================
// Phase 2: System Role Injection
// Phase 1: System Role Injection
// =============================================
// 2. System role injection (agent's system role)
// 1. System role injection (agent's system role)
new SystemRoleInjector({ systemRole }),
// =============================================
// Phase 2.5: First User Message Context Injection
// Phase 2: First User Message Context Injection
// These providers inject content before the first user message
// Order matters: first executed = first in content
// =============================================
// 4. User memory injection (conditionally added, injected first)
// 2. User memory injection (conditionally added, injected first)
...(isUserMemoryEnabled ? [new UserMemoryInjector(userMemory)] : []),
// 3. Group context injection (agent identity and group info for multi-agent chat)
@@ -169,7 +170,7 @@ export class MessagesEngine {
systemPrompt: agentGroup?.systemPrompt,
}),
// 4.5. GTD Plan injection (conditionally added, after user memory, before knowledge)
// 4. GTD Plan injection (conditionally added, after user memory, before knowledge)
...(isGTDPlanEnabled ? [new GTDPlanInjector({ enabled: true, plan: gtd.plan })] : []),
// 5. Knowledge injection (full content for agent files + metadata for knowledge bases)
@@ -179,7 +180,7 @@ export class MessagesEngine {
}),
// =============================================
// Phase 2.6: Additional System Context
// Phase 3: Additional System Context
// =============================================
// 6. Agent Builder context injection (current agent config/meta for editing)
@@ -212,7 +213,10 @@ export class MessagesEngine {
historySummary,
}),
// 10. Page Editor context injection
// 12. Page Selections injection (inject user-selected text into each user message that has them)
new PageSelectionsInjector({ enabled: isPageEditorEnabled }),
// 10. Page Editor context injection (inject current page content to last user message)
new PageEditorContextInjector({
enabled: isPageEditorEnabled,
// Use direct pageContentContext if provided (server-side), otherwise build from initialContext + stepContext (frontend)
@@ -232,37 +236,37 @@ export class MessagesEngine {
: undefined,
}),
// 10.5. GTD Todo injection (conditionally added, at end of last user message)
// 11. GTD Todo injection (conditionally added, at end of last user message)
...(isGTDTodoEnabled ? [new GTDTodoInjector({ enabled: true, todos: gtd.todos })] : []),
// =============================================
// Phase 3: Message Transformation
// Phase 4: Message Transformation
// =============================================
// 11. Input template processing
// 13. Input template processing
new InputTemplateProcessor({ inputTemplate }),
// 11. Placeholder variables processing
// 14. Placeholder variables processing
new PlaceholderVariablesProcessor({
variableGenerators: variableGenerators || {},
}),
// 12. AgentCouncil message flatten (convert role=agentCouncil to standard assistant + tool messages)
// 15. AgentCouncil message flatten (convert role=agentCouncil to standard assistant + tool messages)
new AgentCouncilFlattenProcessor(),
// 13. Group message flatten (convert role=assistantGroup to standard assistant + tool messages)
// 16. Group message flatten (convert role=assistantGroup to standard assistant + tool messages)
new GroupMessageFlattenProcessor(),
// 14. Tasks message flatten (convert role=tasks to individual task messages)
// 17. Tasks message flatten (convert role=tasks to individual task messages)
new TasksFlattenProcessor(),
// 15. Task message processing (convert role=task to assistant with instruction + content)
// 18. Task message processing (convert role=task to assistant with instruction + content)
new TaskMessageProcessor(),
// 15. Supervisor role restore (convert role=supervisor back to role=assistant for model)
// 19. Supervisor role restore (convert role=supervisor back to role=assistant for model)
new SupervisorRoleRestoreProcessor(),
// 15.5. Group orchestration filter (remove supervisor's orchestration messages like broadcast/speak)
// 20. Group orchestration filter (remove supervisor's orchestration messages like broadcast/speak)
// This must be BEFORE GroupRoleTransformProcessor so we filter based on original agentId/tools
...(isAgentGroupEnabled && agentGroup.agentMap && agentGroup.currentAgentId
? [
@@ -277,7 +281,7 @@ export class MessagesEngine {
]
: []),
// 16. Group role transform (convert other agents' messages to user role with speaker tags)
// 21. Group role transform (convert other agents' messages to user role with speaker tags)
// This must be BEFORE ToolCallProcessor so other agents' tool messages are converted first
...(isAgentGroupEnabled && agentGroup.currentAgentId
? [
@@ -289,10 +293,10 @@ export class MessagesEngine {
: []),
// =============================================
// Phase 4: Content Processing
// Phase 5: Content Processing
// =============================================
// 17. Message content processing (image encoding, etc.)
// 22. Message content processing (image encoding, etc.)
new MessageContentProcessor({
fileContext: fileContext || { enabled: true, includeFileUrl: true },
isCanUseVideo: capabilities?.isCanUseVideo || (() => false),
@@ -301,7 +305,7 @@ export class MessagesEngine {
provider,
}),
// 18. Tool call processing
// 23. Tool call processing
new ToolCallProcessor({
genToolCallingName: this.toolNameResolver.generate.bind(this.toolNameResolver),
isCanUseFC: capabilities?.isCanUseFC || (() => true),
@@ -309,10 +313,10 @@ export class MessagesEngine {
provider,
}),
// 19. Tool message reordering
// 24. Tool message reordering
new ToolMessageReorder(),
// 20. Message cleanup (final step, keep only necessary fields)
// 25. Message cleanup (final step, keep only necessary fields)
new MessageCleanupProcessor(),
];
}

View File

@@ -407,4 +407,368 @@ describe('MessagesEngine', () => {
expect(userMessage?.content).toBe('Please respond to: user input');
});
});
describe('Page Editor context', () => {
it('should inject page content to the last user message when pageContentContext is provided', async () => {
const messages: UIChatMessage[] = [
{
content: 'First question',
createdAt: Date.now(),
id: 'msg-1',
role: 'user',
updatedAt: Date.now(),
} as UIChatMessage,
{
content: 'Answer',
createdAt: Date.now(),
id: 'msg-2',
role: 'assistant',
updatedAt: Date.now(),
} as UIChatMessage,
{
content: 'Second question about the page',
createdAt: Date.now(),
id: 'msg-3',
role: 'user',
updatedAt: Date.now(),
} as UIChatMessage,
];
const params = createBasicParams({
messages,
pageContentContext: {
markdown: '# Document Title\n\nDocument content here.',
metadata: {
charCount: 40,
lineCount: 3,
title: 'Test Document',
},
},
});
const engine = new MessagesEngine(params);
const result = await engine.process();
expect(result.messages).toEqual([
{ content: 'First question', role: 'user' },
{ content: 'Answer', role: 'assistant' },
{
content: `Second question about the page
<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->
<context.instruction>following part contains context information injected by the system. Please follow these instructions:
1. Always prioritize handling user-visible content.
2. the context is only required when user's queries rely on it.
</context.instruction>
<current_page_context>
<current_page title="Test Document">
<markdown chars="40" lines="3">
# Document Title
Document content here.
</markdown>
</current_page>
</current_page_context>
<!-- END SYSTEM CONTEXT -->`,
role: 'user',
},
]);
expect(result.metadata.pageEditorContextInjected).toBe(true);
});
it('should not inject page content when not enabled', async () => {
const messages: UIChatMessage[] = [
{
content: 'Question',
createdAt: Date.now(),
id: 'msg-1',
role: 'user',
updatedAt: Date.now(),
} as UIChatMessage,
];
const params = createBasicParams({ messages });
const engine = new MessagesEngine(params);
const result = await engine.process();
expect(result.messages).toEqual([{ content: 'Question', role: 'user' }]);
expect(result.metadata.pageEditorContextInjected).toBeUndefined();
});
});
describe('Page Selections', () => {
it('should inject page selections to each user message that has them', async () => {
const messages: UIChatMessage[] = [
{
content: 'First question with selection',
createdAt: Date.now(),
id: 'msg-1',
metadata: {
pageSelections: [
{
content: 'Selected paragraph 1',
id: 'sel-1',
pageId: 'page-1',
xml: '<p>Selected paragraph 1</p>',
},
],
},
role: 'user',
updatedAt: Date.now(),
} as UIChatMessage,
{
content: 'Answer to first',
createdAt: Date.now(),
id: 'msg-2',
role: 'assistant',
updatedAt: Date.now(),
} as UIChatMessage,
{
content: 'Second question with different selection',
createdAt: Date.now(),
id: 'msg-3',
metadata: {
pageSelections: [
{
content: 'Selected paragraph 2',
id: 'sel-2',
pageId: 'page-1',
xml: '<p>Selected paragraph 2</p>',
},
],
},
role: 'user',
updatedAt: Date.now(),
} as UIChatMessage,
];
const params = createBasicParams({
messages,
pageContentContext: {
markdown: '# Doc',
metadata: { title: 'Doc' },
},
});
const engine = new MessagesEngine(params);
const result = await engine.process();
expect(result.messages).toEqual([
{
content: `First question with selection
<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->
<context.instruction>following part contains context information injected by the system. Please follow these instructions:
1. Always prioritize handling user-visible content.
2. the context is only required when user's queries rely on it.
</context.instruction>
<user_page_selections>
<user_selections count="1">
<selection >
<p>Selected paragraph 1</p>
</selection>
</user_selections>
</user_page_selections>
<!-- END SYSTEM CONTEXT -->`,
role: 'user',
},
{ content: 'Answer to first', role: 'assistant' },
{
content: `Second question with different selection
<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->
<context.instruction>following part contains context information injected by the system. Please follow these instructions:
1. Always prioritize handling user-visible content.
2. the context is only required when user's queries rely on it.
</context.instruction>
<user_page_selections>
<user_selections count="1">
<selection >
<p>Selected paragraph 2</p>
</selection>
</user_selections>
</user_page_selections>
<current_page_context>
<current_page title="Doc">
<markdown chars="5" lines="1">
# Doc
</markdown>
</current_page>
</current_page_context>
<!-- END SYSTEM CONTEXT -->`,
role: 'user',
},
]);
});
it('should skip user messages without pageSelections', async () => {
const messages: UIChatMessage[] = [
{
content: 'No selection here',
createdAt: Date.now(),
id: 'msg-1',
role: 'user',
updatedAt: Date.now(),
} as UIChatMessage,
{
content: 'Answer',
createdAt: Date.now(),
id: 'msg-2',
role: 'assistant',
updatedAt: Date.now(),
} as UIChatMessage,
{
content: 'With selection',
createdAt: Date.now(),
id: 'msg-3',
metadata: {
pageSelections: [
{
content: 'Selected text',
id: 'sel-1',
pageId: 'page-1',
xml: '<span>Selected text</span>',
},
],
},
role: 'user',
updatedAt: Date.now(),
} as UIChatMessage,
];
const params = createBasicParams({
messages,
pageContentContext: {
markdown: '# Doc',
metadata: { title: 'Doc' },
},
});
const engine = new MessagesEngine(params);
const result = await engine.process();
expect(result.messages).toEqual([
{ content: 'No selection here', role: 'user' },
{ content: 'Answer', role: 'assistant' },
{
content: `With selection
<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->
<context.instruction>following part contains context information injected by the system. Please follow these instructions:
1. Always prioritize handling user-visible content.
2. the context is only required when user's queries rely on it.
</context.instruction>
<user_page_selections>
<user_selections count="1">
<selection >
<span>Selected text</span>
</selection>
</user_selections>
</user_page_selections>
<current_page_context>
<current_page title="Doc">
<markdown chars="5" lines="1">
# Doc
</markdown>
</current_page>
</current_page_context>
<!-- END SYSTEM CONTEXT -->`,
role: 'user',
},
]);
});
it('should have only one SYSTEM CONTEXT wrapper when both selections and page content are injected', async () => {
const messages: UIChatMessage[] = [
{
content: 'Question about selection',
createdAt: Date.now(),
id: 'msg-1',
metadata: {
pageSelections: [
{
content: 'Selected text',
id: 'sel-1',
pageId: 'page-1',
xml: '<p>Selected text</p>',
},
],
},
role: 'user',
updatedAt: Date.now(),
} as UIChatMessage,
];
const params = createBasicParams({
messages,
pageContentContext: {
markdown: '# Full Document',
metadata: { title: 'Full Doc' },
},
});
const engine = new MessagesEngine(params);
const result = await engine.process();
expect(result.messages).toEqual([
{
content: `Question about selection
<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->
<context.instruction>following part contains context information injected by the system. Please follow these instructions:
1. Always prioritize handling user-visible content.
2. the context is only required when user's queries rely on it.
</context.instruction>
<user_page_selections>
<user_selections count="1">
<selection >
<p>Selected text</p>
</selection>
</user_selections>
</user_page_selections>
<current_page_context>
<current_page title="Full Doc">
<markdown chars="15" lines="1">
# Full Document
</markdown>
</current_page>
</current_page_context>
<!-- END SYSTEM CONTEXT -->`,
role: 'user',
},
]);
});
it('should not inject selections when page editor is not enabled', async () => {
const messages: UIChatMessage[] = [
{
content: 'Question',
createdAt: Date.now(),
id: 'msg-1',
metadata: {
pageSelections: [
{ content: 'Selected', id: 'sel-1', pageId: 'page-1', xml: '<p>Selected</p>' },
],
},
role: 'user',
updatedAt: Date.now(),
} as UIChatMessage,
];
// No pageContentContext or initialContext.pageEditor means not enabled
const params = createBasicParams({ messages });
const engine = new MessagesEngine(params);
const result = await engine.process();
expect(result.messages).toEqual([{ content: 'Question', role: 'user' }]);
});
});
});

View File

@@ -20,6 +20,9 @@ export interface PageEditorContextInjectorConfig {
* Page Editor Context Injector
* Responsible for injecting current page context at the end of the last user message
* This ensures the model receives the most up-to-date page/document state
*
* Note: Page selections (user-selected text regions) are handled separately by
* PageSelectionsInjector, which injects selections into each user message that has them
*/
export class PageEditorContextInjector extends BaseLastUserContentProvider {
readonly name = 'PageEditorContextInjector';
@@ -37,20 +40,11 @@ export class PageEditorContextInjector extends BaseLastUserContentProvider {
const clonedContext = this.cloneContext(context);
// Skip if Page Editor is not enabled or no page content context
if (!this.config.enabled || !this.config.pageContentContext) {
log('Page Editor not enabled or no pageContentContext, skipping injection');
return this.markAsExecuted(clonedContext);
}
// Check if we have page content to inject
const hasPageContent = this.config.enabled && this.config.pageContentContext;
// Format page content context
const formattedContent = formatPageContentContext(this.config.pageContentContext);
log('Formatted content length:', formattedContent.length);
// Skip if no content to inject
if (!formattedContent) {
log('No content to inject after formatting');
if (!hasPageContent) {
log('No pageContentContext, skipping injection');
return this.markAsExecuted(clonedContext);
}
@@ -64,6 +58,16 @@ export class PageEditorContextInjector extends BaseLastUserContentProvider {
return this.markAsExecuted(clonedContext);
}
// Format page content
const formattedContent = formatPageContentContext(this.config.pageContentContext!);
if (!formattedContent) {
log('No content to inject after formatting');
return this.markAsExecuted(clonedContext);
}
log('Page content formatted, length:', formattedContent.length);
// Check if system context wrapper already exists
// If yes, only insert context block; if no, use full wrapper
const hasExistingWrapper = this.hasExistingSystemContext(clonedContext);

View File

@@ -0,0 +1,65 @@
import { formatPageSelections } from '@lobechat/prompts';
import type { PageSelection } from '@lobechat/types';
import debug from 'debug';
import { BaseEveryUserContentProvider } from '../base/BaseEveryUserContentProvider';
import type { Message, ProcessorOptions } from '../types';
const log = debug('context-engine:provider:PageSelectionsInjector');
export interface PageSelectionsInjectorConfig {
/** Whether Page Selections injection is enabled */
enabled?: boolean;
}
/**
* Page Selections Injector
* Responsible for injecting page selections into each user message that has them
* Unlike PageEditorContextInjector which only injects to the last user message,
* this processor handles selections attached to any user message in the conversation
*
* This injector runs BEFORE PageEditorContextInjector so that:
* - Each user message with selections gets a SYSTEM CONTEXT wrapper
* - PageEditorContextInjector can then reuse the wrapper for the last user message
*/
export class PageSelectionsInjector extends BaseEveryUserContentProvider {
readonly name = 'PageSelectionsInjector';
constructor(
private config: PageSelectionsInjectorConfig = {},
options: ProcessorOptions = {},
) {
super(options);
}
protected buildContentForMessage(
message: Message,
index: number,
): { content: string; contextType: string } | null {
// Skip if not enabled
if (!this.config.enabled) {
return null;
}
// Check if message has pageSelections in metadata
const pageSelections = message.metadata?.pageSelections as PageSelection[] | undefined;
if (!pageSelections || pageSelections.length === 0) {
return null;
}
// Format the selections
const formattedSelections = formatPageSelections(pageSelections);
if (!formattedSelections) {
return null;
}
log(`Building content for message at index ${index} with ${pageSelections.length} selections`);
return {
content: formattedSelections,
contextType: 'user_page_selections',
};
}
}

View File

@@ -0,0 +1,333 @@
import { describe, expect, it } from 'vitest';
import type { PipelineContext } from '../../types';
import { PageSelectionsInjector } from '../PageSelectionsInjector';
describe('PageSelectionsInjector', () => {
const createContext = (messages: any[] = []): PipelineContext => ({
initialState: {
messages: [],
model: 'test-model',
provider: 'test-provider',
},
isAborted: false,
messages,
metadata: {
maxTokens: 4000,
model: 'test-model',
},
});
const createPageSelection = (id: string, xmlContent: string, pageId = 'page-1') => ({
content: xmlContent, // preview content
id,
pageId,
xml: xmlContent, // actual content used by formatPageSelections
});
describe('enabled/disabled', () => {
it('should skip injection when disabled', async () => {
const injector = new PageSelectionsInjector({ enabled: false });
const context = createContext([
{
content: 'Question',
metadata: {
pageSelections: [createPageSelection('sel-1', 'Selected text')],
},
role: 'user',
},
]);
const result = await injector.process(context);
expect(result.messages[0].content).toBe('Question');
expect(result.metadata.PageSelectionsInjectorInjectedCount).toBeUndefined();
});
it('should inject when enabled', async () => {
const injector = new PageSelectionsInjector({ enabled: true });
const context = createContext([
{
content: 'Question',
metadata: {
pageSelections: [createPageSelection('sel-1', 'Selected text')],
},
role: 'user',
},
]);
const result = await injector.process(context);
expect(result.messages[0].content).toContain('Question');
expect(result.messages[0].content).toContain('<user_page_selections>');
expect(result.messages[0].content).toContain('Selected text');
});
});
describe('injection to every user message', () => {
it('should inject selections to each user message that has them', async () => {
const injector = new PageSelectionsInjector({ enabled: true });
const context = createContext([
{
content: 'First question',
metadata: {
pageSelections: [createPageSelection('sel-1', 'First selection')],
},
role: 'user',
},
{ content: 'First answer', role: 'assistant' },
{
content: 'Second question',
metadata: {
pageSelections: [createPageSelection('sel-2', 'Second selection')],
},
role: 'user',
},
{ content: 'Second answer', role: 'assistant' },
{
content: 'Third question without selection',
role: 'user',
},
]);
const result = await injector.process(context);
// First user message should have first selection
expect(result.messages[0].content).toContain('First question');
expect(result.messages[0].content).toContain('First selection');
expect(result.messages[0].content).toContain('<user_page_selections>');
// Second user message should have second selection
expect(result.messages[2].content).toContain('Second question');
expect(result.messages[2].content).toContain('Second selection');
expect(result.messages[2].content).toContain('<user_page_selections>');
// Third user message should NOT have injection (no selections)
expect(result.messages[4].content).toBe('Third question without selection');
// Assistant messages should be unchanged
expect(result.messages[1].content).toBe('First answer');
expect(result.messages[3].content).toBe('Second answer');
// Metadata should show 2 injections
expect(result.metadata.PageSelectionsInjectorInjectedCount).toBe(2);
});
it('should skip user messages without pageSelections', async () => {
const injector = new PageSelectionsInjector({ enabled: true });
const context = createContext([
{ content: 'No selections here', role: 'user' },
{ content: 'Answer', role: 'assistant' },
{
content: 'With selections',
metadata: {
pageSelections: [createPageSelection('sel-1', 'Some text')],
},
role: 'user',
},
]);
const result = await injector.process(context);
expect(result.messages[0].content).toBe('No selections here');
expect(result.messages[2].content).toContain('With selections');
expect(result.messages[2].content).toContain('Some text');
});
it('should skip user messages with empty pageSelections array', async () => {
const injector = new PageSelectionsInjector({ enabled: true });
const context = createContext([
{
content: 'Empty selections',
metadata: { pageSelections: [] },
role: 'user',
},
]);
const result = await injector.process(context);
expect(result.messages[0].content).toBe('Empty selections');
});
});
describe('SYSTEM CONTEXT wrapper', () => {
it('should wrap selection content with SYSTEM CONTEXT markers', async () => {
const injector = new PageSelectionsInjector({ enabled: true });
const context = createContext([
{
content: 'Question',
metadata: {
pageSelections: [createPageSelection('sel-1', 'Selected text')],
},
role: 'user',
},
]);
const result = await injector.process(context);
const content = result.messages[0].content as string;
expect(content).toContain('<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->');
expect(content).toContain('<context.instruction>');
expect(content).toContain('<user_page_selections>');
expect(content).toContain('</user_page_selections>');
expect(content).toContain('<!-- END SYSTEM CONTEXT -->');
});
it('should have only one SYSTEM CONTEXT wrapper per message even with multiple selections', async () => {
const injector = new PageSelectionsInjector({ enabled: true });
const context = createContext([
{
content: 'Question',
metadata: {
pageSelections: [
createPageSelection('sel-1', 'First selection'),
createPageSelection('sel-2', 'Second selection'),
],
},
role: 'user',
},
]);
const result = await injector.process(context);
const content = result.messages[0].content as string;
const startCount = (content.match(/<!-- SYSTEM CONTEXT/g) || []).length;
const endCount = (content.match(/<!-- END SYSTEM CONTEXT/g) || []).length;
expect(startCount).toBe(1);
expect(endCount).toBe(1);
});
it('should create separate SYSTEM CONTEXT wrappers for each user message', async () => {
const injector = new PageSelectionsInjector({ enabled: true });
const context = createContext([
{
content: 'First question',
metadata: {
pageSelections: [createPageSelection('sel-1', 'First selection')],
},
role: 'user',
},
{ content: 'Answer', role: 'assistant' },
{
content: 'Second question',
metadata: {
pageSelections: [createPageSelection('sel-2', 'Second selection')],
},
role: 'user',
},
]);
const result = await injector.process(context);
// Each user message should have its own SYSTEM CONTEXT wrapper
const firstContent = result.messages[0].content as string;
const secondContent = result.messages[2].content as string;
expect(firstContent).toContain('<!-- SYSTEM CONTEXT');
expect(firstContent).toContain('First selection');
expect(secondContent).toContain('<!-- SYSTEM CONTEXT');
expect(secondContent).toContain('Second selection');
});
});
describe('multimodal messages', () => {
it('should handle array content with text parts', async () => {
const injector = new PageSelectionsInjector({ enabled: true });
const context = createContext([
{
content: [
{ text: 'Question with image', type: 'text' },
{ image_url: { url: 'http://example.com/img.png' }, type: 'image_url' },
],
metadata: {
pageSelections: [createPageSelection('sel-1', 'Selected text')],
},
role: 'user',
},
]);
const result = await injector.process(context);
expect(result.messages[0].content[0].text).toContain('Question with image');
expect(result.messages[0].content[0].text).toContain('Selected text');
expect(result.messages[0].content[0].text).toContain('<user_page_selections>');
expect(result.messages[0].content[1]).toEqual({
image_url: { url: 'http://example.com/img.png' },
type: 'image_url',
});
});
});
describe('integration with PageEditorContextInjector', () => {
it('should create wrapper that PageEditorContextInjector can reuse', async () => {
const injector = new PageSelectionsInjector({ enabled: true });
const context = createContext([
{
content: 'Question about the page',
metadata: {
pageSelections: [createPageSelection('sel-1', 'Selected paragraph')],
},
role: 'user',
},
]);
const result = await injector.process(context);
const content = result.messages[0].content as string;
// Verify the wrapper structure is correct for reuse
expect(content).toContain('<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->');
expect(content).toContain('<context.instruction>');
expect(content).toContain('<!-- END SYSTEM CONTEXT -->');
// Verify the content is in the right position (between instruction and end marker)
const instructionIndex = content.indexOf('</context.instruction>');
const selectionsIndex = content.indexOf('<user_page_selections>');
const endIndex = content.indexOf('<!-- END SYSTEM CONTEXT -->');
expect(instructionIndex).toBeLessThan(selectionsIndex);
expect(selectionsIndex).toBeLessThan(endIndex);
});
});
describe('metadata', () => {
it('should set metadata when injections are made', async () => {
const injector = new PageSelectionsInjector({ enabled: true });
const context = createContext([
{
content: 'Question',
metadata: {
pageSelections: [createPageSelection('sel-1', 'Text')],
},
role: 'user',
},
]);
const result = await injector.process(context);
expect(result.metadata.PageSelectionsInjectorInjectedCount).toBe(1);
});
it('should not set metadata when no injections are made', async () => {
const injector = new PageSelectionsInjector({ enabled: true });
const context = createContext([{ content: 'No selections', role: 'user' }]);
const result = await injector.process(context);
expect(result.metadata.PageSelectionsInjectorInjectedCount).toBeUndefined();
});
});
});

View File

@@ -7,6 +7,7 @@ export { GTDTodoInjector } from './GTDTodoInjector';
export { HistorySummaryProvider } from './HistorySummary';
export { KnowledgeInjector } from './KnowledgeInjector';
export { PageEditorContextInjector } from './PageEditorContextInjector';
export { PageSelectionsInjector } from './PageSelectionsInjector';
export { SystemRoleInjector } from './SystemRoleInjector';
export { ToolSystemRoleProvider } from './ToolSystemRole';
export { UserMemoryInjector } from './UserMemoryInjector';
@@ -27,11 +28,12 @@ export type {
GroupContextInjectorConfig,
GroupMemberInfo as GroupContextMemberInfo,
} from './GroupContextInjector';
export type { HistorySummaryConfig } from './HistorySummary';
export type { GTDPlan, GTDPlanInjectorConfig } from './GTDPlanInjector';
export type { GTDTodoInjectorConfig, GTDTodoItem, GTDTodoList } from './GTDTodoInjector';
export type { HistorySummaryConfig } from './HistorySummary';
export type { KnowledgeInjectorConfig } from './KnowledgeInjector';
export type { PageEditorContextInjectorConfig } from './PageEditorContextInjector';
export type { PageSelectionsInjectorConfig } from './PageSelectionsInjector';
export type { SystemRoleInjectorConfig } from './SystemRoleInjector';
export type { ToolSystemRoleConfig } from './ToolSystemRole';
export type { UserMemoryInjectorConfig } from './UserMemoryInjector';

View File

@@ -1 +1,2 @@
export * from './pageContentContext';
export * from './pageSelectionContext';

View File

@@ -0,0 +1,28 @@
import type { PageSelection } from '@lobechat/types';
/**
* Format page selections into a system prompt context
* Each selection is wrapped in a <selection> tag with metadata
*/
export const formatPageSelections = (selections: PageSelection[]): string => {
if (!selections || selections.length === 0) {
return '';
}
const formattedSelections = selections
.map((sel) => {
const lineInfo =
sel.startLine !== undefined
? ` lines="${sel.startLine}-${sel.endLine ?? sel.startLine}"`
: '';
return `<selection ${lineInfo}>
${sel.xml}
</selection>`;
})
.join('\n');
return `<user_selections count="${selections.length}">
${formattedSelections}
</user_selections>`;
};

View File

@@ -1,6 +1,7 @@
import { z } from 'zod';
import { UIChatMessage } from './message';
import { PageSelection, PageSelectionSchema } from './message/ui/params';
import { OpenAIChatMessage } from './openai/chat';
import { LobeUniformTool, LobeUniformToolSchema } from './tool';
import { ChatTopic } from './topic';
@@ -10,6 +11,8 @@ export interface SendNewMessage {
content: string;
// if message has attached with files, then add files to message and the agent
files?: string[];
/** Page selections attached to this message (for Ask AI functionality) */
pageSelections?: PageSelection[];
parentId?: string;
}
@@ -83,6 +86,7 @@ export const AiSendMessageServerSchema = z.object({
newUserMessage: z.object({
content: z.string(),
files: z.array(z.string()).optional(),
pageSelections: z.array(PageSelectionSchema).optional(),
parentId: z.string().optional(),
}),
sessionId: z.string().optional(),

View File

@@ -2,5 +2,6 @@ export * from './base';
export * from './image';
export * from './messageGroup';
export * from './metadata';
export * from './pageSelection';
export * from './tools';
export * from './translate';

View File

@@ -1,6 +1,8 @@
/* eslint-disable sort-keys-fix/sort-keys-fix , typescript-sort-keys/interface */
import { z } from 'zod';
import { PageSelection, PageSelectionSchema } from './pageSelection';
export interface ModelTokensUsage {
// Input tokens breakdown
/**
@@ -80,6 +82,7 @@ export const MessageMetadataSchema = ModelUsageSchema.merge(ModelPerformanceSche
inspectExpanded: z.boolean().optional(),
isMultimodal: z.boolean().optional(),
isSupervisor: z.boolean().optional(),
pageSelections: z.array(PageSelectionSchema).optional(),
});
export interface ModelUsage extends ModelTokensUsage {
@@ -147,4 +150,9 @@ export interface MessageMetadata extends ModelUsage, ModelPerformance {
*/
instruction?: string;
taskTitle?: string;
/**
* Page selections attached to user message
* Used for Ask AI functionality to persist selection context
*/
pageSelections?: PageSelection[];
}

View File

@@ -0,0 +1,36 @@
/* eslint-disable sort-keys-fix/sort-keys-fix , typescript-sort-keys/interface */
import { z } from 'zod';
/**
* Page selection represents a user-selected text region in a page/document.
* Used for Ask AI functionality to persist selection context with user messages.
*/
export interface PageSelection {
/** Selection unique identifier */
id: string;
anchor?: {
startNodeId: string;
endNodeId: string;
startOffset: number;
endOffset: number;
};
/** Selected content (plain text or markdown) */
content: string;
/** XML structure of the selected content (for positioning edits) */
xml?: string;
/** Page ID the selection belongs to */
pageId: string;
/** Start line number */
startLine?: number;
/** End line number */
endLine?: number;
}
export const PageSelectionSchema = z.object({
id: z.string(),
content: z.string(),
xml: z.string().optional(),
pageId: z.string(),
startLine: z.number().optional(),
endLine: z.number().optional(),
});

View File

@@ -5,6 +5,8 @@ import { ConversationContext } from '../../conversation';
import { UploadFileItem } from '../../files';
import { MessageSemanticSearchChunk } from '../../rag';
import { ChatMessageError, ChatMessageErrorSchema } from '../common/base';
// Import for local use
import type { PageSelection } from '../common/pageSelection';
import { ChatPluginPayload, ToolInterventionSchema } from '../common/tools';
import { UIChatMessage } from './chat';
import { SemanticSearchChunkSchema } from './rag';
@@ -78,6 +80,10 @@ export interface ChatContextContent {
*/
format?: 'xml' | 'text' | 'markdown';
id: string;
/**
* Page ID the selection belongs to (for page editor selections)
*/
pageId?: string;
/**
* Optional short preview for displaying in UI.
*/
@@ -86,6 +92,10 @@ export interface ChatContextContent {
type: 'text';
}
// Re-export PageSelection from common for backwards compatibility
export type { PageSelection } from '../common/pageSelection';
export { PageSelectionSchema } from '../common/pageSelection';
export interface SendMessageParams {
/**
* create a thread
@@ -119,8 +129,14 @@ export interface SendMessageParams {
parentId?: string;
/**
* Additional contextual snippets (e.g., text selections) attached to the request.
* @deprecated Use pageSelections instead for page editor selections
*/
contexts?: ChatContextContent[];
/**
* Page selections attached to the message (for Ask AI functionality)
* These will be persisted to the database and injected via context-engine
*/
pageSelections?: PageSelection[];
}
export interface SendGroupMessageParams {

View File

@@ -30,7 +30,7 @@ const ContextList = memo(() => {
const rawSelectionList = useFileStore(fileChatSelectors.chatContextSelections);
const showSelectionList = useFileStore(fileChatSelectors.chatContextSelectionHasItem);
const clearChatContextSelections = useFileStore((s) => s.clearChatContextSelections);
console.log(rawSelectionList);
// Clear selections only when agentId changes (not on initial mount)
useEffect(() => {
if (prevAgentIdRef.current !== undefined && prevAgentIdRef.current !== agentId) {

View File

@@ -110,8 +110,16 @@ const ChatInput = memo<ChatInputProps>(
fileStore.clearChatUploadFileList();
fileStore.clearChatContextSelections();
// Convert ChatContextContent to PageSelection for persistence
const pageSelections = currentContextList.map((ctx) => ({
content: ctx.preview || '',
id: ctx.id,
pageId: ctx.pageId || '',
xml: ctx.content,
}));
// Fire and forget - send with captured message
await sendMessage({ contexts: currentContextList, files: currentFileList, message });
await sendMessage({ files: currentFileList, message, pageSelections });
},
[isAIGenerating, sendMessage],
);

View File

@@ -7,13 +7,19 @@ import { type UIChatMessage } from '@/types/index';
import { useMarkdown } from '../useMarkdown';
import FileListViewer from './FileListViewer';
import ImageFileListViewer from './ImageFileListViewer';
import PageSelections from './PageSelections';
import VideoFileListViewer from './VideoFileListViewer';
const UserMessageContent = memo<UIChatMessage>(
({ id, content, imageList, videoList, fileList }) => {
({ id, content, imageList, videoList, fileList, metadata }) => {
const markdownProps = useMarkdown(id);
const pageSelections = metadata?.pageSelections;
return (
<Flexbox gap={8} id={id}>
{pageSelections && pageSelections.length > 0 && (
<PageSelections selections={pageSelections} />
)}
{content && <MarkdownMessage {...markdownProps}>{content}</MarkdownMessage>}
{imageList && imageList?.length > 0 && <ImageFileListViewer items={imageList} />}
{videoList && videoList?.length > 0 && <VideoFileListViewer items={videoList} />}

View File

@@ -0,0 +1,62 @@
import { Flexbox } from '@lobehub/ui';
import { createStaticStyles } from 'antd-style';
import { memo } from 'react';
import type { PageSelection } from '@/types/index';
const styles = createStaticStyles(({ css, cssVar }) => ({
container: css`
cursor: pointer;
position: relative;
border-radius: 8px;
:hover {
background: ${cssVar.colorFillQuaternary};
}
`,
content: css`
overflow: hidden;
display: -webkit-box;
-webkit-box-orient: vertical;
-webkit-line-clamp: 3;
font-size: 12px;
line-height: 1.5;
color: ${cssVar.colorTextSecondary};
white-space: pre-wrap;
`,
quote: css`
inset-block-start: 2px;
inset-inline-start: 0;
font-family: Georgia, serif;
font-size: 28px;
line-height: 1;
color: ${cssVar.colorTextQuaternary};
`,
wrapper: css``,
}));
interface PageSelectionsProps {
selections: PageSelection[];
}
const PageSelections = memo<PageSelectionsProps>(({ selections }) => {
if (!selections || selections.length === 0) return null;
return (
<Flexbox gap={8}>
{selections.map((selection) => (
<Flexbox className={styles.container} key={selection.id}>
<Flexbox className={styles.wrapper} gap={4} horizontal padding={4}>
{/* eslint-disable-next-line react/no-unescaped-entities */}
<span className={styles.quote}>"</span>
<div className={styles.content}>{selection.content}</div>
</Flexbox>
</Flexbox>
))}
</Flexbox>
);
});
export default PageSelections;

View File

@@ -12,6 +12,8 @@ import { useTranslation } from 'react-i18next';
import { useFileStore } from '@/store/file';
import { useGlobalStore } from '@/store/global';
import { usePageEditorStore } from '../store';
const styles = createStaticStyles(({ css }) => ({
askCopilot: css`
border-radius: 6px;
@@ -26,6 +28,7 @@ const styles = createStaticStyles(({ css }) => ({
export const useAskCopilotItem = (editor: IEditor | undefined): ChatInputActionsProps['items'] => {
const { t } = useTranslation('common');
const addSelectionContext = useFileStore((s) => s.addChatContextSelection);
const pageId = usePageEditorStore((s) => s.documentId);
return useMemo(() => {
if (!editor) return [];
@@ -60,6 +63,7 @@ export const useAskCopilotItem = (editor: IEditor | undefined): ChatInputActions
content,
format,
id: `selection-${nanoid(6)}`,
pageId,
preview,
title: 'Selection',
type: 'text',
@@ -95,5 +99,5 @@ export const useAskCopilotItem = (editor: IEditor | undefined): ChatInputActions
onClick: () => {},
},
];
}, [addSelectionContext, editor, t]);
}, [addSelectionContext, editor, pageId, t]);
};

View File

@@ -234,8 +234,9 @@ export default {
'operation.sendMessage': 'Sending message',
'owner': 'Group owner',
'pageCopilot.title': 'Page Agent',
'pageCopilot.welcome':
'**Clearer, sharper writing**\n\nDraft, rewrite, or polish—tell me your intent and Ill refine the rest.',
'pageCopilot.welcome': `**Clearer, sharper writing**\n\nDraft, rewrite, or polish—tell me your intent and I'll refine the rest.`,
'pageSelection.lines': 'Lines {{start}}-{{end}}',
'pageSelection.reference': 'Selected Text',
'pin': 'Pin',
'pinOff': 'Unpin',
'prompts.summaryExpert':

View File

@@ -123,11 +123,18 @@ export const aiChatRouter = router({
// create user message
log('creating user message with content length: %d', input.newUserMessage.content.length);
// Build user message metadata with pageSelections if present
const userMessageMetadata = input.newUserMessage.pageSelections?.length
? { pageSelections: input.newUserMessage.pageSelections }
: undefined;
const userMessageItem = await ctx.messageModel.create({
agentId: input.agentId,
content: input.newUserMessage.content,
files: input.newUserMessage.files,
groupId: input.groupId,
metadata: userMessageMetadata,
parentId: input.newUserMessage.parentId,
role: 'user',
sessionId,

View File

@@ -77,11 +77,11 @@ export const conversationLifecycle: StateCreator<
sendMessage: async ({
message,
files,
contexts,
onlyAddUserMessage,
context,
messages: inputMessages,
parentId: inputParentId,
pageSelections,
}) => {
const { internal_execAgentRuntime, mainInputEditor } = get();
@@ -186,6 +186,8 @@ export const conversationLifecycle: StateCreator<
threadId: operationContext.threadId ?? undefined,
imageList: tempImages.length > 0 ? tempImages : undefined,
videoList: tempVideos.length > 0 ? tempVideos : undefined,
// Pass pageSelections metadata for immediate display
metadata: pageSelections?.length ? { pageSelections } : undefined,
},
{ operationId, tempMessageId: tempId },
);
@@ -222,7 +224,7 @@ export const conversationLifecycle: StateCreator<
const topicId = operationContext.topicId;
data = await aiChatService.sendMessageInServer(
{
newUserMessage: { content: message, files: fileIdList, parentId },
newUserMessage: { content: message, files: fileIdList, pageSelections, parentId },
// if there is topicIdthen add topicId to message
topicId: topicId ?? undefined,
threadId: operationContext.threadId ?? undefined,
@@ -372,26 +374,10 @@ export const conversationLifecycle: StateCreator<
messageMapKey(execContext),
)(get());
const contextMessages =
contexts?.map((item, index) => {
const now = Date.now();
const title = item.title ? `${item.title}\n` : '';
return {
content: `Context ${index + 1}:\n${title}${item.content}`,
createdAt: now,
id: `ctx_${tempId}_${index}`,
role: 'system' as const,
updatedAt: now,
};
}) ?? [];
const runtimeMessages =
contextMessages.length > 0 ? [...displayMessages, ...contextMessages] : displayMessages;
try {
await internal_execAgentRuntime({
context: execContext,
messages: runtimeMessages,
messages: displayMessages,
parentMessageId: data.assistantMessageId,
parentMessageType: 'assistant',
parentOperationId: operationId, // Pass as parent operation