feat(database): topic metadata for user memory extractor (#10569)

docs/development/database-schema.dbml

@@ -1007,6 +1007,7 @@ table topics {
     session_id [name: 'topics_session_id_idx']
     group_id [name: 'topics_group_id_idx']
     agent_id [name: 'topics_agent_id_idx']
+    () [name: 'topics_extract_status_gin_idx']
   }
 }
 

packages/database/migrations/0057_add_topic_user_memory_extract_status.sql

@@ -0,0 +1 @@
+CREATE INDEX IF NOT EXISTS "topics_extract_status_gin_idx" ON "topics" USING gin ((metadata->'userMemoryExtractStatus') jsonb_path_ops);

packages/database/migrations/meta/_journal.json

@@ -399,6 +399,13 @@
       "when": 1764685643024,
       "tag": "0056_update_agent_slug_index",
       "breakpoints": true
+    },
+    {
+      "idx": 57,
+      "version": "7",
+      "when": 1764734167674,
+      "tag": "0057_add_topic_user_memory_extract_status",
+      "breakpoints": true
     }
   ],
   "version": "6"

packages/database/src/core/migrations.json

@@ -223,7 +223,10 @@
     "hash": "9646161fa041354714f823d726af27247bcd6e60fa3be5698c0d69f337a5700b"
   },
   {
-    "sql": ["DROP TABLE \"user_budgets\";", "\nDROP TABLE \"user_subscriptions\";"],
+    "sql": [
+      "DROP TABLE \"user_budgets\";",
+      "\nDROP TABLE \"user_subscriptions\";"
+    ],
     "bps": true,
     "folderMillis": 1729699958471,
     "hash": "7dad43a2a25d1aec82124a4e53f8d82f8505c3073f23606c1dc5d2a4598eacf9"
@@ -295,7 +298,9 @@
     "hash": "845a692ceabbfc3caf252a97d3e19a213bc0c433df2689900135f9cfded2cf49"
   },
   {
-    "sql": ["ALTER TABLE \"messages\" ADD COLUMN \"reasoning\" jsonb;"],
+    "sql": [
+      "ALTER TABLE \"messages\" ADD COLUMN \"reasoning\" jsonb;"
+    ],
     "bps": true,
     "folderMillis": 1737609172353,
     "hash": "2cb36ae4fcdd7b7064767e04bfbb36ae34518ff4bb1b39006f2dd394d1893868"
@@ -510,7 +515,9 @@
     "hash": "a7ccf007fd185ff922823148d1eae6fafe652fc98d2fd2793f84a84f29e93cd1"
   },
   {
-    "sql": ["ALTER TABLE \"ai_providers\" ADD COLUMN \"config\" jsonb;"],
+    "sql": [
+      "ALTER TABLE \"ai_providers\" ADD COLUMN \"config\" jsonb;"
+    ],
     "bps": true,
     "folderMillis": 1749309388370,
     "hash": "39cea379f08ee4cb944875c0b67f7791387b508c2d47958bb4cd501ed1ef33eb"
@@ -628,7 +635,9 @@
     "hash": "1ba9b1f74ea13348da98d6fcdad7867ab4316ed565bf75d84d160c526cdac14b"
   },
   {
-    "sql": ["ALTER TABLE \"agents\" ADD COLUMN IF NOT EXISTS \"virtual\" boolean DEFAULT false;"],
+    "sql": [
+      "ALTER TABLE \"agents\" ADD COLUMN IF NOT EXISTS \"virtual\" boolean DEFAULT false;"
+    ],
     "bps": true,
     "folderMillis": 1759116400580,
     "hash": "433ddae88e785f2db734e49a4c115eee93e60afe389f7919d66e5ba9aa159a37"
@@ -678,13 +687,17 @@
     "hash": "4bdc6505797d7a33b622498c138cfd47f637239f6905e1c484cd01d9d5f21d6b"
   },
   {
-    "sql": ["ALTER TABLE \"user_settings\" ADD COLUMN IF NOT EXISTS \"image\" jsonb;"],
+    "sql": [
+      "ALTER TABLE \"user_settings\" ADD COLUMN IF NOT EXISTS \"image\" jsonb;"
+    ],
     "bps": true,
     "folderMillis": 1760108430562,
     "hash": "ce09b301abb80f6563abc2f526bdd20b4f69bae430f09ba2179b9e3bfec43067"
   },
   {
-    "sql": ["ALTER TABLE \"documents\" ADD COLUMN IF NOT EXISTS \"editor_data\" jsonb;"],
+    "sql": [
+      "ALTER TABLE \"documents\" ADD COLUMN IF NOT EXISTS \"editor_data\" jsonb;"
+    ],
     "bps": true,
     "folderMillis": 1761554153406,
     "hash": "bf2f21293e90e11cf60a784cf3ec219eafa95f7545d7d2f9d1449c0b0949599a"
@@ -764,13 +777,17 @@
     "hash": "923ccbdf46c32be9a981dabd348e6923b4a365444241e9b8cc174bf5b914cbc5"
   },
   {
-    "sql": ["ALTER TABLE \"agents\" ADD COLUMN IF NOT EXISTS \"market_identifier\" text;\n"],
+    "sql": [
+      "ALTER TABLE \"agents\" ADD COLUMN IF NOT EXISTS \"market_identifier\" text;\n"
+    ],
     "bps": true,
     "folderMillis": 1762870034882,
     "hash": "4178aacb4b8892b7fd15d29209bbf9b1d1f9d7c406ba796f27542c0bcd919680"
   },
   {
-    "sql": ["ALTER TABLE \"message_plugins\" ADD COLUMN IF NOT EXISTS \"intervention\" jsonb;\n"],
+    "sql": [
+      "ALTER TABLE \"message_plugins\" ADD COLUMN IF NOT EXISTS \"intervention\" jsonb;\n"
+    ],
     "bps": true,
     "folderMillis": 1762911968658,
     "hash": "552a032cc0e595277232e70b5f9338658585bafe9481ae8346a5f322b673a68b"
@@ -799,7 +816,9 @@
     "hash": "f823b521f4d25e5dc5ab238b372727d2d2d7f0aed27b5eabc8a9608ce4e50568"
   },
   {
-    "sql": ["ALTER TABLE \"agents\" ADD COLUMN IF NOT EXISTS \"editor_data\" jsonb;"],
+    "sql": [
+      "ALTER TABLE \"agents\" ADD COLUMN IF NOT EXISTS \"editor_data\" jsonb;"
+    ],
     "bps": true,
     "folderMillis": 1764215503726,
     "hash": "4188893a9083b3c7baebdbad0dd3f9d9400ede7584ca2394f5c64305dc9ec7b0"
@@ -840,7 +859,9 @@
     "hash": "2c103eee82bdf329944fb622dd9c2b9f20df80eb54f23eb9254d2285de413099"
   },
   {
-    "sql": ["ALTER TABLE \"user_settings\" ADD COLUMN IF NOT EXISTS \"market\" jsonb;"],
+    "sql": [
+      "ALTER TABLE \"user_settings\" ADD COLUMN IF NOT EXISTS \"market\" jsonb;"
+    ],
     "bps": true,
     "folderMillis": 1764335703306,
     "hash": "28c0d738c0b1fdf5fd871363be1a1477b4accbabdc140fe8dc6e9b339aae2c89"
@@ -910,5 +931,13 @@
     "bps": true,
     "folderMillis": 1764685643024,
     "hash": "6e7ac7f964eb03efa3cb0d2fd35ded23e25c3abf955c4c2a51418f8daef54af9"
+  },
+  {
+    "sql": [
+      "CREATE INDEX IF NOT EXISTS \"topics_extract_status_gin_idx\" ON \"topics\" USING gin ((metadata->'userMemoryExtractStatus') jsonb_path_ops);\n"
+    ],
+    "bps": true,
+    "folderMillis": 1764734167674,
+    "hash": "89c134be2948d3afc360d6bac11dea0c6fd5c902bf6093ed077033adb920fd02"
   }
-]
+]

packages/database/src/schemas/topic.ts

@@ -1,5 +1,6 @@
 /* eslint-disable sort-keys-fix/sort-keys-fix  */
 import type { ChatTopicMetadata } from '@lobechat/types';
+import { sql } from 'drizzle-orm';
 import { boolean, index, jsonb, pgTable, primaryKey, text, uniqueIndex } from 'drizzle-orm/pg-core';
 import { createInsertSchema } from 'drizzle-zod';
 
@@ -39,6 +40,10 @@ export const topics = pgTable(
     index('topics_session_id_idx').on(t.sessionId),
     index('topics_group_id_idx').on(t.groupId),
     index('topics_agent_id_idx').on(t.agentId),
+    index('topics_extract_status_gin_idx').using(
+      'gin',
+      sql`(metadata->'userMemoryExtractStatus') jsonb_path_ops`,
+    ),
   ],
 );
 

packages/types/src/topic/topic.ts

@@ -24,9 +24,21 @@ export interface GroupedTopic {
   title?: string;
 }
 
+export interface TopicUserMemoryExtractRunState {
+  error?: string;
+  lastConversationDigest?: string;
+  lastMessageAt?: string;
+  lastRunAt?: string;
+  messageCount?: number;
+  processedMemoryCount?: number;
+  version?: string;
+}
+
 export interface ChatTopicMetadata {
   model?: string;
   provider?: string;
+  userMemoryExtractRunState?: TopicUserMemoryExtractRunState;
+  userMemoryExtractStatus?: 'pending' | 'completed' | 'failed';
 }
 
 export interface ChatTopicSummary {