diff --git a/.env.example b/.env.example index aa1088f..a8a8bb7 100644 --- a/.env.example +++ b/.env.example @@ -10,3 +10,9 @@ UPLOAD_MAX_PIXELS=30000000 VISION_MODEL=gemma3:4b VISION_DEBUG=false VISION_DEBUG_SHA256_ENABLED=false +# Required to enable automatic context compaction; use the exact hosted Ollama tag. +CONTEXT_MODEL= +CONTEXT_COMPACTION_TRIGGER_TOKENS=6000 +CONTEXT_COMPACTION_RECENT_TOKENS=2500 +CONTEXT_COMPACTION_MIN_MESSAGES=4 +CONTEXT_COMPACTION_MAX_CHARS=8000 diff --git a/src/config.js b/src/config.js index 047501b..a063d6f 100644 --- a/src/config.js +++ b/src/config.js @@ -69,9 +69,14 @@ export function createConfig(overrides = {}) { visionDebugSha256Enabled: bool(process.env.VISION_DEBUG_SHA256_ENABLED, false), chatLimit: positiveInt(process.env.CHAT_LIMIT, 10), chatHistoryLimit: positiveInt(process.env.CHAT_HISTORY_LIMIT, 20), - chatSummaryUpdateEveryMessages: positiveInt(process.env.CHAT_SUMMARY_UPDATE_EVERY_MESSAGES, 8), + chatSummaryUpdateEveryMessages: process.env.CHAT_SUMMARY_UPDATE_EVERY_MESSAGES == null ? 8 : nonNegativeInt(process.env.CHAT_SUMMARY_UPDATE_EVERY_MESSAGES, 8), chatSummaryMaxChars: positiveInt(process.env.CHAT_SUMMARY_MAX_CHARS, 2200), - contextSummaryModel: process.env.CONTEXT_SUMMARY_MODEL || "nomic-embed-text:latest", + // The only model used for persisted context compaction. Leave unset to disable it. + contextModel: process.env.CONTEXT_MODEL || null, + contextCompactionTriggerTokens: positiveInt(process.env.CONTEXT_COMPACTION_TRIGGER_TOKENS, 6000), + contextCompactionRecentTokens: positiveInt(process.env.CONTEXT_COMPACTION_RECENT_TOKENS, 2500), + contextCompactionMinMessages: positiveInt(process.env.CONTEXT_COMPACTION_MIN_MESSAGES, 4), + contextCompactionMaxChars: positiveInt(process.env.CONTEXT_COMPACTION_MAX_CHARS, 8000), chatMaxRetries: nonNegativeInt(process.env.CHAT_MAX_RETRIES, 5), titleModel: process.env.TITLE_MODEL || "nomic-embed-text:latest", modelApiBaseUrl: process.env.MODEL_API_BASE_URL || "https://ai.krishd.ch/api", diff --git a/src/db/chatRepository.js b/src/db/chatRepository.js index 0f6b0f5..fadd17d 100644 --- a/src/db/chatRepository.js +++ b/src/db/chatRepository.js @@ -121,6 +121,7 @@ export function createChatStatements(db) { updateChatPersonality: db.prepare("UPDATE chat_sessions SET personality_id = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ? AND username = ?"), updateChatSessionScene: db.prepare("UPDATE chat_sessions SET scenario_prompt = ?, scenario_summary = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ? AND username = ?"), updateChatContextSummary: db.prepare("UPDATE chat_sessions SET context_summary = ?, context_summary_message_id = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ? AND username = ?"), + updateChatContextSummaryIfUnchanged: db.prepare("UPDATE chat_sessions SET context_summary = ?, context_summary_message_id = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ? AND username = ? AND context_summary IS ? AND context_summary_message_id = ?"), updateChatGenerationSettings: db.prepare(` UPDATE chat_sessions SET preferred_model = ?, temperature = ?, context_length = ?, response_length = ?, system_instruction = ?, updated_at = CURRENT_TIMESTAMP @@ -448,6 +449,7 @@ const failChatTurnTransaction = db.transaction((chatId, userMessageId, errorMess updateChatPersonality: (chatId, username, personalityId) => statements.updateChatPersonality.run(personalityId, chatId, username), updateChatScene: (chatId, username, scenarioPrompt, scenarioSummary) => statements.updateChatSessionScene.run(scenarioPrompt, scenarioSummary, chatId, username), updateChatContextSummary: (chatId, username, summary, messageId) => statements.updateChatContextSummary.run(summary, Number(messageId || 0), chatId, username), + updateChatContextSummaryIfUnchanged: (chatId, username, summary, messageId, previousSummary, previousMessageId) => statements.updateChatContextSummaryIfUnchanged.run(summary, Number(messageId || 0), chatId, username, previousSummary, Number(previousMessageId || 0)), getPersonaChatState: (chatId, username) => statements.getPersonaChatState.get(chatId, username) || null, updatePersonaChatState(username, chat, state) { statements.upsertPersonaChatState.run( diff --git a/src/services/chatService.js b/src/services/chatService.js index 0fcdb15..2aed6a7 100644 --- a/src/services/chatService.js +++ b/src/services/chatService.js @@ -22,6 +22,7 @@ import { SearXNGSearchProvider, WebSearchError, decideWebSearch, retrieveWebSear import { supportsReasoning } from "../../public/js/modelCapabilities.js"; import {VISION_EVIDENCE_GUIDANCE} from "./visionAnalysis.js"; import {createConversationContextBuilder} from "./conversationContext.js"; +import {COMPACTION_SCHEMA, COMPACTION_SYSTEM_PROMPT, buildCompactionRequest, estimateTokens, isCompactionEligibleMessage, normalizeCompaction} from "./contextCompaction.js"; export function createChatService(repositories, modelService, config, webSearchProvider = new SearXNGSearchProvider(config), attachmentService = null, visionDebugService = null) { const formatChatMessage = (row, related = null) => { @@ -39,7 +40,6 @@ export function createChatService(repositories, modelService, config, webSearchP return message && attachments?.length ? { ...message, attachments } : message; }; const chatGenerationQueues = new Map(); - const summaryQueues = new Map(); const activeGenerations = new Map(); const regeneratingMessages = new Map(); const maxRetries = Number(config.chatMaxRetries ?? 5); @@ -152,7 +152,7 @@ export function createChatService(repositories, modelService, config, webSearchP signal?.throwIfAborted(); const title = String(JSON.parse(response)?.title || "").replace(/[\r\n]+/g, " ").replace(/["“”']/g, "").replace(/\s+/g, " ").trim(); if (title && title.length <= 70 && title.split(/\s+/).length <= 8) return title; - } catch (error) { + } catch { signal?.throwIfAborted(); } return "New chat"; @@ -183,7 +183,7 @@ export function createChatService(repositories, modelService, config, webSearchP return `${normalized.slice(0, Math.max(0, config.chatSummaryMaxChars - 3)).trimEnd()}...`; }; - const maybeUpdateContextSummary = async ({ user, chatId, selectedModel }) => { + const maybeUpdateContextSummary = async ({ user, chatId }) => { if (config.chatSummaryUpdateEveryMessages <= 0) return; const session = repositories.getChat(chatId, user); if (!session) return; @@ -199,7 +199,8 @@ export function createChatService(repositories, modelService, config, webSearchP if (summarizable.length < config.chatSummaryUpdateEveryMessages) return; const previousSummary = String(session.context_summary || "").trim(); - const summary = await modelService.generateReply(config.contextSummaryModel || "nomic-embed-text:latest", [ + if (!config.contextModel) return; + const summary = await modelService.generateReply(config.contextModel, [ { role: "system", content: [ @@ -237,14 +238,62 @@ export function createChatService(repositories, modelService, config, webSearchP }; const scheduleContextSummaryUpdate = ({ user, chatId, selectedModel }) => { - const previous = summaryQueues.get(chatId) || Promise.resolve(); - const current = previous - .catch(() => undefined) - .then(() => updateContextSummaryWithoutBreakingReply({ user, chatId, selectedModel })); - summaryQueues.set(chatId, current); - void current.finally(() => { - if (summaryQueues.get(chatId) === current) summaryQueues.delete(chatId); - }); + // Compaction is performed synchronously before the next main-model request. + void user; void chatId; void selectedModel; void updateContextSummaryWithoutBreakingReply; + }; + + const ensureContextCompaction = async ({user, chatId, currentMessage, signal = null}) => { + if (!config.contextModel) return null; + const session = repositories.getChat(chatId, user); + if (!session) return null; + const allMessages = repositories.listActiveBranchMessages(chatId) + .filter(isCompactionEligibleMessage); + const previousSummary = String(session.context_summary || "").trim(); + const compactedThrough = Number(session.context_summary_message_id || 0); + const activeMessages = allMessages.filter(message => Number(message.id) > compactedThrough); + const contextTokens = estimateTokens(previousSummary) + activeMessages.reduce((sum, item) => sum + estimateTokens(item.content), 0) + estimateTokens(currentMessage); + if (contextTokens < Number(config.contextCompactionTriggerTokens || 6000)) return null; + let recentTokens = 0; + let splitIndex = activeMessages.length; + for (let index = activeMessages.length - 1; index >= 0; index -= 1) { + recentTokens += estimateTokens(activeMessages[index].content); + if (recentTokens > Number(config.contextCompactionRecentTokens || 2500) && activeMessages.length - index >= 2) { + splitIndex = index; + break; + } + } + const summarizable = activeMessages.slice(0, splitIndex); + if (summarizable.length < Number(config.contextCompactionMinMessages || 4)) return null; + const latestSummarized = summarizable.at(-1); + const startedAt = Date.now(); + console.info("CONTEXT COMPACTION", JSON.stringify({conversationId: chatId, triggered: true, contextTokens, + compactedTokens: summarizable.reduce((sum, item) => sum + estimateTokens(item.content), 0), messages: summarizable.length})); + let rawResult; + try { + rawResult = await modelService.generateReply(config.contextModel, [ + {role: "system", content: COMPACTION_SYSTEM_PROMPT}, + {role: "user", content: buildCompactionRequest(previousSummary, summarizable)} + ], {signal, reasoning: false, generation: {temperature: 0, responseLength: 1600}, format: COMPACTION_SCHEMA}); + } catch (error) { + console.warn("CONTEXT COMPACTION FAILED", JSON.stringify({conversationId: chatId, error: String(error?.message || error)})); + return null; + } + const compacted = normalizeCompaction(rawResult, Number(config.contextCompactionMaxChars || 8000)); + if (!compacted) { + console.warn("CONTEXT COMPACTION INVALID", JSON.stringify({conversationId: chatId, durationMs: Date.now() - startedAt, + resultChars: typeof rawResult === "string" ? rawResult.length : null})); + return null; + } + const serialized = JSON.stringify(compacted); + const update = repositories.updateChatContextSummaryIfUnchanged?.(chatId, user, serialized, latestSummarized.id, + session.context_summary, session.context_summary_message_id); + if (repositories.updateChatContextSummaryIfUnchanged && !update?.changes) { + console.warn("CONTEXT COMPACTION STALE", JSON.stringify({conversationId: chatId, durationMs: Date.now() - startedAt})); + return repositories.getChat(chatId, user); + } + console.info("CONTEXT COMPACTION COMPLETE", JSON.stringify({conversationId: chatId, durationMs: Date.now() - startedAt, + compactedMessages: summarizable.length, resultChars: serialized.length})); + return {...session, context_summary: serialized, context_summary_message_id: latestSummarized.id}; }; const getOrCreatePersonaChat = (user, persona, userPersonaId = null, scenarioPrompt = null, scenarioSummary = null) => { @@ -420,7 +469,6 @@ export function createChatService(repositories, modelService, config, webSearchP const generateChatTurn = ({ user, chatId, message, model, attachments = [], reuseChatId = null, userMessageId: resumeMessageId = null, webSearch = true, reasoning = false, onStatus = null, onChunk = null, onThinking = null, signal = null, parentMessageId = undefined, requestId = null }) => withChatGenerationLock(chatId, async () => { - await (summaryQueues.get(chatId) || Promise.resolve()); const session = repositories.getChat(chatId, user); if (!session) return { error: "Chat not found", status: 404 }; if (parentMessageId !== undefined) { @@ -522,6 +570,11 @@ export function createChatService(repositories, modelService, config, webSearchP })); } } + const compactedSession = await ensureContextCompaction({user, chatId, currentMessage: message, signal: activeSignal}); + if (compactedSession) { + session.context_summary = compactedSession.context_summary; + session.context_summary_message_id = compactedSession.context_summary_message_id; + } onStatus?.({ status: "generating", searchState, ...(sources !== null ? { sourceCount: sources.length } : {}) }); payload = buildConversationContext({ user, chatId, session, message: null, upToMessageId: userMessageId, historyOverride: imageHistory, visionSelection, contextSummaryOverride: session.context_summary, sources }); @@ -681,7 +734,7 @@ export function createChatService(repositories, modelService, config, webSearchP }, persistChatMessageRetryState, buildMessagesPayload: buildConversationContext, - async generateInitialChatTitle({user, chatId, message, model, signal = null}) { + async generateInitialChatTitle({user, chatId, message, signal = null}) { const session = repositories.getChat(chatId, user); if (!session) return {error: "Chat not found", status: 404}; if (session.assistant_persona_id || session.title !== "New chat" || repositories.getLatestChatMessage(chatId)) { @@ -703,7 +756,6 @@ export function createChatService(repositories, modelService, config, webSearchP if (chatGenerationQueues.has(chatId)) return { error: "A response is already being generated", status: 409 }; regeneratingMessages.set(chatId, targetMessage.id); return withChatGenerationLock(chatId, async () => { - await (summaryQueues.get(chatId) || Promise.resolve()); const session = repositories.getChat(chatId, user); if (!session) return { error: "Chat not found", status: 404 }; diff --git a/src/services/conversationContext.js b/src/services/conversationContext.js index ab5dbbe..e7b0ff1 100644 --- a/src/services/conversationContext.js +++ b/src/services/conversationContext.js @@ -28,9 +28,11 @@ export function createConversationContextBuilder({repositories, attachmentServic if (!activeSession) return null; const activePersona = repositories.getAssistantPersonaForChat(chatId, user); const activeUserPersona = getUserPersonaForSession(chatId, user, activePersona); - const history = historyOverride || (upToMessageId == null + let history = historyOverride || (upToMessageId == null ? repositories.getRecentChatMessages(chatId).reverse() : repositories.getRecentChatMessagesUpToId(chatId, upToMessageId).reverse()); + const compactedThrough = Number(activeSession.context_summary_message_id || 0); + if (compactedThrough) history = history.filter(entry => Number(entry.id || 0) > compactedThrough); const conversation = message == null ? history : [...history, {role: "user", content: message}]; const messagesPayload = []; // Search results stay untrusted user context, followed by their grounding rule. diff --git a/test/app.test.js b/test/app.test.js index e2a0c41..d792a93 100644 --- a/test/app.test.js +++ b/test/app.test.js @@ -754,10 +754,10 @@ test("chat context summary rolls up older messages into future payloads", async let latestPayload = []; let summaryModel = null; const modelService = { - async generateReply(model, messagesPayload) { - if (messagesPayload.some((message) => message.content.includes("Merge the previous summary"))) { + async generateReply(model, messagesPayload, options = {}) { + if (options.format && messagesPayload.some((message) => message.content.includes("Newly expired conversation records"))) { summaryModel = model; - return "Summary: user greeted the assistant and asked for continuity."; + return JSON.stringify({summary: "user greeted the assistant and asked for continuity.", current_goals: [], important_facts: ["user greeted the assistant"], decisions: [], constraints: [], technical_state: [], open_tasks: [], problems_and_solutions: [], user_preferences: [], important_exact_values: []}); } latestPayload = messagesPayload; replyCount += 1; @@ -781,7 +781,7 @@ test("chat context summary rolls up older messages into future payloads", async }; const server = await startTestServer({ modelService, - config: {chatHistoryLimit: 2, chatSummaryUpdateEveryMessages: 2} + config: {chatHistoryLimit: 2, chatSummaryUpdateEveryMessages: 2, contextModel: "configured:context", contextCompactionTriggerTokens: 1, contextCompactionRecentTokens: 1, contextCompactionMinMessages: 2} }); const jar = new CookieJar(); @@ -812,7 +812,7 @@ test("chat context summary rolls up older messages into future payloads", async assert.ok(latestPayload.some((message) => message.role === "system" && message.content.includes("CONVERSATION SUMMARY"))); assert.ok(latestPayload.some((message) => message.content.includes("user greeted the assistant"))); - assert.equal(summaryModel, "nomic-embed-text:latest"); + assert.equal(summaryModel, "configured:context"); await server.close(); });