Add context compaction service: schema validation, token estimation, and message normalization with tests

src/services/contextCompaction.js

@@ -0,0 +1,88 @@
+export const COMPACTION_SYSTEM_PROMPT = [
+    "You are a context-compaction model.",
+    "Your output will be given to another AI model as memory of an earlier conversation.",
+    "Compress aggressively, but never discard information that may affect future answers.",
+    "Preserve current goals, user requirements, explicit constraints, decisions already made, unfinished work, project state, technical configuration, model names, versions, filenames, paths, URLs when relevant, commands, ports, identifiers, numeric values, API/interface decisions, errors, causes, attempted and successful fixes, user preferences, and relationships between important entities.",
+    "Remove greetings, acknowledgements, filler, repeated explanations, redundant wording, and obsolete intermediate discussion that no longer affects the current state.",
+    "Never invent information or silently change exact technical values. Do not replace precise facts with vague wording.",
+    "When newer information supersedes older information, preserve the newer state and mention the superseded state only if still relevant.",
+    "Distinguish facts from unresolved assumptions. Keep unresolved questions and tasks. Prefer concise factual statements.",
+    "Output only the requested structured JSON format."
+].join("\n");
+
+export const COMPACTION_SCHEMA = {
+    type: "object",
+    additionalProperties: false,
+    properties: {
+        summary: {type: "string"},
+        current_goals: {type: "array", items: {type: "string"}},
+        important_facts: {type: "array", items: {type: "string"}},
+        decisions: {type: "array", items: {type: "string"}},
+        constraints: {type: "array", items: {type: "string"}},
+        technical_state: {type: "array", items: {type: "string"}},
+        open_tasks: {type: "array", items: {type: "string"}},
+        problems_and_solutions: {type: "array", items: {type: "string"}},
+        user_preferences: {type: "array", items: {type: "string"}},
+        important_exact_values: {type: "array", items: {type: "string"}}
+    },
+    required: ["summary", "current_goals", "important_facts", "decisions", "constraints", "technical_state", "open_tasks", "problems_and_solutions", "user_preferences", "important_exact_values"]
+};
+
+const FIELDS = COMPACTION_SCHEMA.required;
+
+export function estimateTokens(value) {
+    return Math.ceil(String(value || "").length / 4);
+}
+
+export function normalizeCompaction(value, maxChars = 10000) {
+    let parsed = value;
+    if (typeof value === "string") {
+        try { parsed = JSON.parse(value.replace(/^```(?:json)?\s*|\s*```$/gi, "").trim()); }
+        catch { return null; }
+    }
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
+    const normalized = {};
+    for (const field of FIELDS) {
+        if (field === "summary") normalized[field] = typeof parsed[field] === "string" ? parsed[field].trim() : "";
+        else if (!Array.isArray(parsed[field]) || parsed[field].some(item => typeof item !== "string")) return null;
+        else normalized[field] = parsed[field].map(item => item.trim()).filter(Boolean);
+    }
+    const serialized = JSON.stringify(normalized);
+    const hasContent = normalized.summary || FIELDS.some(field => field !== "summary" && normalized[field].length);
+    if (!hasContent || serialized.length > maxChars) return null;
+    return normalized;
+}
+
+function compactMessage(message) {
+    const lines = [`${message.role === "bot" ? "Assistant" : message.role}: ${String(message.content || "").trim()}`];
+    if (message.thinking) lines.push(`Assistant reasoning outcome: ${String(message.thinking).slice(0, 1200)}`);
+    if (message.error_message) lines.push(`Error: ${String(message.error_message).slice(0, 1000)}`);
+    if (message.delivery_status && message.delivery_status !== "complete") lines.push(`Delivery status: ${message.delivery_status}`);
+    if (message.sources) {
+        try {
+            const sources = typeof message.sources === "string" ? JSON.parse(message.sources) : message.sources;
+            if (Array.isArray(sources) && sources.length) lines.push(`Sources: ${sources.slice(0, 5).map(source => `${source.title || ""} ${source.url || ""} ${String(source.snippet || "").slice(0, 500)}`).join(" | ")}`);
+        } catch { /* malformed optional metadata must not block compaction */ }
+    }
+    return lines.join("\n");
+}
+
+export function isCompactionEligibleMessage(message) {
+    return Boolean(message && String(message.content || message.error_message || "").trim())
+        && (!message.delivery_status || message.delivery_status === "complete" || message.delivery_status === "failed");
+}
+
+export function formatMessagesForCompaction(messages) {
+    return messages.map(compactMessage).join("\n\n");
+}
+
+export function buildCompactionRequest(previousSummary, messages) {
+    return [
+        previousSummary ? `Existing compacted state (update it; do not repeat it blindly):\n${previousSummary}` : "Existing compacted state: none",
+        "",
+        "Newly expired conversation records:",
+        formatMessagesForCompaction(messages),
+        "",
+        "Return exactly one JSON object matching the supplied schema."
+    ].join("\n");
+}

test/contextCompaction.test.js

@@ -0,0 +1,26 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import {buildCompactionRequest, estimateTokens, normalizeCompaction} from "../src/services/contextCompaction.js";
+
+const valid = {
+    summary: "The user is implementing context compaction.",
+    current_goals: ["Keep recent messages verbatim"],
+    important_facts: [], decisions: ["Use the existing Ollama client"], constraints: [],
+    technical_state: ["The context model is configured separately"], open_tasks: [],
+    problems_and_solutions: [], user_preferences: [], important_exact_values: ["CONTEXT_MODEL"]
+};
+
+test("compaction output is strictly validated and bounded", () => {
+    assert.deepEqual(normalizeCompaction(JSON.stringify(valid), 5000), valid);
+    assert.equal(normalizeCompaction("not json"), null);
+    assert.equal(normalizeCompaction(JSON.stringify({...valid, summary: "", current_goals: [], important_facts: [], decisions: [], constraints: [], technical_state: [], open_tasks: [], problems_and_solutions: [], user_preferences: [], important_exact_values: []}), 5000), null);
+    assert.equal(normalizeCompaction(JSON.stringify(valid), 10), null);
+});
+
+test("compaction request merges existing state with only expired records", () => {
+    const request = buildCompactionRequest("existing fact", [{role: "user", content: "old request"}]);
+    assert.match(request, /existing fact/);
+    assert.match(request, /old request/);
+    assert.match(request, /Newly expired conversation records/);
+    assert.equal(estimateTokens("1234"), 1);
+});