From 65e4162651aec0ff940a03013dea64f22f3501f2 Mon Sep 17 00:00:00 2001 From: fretuks Date: Thu, 2 Jul 2026 12:58:14 +0200 Subject: [PATCH] make huge overhauls to backend --- public/app/dom.js | 6 + public/index.html | 3 + public/script.js | 102 ++++++++++++ public/settings.html | 46 +++++ public/settings.js | 76 ++++++++- public/style.css | 46 +++++ src/app.js | 2 +- src/config.js | 2 + src/db/repositories.js | 67 ++++++++ src/db/schema.js | 41 +++++ src/middleware/rateLimiters.js | 42 +++++ src/routes/chats.js | 71 +++++++- src/routes/system.js | 38 ++++- src/services/chatService.js | 93 +++++++++++ src/services/modelService.js | 58 ++++++- src/services/workspaceImportService.js | 194 ++++++++++++++++++++++ test/app.test.js | 221 +++++++++++++++++++++++++ test/config.test.js | 65 ++++++++ 18 files changed, 1159 insertions(+), 14 deletions(-) create mode 100644 src/middleware/rateLimiters.js create mode 100644 src/services/workspaceImportService.js create mode 100644 test/config.test.js diff --git a/public/app/dom.js b/public/app/dom.js index 6df6d4a..ec67011 100644 --- a/public/app/dom.js +++ b/public/app/dom.js @@ -38,8 +38,11 @@ const clearChatBtn = $("clearChat"); const exportChatBtn = $("exportChat"); const pinChatBtn = $("pinChat"); const moveChatFolderBtn = $("moveChatFolder"); +const manageChatMemoryBtn = $("manageChatMemory"); const archiveChatBtn = $("archiveChat"); const backupWorkspaceBtn = $("backupWorkspace"); +const importWorkspaceBtn = $("importWorkspace"); +const importWorkspaceFileInput = $("importWorkspaceFile"); const activeChatTitle = $("activeChatTitle"); const sessionUser = $("sessionUser"); const chatActivityOverlay = $("chatActivityOverlay"); @@ -133,8 +136,11 @@ export { exportChatBtn, pinChatBtn, moveChatFolderBtn, + manageChatMemoryBtn, archiveChatBtn, backupWorkspaceBtn, + importWorkspaceBtn, + importWorkspaceFileInput, loginForm, loginPasswordInput, loginSubmit, diff --git a/public/index.html b/public/index.html index 85e4515..dd70f3d 100644 --- a/public/index.html +++ b/public/index.html @@ -208,7 +208,10 @@
+ + +
diff --git a/public/script.js b/public/script.js index 0df99c8..805aaed 100644 --- a/public/script.js +++ b/public/script.js @@ -26,6 +26,9 @@ import { backupWorkspaceBtn, clearUserPersonaBtn, exportChatBtn, + importWorkspaceBtn, + manageChatMemoryBtn, + importWorkspaceFileInput, moveChatFolderBtn, loginForm, loginPasswordInput, @@ -891,6 +894,7 @@ function updateChatActionState() { if (hasChat) pinChatBtn.textContent = activeChat.is_pinned ? "Unpin" : "Pin"; } if (moveChatFolderBtn) moveChatFolderBtn.disabled = !hasChat; + if (manageChatMemoryBtn) manageChatMemoryBtn.disabled = !hasChat; if (archiveChatBtn) { archiveChatBtn.disabled = !hasChat; if (hasChat) archiveChatBtn.textContent = activeChat.archived_at ? "Unarchive" : "Archive"; @@ -898,6 +902,7 @@ function updateChatActionState() { clearChatBtn.disabled = !hasChat; exportChatBtn.disabled = !hasChat || currentMessages.length === 0; if (backupWorkspaceBtn) backupWorkspaceBtn.disabled = !currentUsername; + if (importWorkspaceBtn) importWorkspaceBtn.disabled = !currentUsername; } function renderPersonaSection() { @@ -1497,6 +1502,92 @@ async function backupWorkspace() { setNotice("Workspace backup downloaded.", "success"); } +function formatImportPreview(preview, warnings = []) { + const counts = [ + `${preview.personas || 0} persona${preview.personas === 1 ? "" : "s"}`, + `${preview.chats || 0} chat${preview.chats === 1 ? "" : "s"}`, + `${preview.messages || 0} message${preview.messages === 1 ? "" : "s"}`, + `${preview.templates || 0} template${preview.templates === 1 ? "" : "s"}` + ].join(", "); + const warningText = warnings.length ? ` Warnings: ${warnings.slice(0, 4).join("; ")}${warnings.length > 4 ? "; ..." : ""}` : ""; + return `Import preview: ${counts}.${warningText}`; +} + +async function importWorkspaceFromFile(file) { + if (!file) return; + setChatLoading(true, "Reading workspace backup..."); + let workspace; + try { + workspace = JSON.parse(await file.text()); + } catch { + setChatLoading(false); + return setNotice("Import failed: choose a valid JSON workspace backup.", "error"); + } + + const preview = await post("/imports/workspace/preview", {workspace}); + if (preview.error || preview.ok === false) { + setChatLoading(false); + const detail = (preview.errors || []).slice(0, 4).join("; "); + return setNotice(detail || preview.error || "Workspace import validation failed.", "error"); + } + + setChatLoading(false); + const confirmed = await confirmPopup({ + eyebrow: "Workspace import", + title: "Review import", + description: formatImportPreview(preview.preview || {}, preview.warnings || []), + confirmLabel: "Import" + }); + if (!confirmed) return; + + setChatLoading(true, "Importing workspace..."); + const imported = await post("/imports/workspace", { + workspace, + allowDuplicates: Boolean(preview.preview?.duplicates?.length) + }); + if (imported.error) { + setChatLoading(false); + const detail = (imported.errors || imported.warnings || []).slice(0, 4).join("; "); + return setNotice(detail || imported.error, "error"); + } + await Promise.all([loadPersonas(), loadChatSessions()]); + setChatLoading(false); + setNotice(`Workspace imported: ${imported.imported.personas} personas, ${imported.imported.chats} chats, ${imported.imported.templates} templates.`, "success"); +} + +function formatMemoryList(memories) { + if (!memories.length) return "No memory facts saved for this chat yet."; + return memories.map((memory, index) => `${index + 1}. ${memory.fact}`).join("\n"); +} + +async function manageChatMemory() { + if (!activeChatId) return; + const loaded = await get(`/chats/${activeChatId}/memories`); + if (loaded.error) return setNotice(loaded.error, "error"); + const memories = loaded.memories || []; + const action = await promptPopup({ + eyebrow: "Memory", + title: "Memory facts", + description: `${formatMemoryList(memories)}\n\nType a new fact to add it, or type delete 1 to remove a fact.`, + label: "Fact or command", + placeholder: "The user prefers terse answers.", + confirmLabel: "Save" + }); + const value = String(action || "").trim(); + if (!value) return; + const deleteMatch = value.match(/^delete\s+(\d+)$/i); + if (deleteMatch) { + const target = memories[Number(deleteMatch[1]) - 1]; + if (!target) return setNotice("Memory number not found.", "error"); + const removed = await del(`/chats/${activeChatId}/memories/${target.id}`); + if (removed.error) return setNotice(removed.error, "error"); + return setNotice("Memory fact deleted.", "success"); + } + const saved = await post(`/chats/${activeChatId}/memories`, {fact: value}); + if (saved.error) return setNotice(saved.error, "error"); + return setNotice("Memory fact saved.", "success"); +} + async function deleteChat(id) { setChatLoading(true, "Deleting chat..."); const chat = getChatById(id); @@ -1652,6 +1743,9 @@ const chatActions = { if (folderName === null) return; await updateChatOrganization(chat.id, {folderName: String(folderName || "").trim() || null}); }, + memory: async () => { + await manageChatMemory(); + }, pin: async () => { const chat = getChatById(activeChatId); if (chat) await updateChatOrganization(chat.id, {isPinned: !chat.is_pinned}); @@ -1700,6 +1794,14 @@ if (backupWorkspaceBtn) backupWorkspaceBtn.addEventListener("click", () => { closeChatDrawer(); void backupWorkspace(); }); +if (importWorkspaceBtn) importWorkspaceBtn.addEventListener("click", () => { + closeChatDrawer(); + importWorkspaceFileInput.value = ""; + importWorkspaceFileInput.click(); +}); +if (importWorkspaceFileInput) importWorkspaceFileInput.addEventListener("change", () => { + void importWorkspaceFromFile(importWorkspaceFileInput.files?.[0]); +}); chatSearchInput.addEventListener("input", () => { void loadChatSessions(); }); modelSelect.addEventListener("change", () => { renderModelSection(); diff --git a/public/settings.html b/public/settings.html index a65c7cb..ccac9aa 100644 --- a/public/settings.html +++ b/public/settings.html @@ -29,6 +29,7 @@ +
@@ -139,6 +140,51 @@
+ + diff --git a/public/settings.js b/public/settings.js index 8711ce7..8b33fed 100644 --- a/public/settings.js +++ b/public/settings.js @@ -3,6 +3,18 @@ const settingsNotice = $("settingsNotice"); const settingsThemeSelect = $("settingsThemeSelect"); const themeModeSelect = $("themeModeSelect"); const workspaceModeSelect = $("workspaceModeSelect"); +const diagnosticsNavItem = $("diagnosticsNavItem"); +const refreshDiagnosticsBtn = $("refreshDiagnostics"); +const diagnosticsStatus = $("diagnosticsStatus"); +const diagnosticsBackend = $("diagnosticsBackend"); +const diagnosticsLatency = $("diagnosticsLatency"); +const diagnosticsCheckedAt = $("diagnosticsCheckedAt"); +const diagnosticsModelCount = $("diagnosticsModelCount"); +const diagnosticsCacheAge = $("diagnosticsCacheAge"); +const diagnosticsActiveRequests = $("diagnosticsActiveRequests"); +const diagnosticsLastFailure = $("diagnosticsLastFailure"); +const diagnosticsModels = $("diagnosticsModels"); +const diagnosticsWarnings = $("diagnosticsWarnings"); const currentUsernameInput = $("currentUsername"); const newUsernameInput = $("newUsername"); const usernamePasswordInput = $("usernamePassword"); @@ -56,6 +68,7 @@ let userPersonas = []; let publishedPersonaIds = new Set(); let editingPersonaId = null; let settingsPopupResolver = null; +let isAdminUser = false; async function request(url, data, method = "POST") { try { @@ -141,6 +154,10 @@ function applyThemeMode(modeKey, persist = true) { function setSettingsView(view) { const normalizedView = ["my-ai-characters", "my-personas"].includes(view) ? "my-roleplay-companions" : view; + if (normalizedView === "diagnostics" && !isAdminUser) { + setNotice("Diagnostics are available to admin users only.", "error"); + return; + } settingsNavItems.forEach((item) => { const isActive = item.dataset.settingsView === normalizedView; item.classList.toggle("active", isActive); @@ -154,6 +171,57 @@ function setSettingsView(view) { const params = new URLSearchParams(window.location.search); params.set("view", normalizedView); history.replaceState(null, "", `${window.location.pathname}?${params.toString()}`); + if (normalizedView === "diagnostics") void loadDiagnostics(); +} + +function formatDuration(ms) { + if (!Number.isFinite(ms)) return "n/a"; + if (ms < 1000) return `${Math.round(ms)} ms`; + return `${(ms / 1000).toFixed(1)} s`; +} + +function renderDiagnosticsList(element, items, emptyText) { + element.innerHTML = ""; + if (!items.length) { + const empty = document.createElement("p"); + empty.className = "status persona-status"; + empty.textContent = emptyText; + element.appendChild(empty); + return; + } + items.forEach((item) => { + const row = document.createElement("div"); + row.className = "diagnostics-list-item"; + row.textContent = item; + element.appendChild(row); + }); +} + +async function loadDiagnostics() { + if (!isAdminUser) return false; + diagnosticsStatus.textContent = "Checking..."; + const res = await get("/admin/diagnostics"); + if (res.error) { + setNotice(res.error, "error"); + diagnosticsStatus.textContent = "Unavailable"; + return false; + } + + const model = res.model || {}; + const lastFailure = model.lastFailure; + const activeRequests = (model.activeRequests || []).reduce((sum, item) => sum + Number(item.activeRequests || 0), 0); + diagnosticsStatus.textContent = model.ok ? "Online" : "Degraded"; + diagnosticsBackend.textContent = model.backendUrl || "Backend unavailable"; + diagnosticsLatency.textContent = formatDuration(model.latencyMs ?? model.listLatencyMs); + diagnosticsCheckedAt.textContent = model.checkedAt ? `Checked ${new Date(model.checkedAt).toLocaleString()}` : "Not checked yet"; + diagnosticsModelCount.textContent = String(model.modelCount || 0); + diagnosticsCacheAge.textContent = `Cache age ${formatDuration(model.cacheAgeMs)}`; + diagnosticsActiveRequests.textContent = String(activeRequests); + diagnosticsLastFailure.textContent = lastFailure ? `${lastFailure.error} (${new Date(lastFailure.at).toLocaleString()})` : "No recorded failures"; + renderDiagnosticsList(diagnosticsModels, model.availableModels || [], "No models returned by the backend."); + renderDiagnosticsList(diagnosticsWarnings, res.configWarnings || [], "No config warnings."); + setNotice("Diagnostics loaded.", model.ok ? "success" : "error"); + return true; } function formatMeta(persona) { @@ -266,6 +334,8 @@ async function loadProfile() { } currentUsernameInput.value = res.username || ""; newUsernameInput.value = res.username || ""; + isAdminUser = res.username === "admin"; + diagnosticsNavItem.classList.toggle("hidden", !isAdminUser); return true; } @@ -444,6 +514,7 @@ workspaceModeSelect.addEventListener("change", (event) => { localStorage.setItem("krishd-workspace-mode", mode); setNotice(`Workspace mode set to ${mode}.`, "success"); }); +refreshDiagnosticsBtn.addEventListener("click", () => { void loadDiagnostics(); }); settingsNavItems.forEach((item) => item.addEventListener("click", () => setSettingsView(item.dataset.settingsView))); createAiCharacterBtn.addEventListener("click", () => openPersonaModal(null, "assistant")); @@ -480,9 +551,10 @@ window.addEventListener("load", async () => { workspaceModeSelect.value = localStorage.getItem("krishd-workspace-mode") || "basic"; const params = new URLSearchParams(window.location.search); const view = params.get("view") || "personal"; + const profileOk = await loadProfile(); setSettingsView(view); - const [profileOk, personasOk] = await Promise.all([loadProfile(), loadPersonas()]); - if (profileOk && personasOk) setNotice("Settings loaded."); + const personasOk = await loadPersonas(); + if (profileOk && personasOk && view !== "diagnostics") setNotice("Settings loaded."); const create = params.get("create"); if (create === "assistant") { diff --git a/public/style.css b/public/style.css index 895bf71..8827b19 100644 --- a/public/style.css +++ b/public/style.css @@ -2351,6 +2351,52 @@ select:focus { align-items: end; } +.diagnostics-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); + gap: 12px; +} + +.diagnostics-card { + border: 1px solid var(--pill-border); + border-radius: 8px; + padding: 14px; + background: var(--pill-surface); +} + +.diagnostics-card span { + display: block; + color: var(--text-muted); + font-size: 0.82rem; + font-weight: 700; + text-transform: uppercase; +} + +.diagnostics-card strong { + display: block; + margin-top: 6px; + font-size: 1.4rem; +} + +.diagnostics-columns { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); + gap: 16px; +} + +.diagnostics-list { + display: grid; + gap: 8px; +} + +.diagnostics-list-item { + border: 1px solid var(--pill-border); + border-radius: 8px; + padding: 10px 12px; + background: var(--surface-strong); + overflow-wrap: anywhere; +} + .settings-theme-note { display: grid; gap: 4px; diff --git a/src/app.js b/src/app.js index 2d80862..93790ee 100644 --- a/src/app.js +++ b/src/app.js @@ -192,7 +192,7 @@ export function createApp(options = {}) { app.use(createSettingsRouter({repositories})); app.use(createChatsRouter({repositories, chatService, modelService, config, chatRateLimiters})); app.use(createPersonasRouter({repositories, chatService, modelService, config})); - app.use(createSystemRouter({modelService})); + app.use(createSystemRouter({modelService, config})); app.use(createPagesRouter(config)); app.use(notFoundHandler); app.use(errorHandler); diff --git a/src/config.js b/src/config.js index e94d3a5..627bc6f 100644 --- a/src/config.js +++ b/src/config.js @@ -47,6 +47,8 @@ export function createConfig(overrides = {}) { slowRequestWarnMs: int(process.env.SLOW_REQUEST_WARN_MS, 50), chatLimit: int(process.env.CHAT_LIMIT, 10), chatHistoryLimit: int(process.env.CHAT_HISTORY_LIMIT, 20), + chatSummaryUpdateEveryMessages: int(process.env.CHAT_SUMMARY_UPDATE_EVERY_MESSAGES, 8), + chatSummaryMaxChars: int(process.env.CHAT_SUMMARY_MAX_CHARS, 2200), modelApiBaseUrl: process.env.MODEL_API_BASE_URL || "https://ai.krishd.ch/api", modelRequestTimeoutMs: int(process.env.MODEL_REQUEST_TIMEOUT_MS, 15000), modelListTimeoutMs: int(process.env.MODEL_LIST_TIMEOUT_MS, 4000), diff --git a/src/db/repositories.js b/src/db/repositories.js index cc08cb6..ab1fafe 100644 --- a/src/db/repositories.js +++ b/src/db/repositories.js @@ -63,6 +63,8 @@ export function createRepositories(db, config) { cs.archived_at, cs.scenario_prompt, cs.scenario_summary, + cs.context_summary, + cs.context_summary_message_id, cs.created_at, cs.updated_at FROM chat_sessions cs @@ -88,6 +90,8 @@ export function createRepositories(db, config) { cs.archived_at, cs.scenario_prompt, cs.scenario_summary, + cs.context_summary, + cs.context_summary_message_id, cs.created_at, cs.updated_at FROM chat_sessions cs @@ -112,6 +116,8 @@ export function createRepositories(db, config) { cs.archived_at, cs.scenario_prompt, cs.scenario_summary, + cs.context_summary, + cs.context_summary_message_id, cs.created_at, cs.updated_at FROM chat_sessions cs @@ -125,6 +131,7 @@ export function createRepositories(db, config) { `), updateChatSessionTitle: db.prepare("UPDATE chat_sessions SET title = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ? AND username = ?"), updateChatSessionScene: db.prepare("UPDATE chat_sessions SET scenario_prompt = ?, scenario_summary = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ? AND username = ?"), + updateChatContextSummary: db.prepare("UPDATE chat_sessions SET context_summary = ?, context_summary_message_id = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ? AND username = ?"), updateChatSessionOrganization: db.prepare(` UPDATE chat_sessions SET folder_name = ?, @@ -201,6 +208,33 @@ export function createRepositories(db, config) { ORDER BY id DESC LIMIT 1 `), + listChatMemories: db.prepare(` + SELECT id, username, chat_id, assistant_persona_id, user_persona_id, fact, created_at, updated_at + FROM chat_memories + WHERE chat_id = ? + AND username = ? + ORDER BY datetime(updated_at) DESC, id DESC + `), + getChatMemory: db.prepare(` + SELECT id, username, chat_id, assistant_persona_id, user_persona_id, fact, created_at, updated_at + FROM chat_memories + WHERE id = ? + AND chat_id = ? + AND username = ? + `), + insertChatMemory: db.prepare(` + INSERT INTO chat_memories (username, chat_id, assistant_persona_id, user_persona_id, fact) + VALUES (?, ?, ?, ?, ?) + `), + updateChatMemory: db.prepare(` + UPDATE chat_memories + SET fact = ?, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? + AND chat_id = ? + AND username = ? + `), + deleteChatMemory: db.prepare("DELETE FROM chat_memories WHERE id = ? AND chat_id = ? AND username = ?"), updateChatMessage: db.prepare("UPDATE chat_messages SET content = ? WHERE id = ? AND chat_id = ?"), updateChatMessageWithRetryState: db.prepare("UPDATE chat_messages SET content = ?, retry_variants = ?, retry_active_index = ?, retry_retries_used = ?, retry_prompt_message_id = ? WHERE id = ? AND chat_id = ?"), deleteChatMessage: db.prepare("DELETE FROM chat_messages WHERE id = ? AND chat_id = ?"), @@ -221,6 +255,8 @@ export function createRepositories(db, config) { cs.archived_at, cs.scenario_prompt, cs.scenario_summary, + cs.context_summary, + cs.context_summary_message_id, cs.created_at, cs.updated_at, cm.content AS matched_content @@ -507,6 +543,7 @@ export function createRepositories(db, config) { updatePersonaMarketCreator: db.prepare("UPDATE persona_market SET creator_username = ? WHERE creator_username = ?"), updatePersonaVersionsUsername: db.prepare("UPDATE persona_versions SET username = ? WHERE username = ?"), updatePromptTemplatesUsername: db.prepare("UPDATE prompt_templates SET username = ? WHERE username = ?"), + updateChatMemoriesUsername: db.prepare("UPDATE chat_memories SET username = ? WHERE username = ?"), updateMarketFavoritesUsername: db.prepare("UPDATE persona_market_favorites SET username = ? WHERE username = ?"), updateMarketRatingsUsername: db.prepare("UPDATE persona_market_ratings SET username = ? WHERE username = ?"), updateMarketReportsUsername: db.prepare("UPDATE persona_market_reports SET reporter_username = ? WHERE reporter_username = ?"), @@ -523,6 +560,7 @@ export function createRepositories(db, config) { statements.updatePersonaMarketCreator.run(nextUsername, currentUsername); statements.updatePersonaVersionsUsername.run(nextUsername, currentUsername); statements.updatePromptTemplatesUsername.run(nextUsername, currentUsername); + statements.updateChatMemoriesUsername.run(nextUsername, currentUsername); statements.updateMarketFavoritesUsername.run(nextUsername, currentUsername); statements.updateMarketRatingsUsername.run(nextUsername, currentUsername); statements.updateMarketReportsUsername.run(nextUsername, currentUsername); @@ -597,6 +635,14 @@ export function createRepositories(db, config) { chat.scenario_prompt ?? null, chat.scenario_summary ?? null ).lastInsertRowid; + if (chat.context_summary) { + statements.updateChatContextSummary.run( + chat.context_summary, + Number(chat.context_summary_message_id || 0), + chatId, + username + ); + } for (const message of chat.messages || []) { statements.insertChatMessage.run( chatId, @@ -609,6 +655,11 @@ export function createRepositories(db, config) { message.retryPromptMessageId ?? null ); } + for (const memory of chat.memories || []) { + const fact = String(memory?.fact || "").trim(); + if (!fact) continue; + statements.insertChatMemory.run(username, chatId, null, null, fact); + } imported.chats += 1; } @@ -644,6 +695,7 @@ export function createRepositories(db, config) { }, updateChatTitle: (chatId, username, title) => statements.updateChatSessionTitle.run(title, chatId, username), updateChatScene: (chatId, username, scenarioPrompt, scenarioSummary) => statements.updateChatSessionScene.run(scenarioPrompt, scenarioSummary, chatId, username), + updateChatContextSummary: (chatId, username, summary, messageId) => statements.updateChatContextSummary.run(summary, Number(messageId || 0), chatId, username), updateChatOrganization(chatId, username, {folderName = null, isPinned = false, archivedAt = null}) { return statements.updateChatSessionOrganization.run(folderName, isPinned ? 1 : 0, archivedAt, chatId, username); }, @@ -656,6 +708,20 @@ export function createRepositories(db, config) { getChatMessageByIndex: (chatId, index) => statements.getChatMessageByIndex.get(chatId, index), getLatestChatMessage: (chatId) => statements.getLatestChatMessage.get(chatId), getPreviousUserMessage: (chatId, messageId) => statements.getPreviousUserMessage.get(chatId, messageId), + listChatMemories: (chatId, username) => statements.listChatMemories.all(chatId, username), + getChatMemory: (chatId, memoryId, username) => statements.getChatMemory.get(memoryId, chatId, username), + createChatMemory(username, chat, fact) { + const result = statements.insertChatMemory.run( + username, + chat.id, + chat.assistant_persona_id ?? null, + chat.user_persona_id ?? null, + String(fact || "").trim() + ); + return statements.getChatMemory.get(result.lastInsertRowid, chat.id, username); + }, + updateChatMemory: (chatId, memoryId, username, fact) => statements.updateChatMemory.run(String(fact || "").trim(), memoryId, chatId, username), + deleteChatMemory: (chatId, memoryId, username) => statements.deleteChatMemory.run(memoryId, chatId, username), insertChatMessage(chatId, role, content, retryState = null, modelName = null) { return statements.insertChatMessage.run( chatId, @@ -896,6 +962,7 @@ export function createRepositories(db, config) { exportedAt: new Date().toISOString(), chats: this.listChats(username).map((chat) => ({ ...chat, + memories: this.listChatMemories(chat.id, username), messages: this.listChatMessages(chat.id).map((message) => ({ ...message, retryVariants: parseJsonArray(message.retry_variants) diff --git a/src/db/schema.js b/src/db/schema.js index 863da90..7ee0910 100644 --- a/src/db/schema.js +++ b/src/db/schema.js @@ -23,6 +23,8 @@ export function initializeSchema(db) { archived_at DATETIME, scenario_prompt TEXT, scenario_summary TEXT, + context_summary TEXT, + context_summary_message_id INTEGER DEFAULT 0, created_at DATETIME DEFAULT CURRENT_TIMESTAMP, updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (assistant_persona_id) REFERENCES personas (id) ON DELETE SET NULL, @@ -47,6 +49,24 @@ export function initializeSchema(db) { ) `).run(); + db.prepare(` + CREATE TABLE IF NOT EXISTS chat_memories + ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + username TEXT NOT NULL, + chat_id INTEGER NOT NULL, + assistant_persona_id INTEGER, + user_persona_id INTEGER, + fact TEXT NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (username) REFERENCES users (username) ON DELETE CASCADE, + FOREIGN KEY (chat_id) REFERENCES chat_sessions (id) ON DELETE CASCADE, + FOREIGN KEY (assistant_persona_id) REFERENCES personas (id) ON DELETE SET NULL, + FOREIGN KEY (user_persona_id) REFERENCES personas (id) ON DELETE SET NULL + ) + `).run(); + db.prepare(` CREATE TABLE IF NOT EXISTS personas ( @@ -227,7 +247,28 @@ export function runMigrations(db, {dropLegacyChats = true} = {}) { ensureColumn("chat_sessions", chatSessionColumns, "archived_at", "ALTER TABLE chat_sessions ADD COLUMN archived_at DATETIME"); ensureColumn("chat_sessions", chatSessionColumns, "scenario_prompt", "ALTER TABLE chat_sessions ADD COLUMN scenario_prompt TEXT"); ensureColumn("chat_sessions", chatSessionColumns, "scenario_summary", "ALTER TABLE chat_sessions ADD COLUMN scenario_summary TEXT"); + ensureColumn("chat_sessions", chatSessionColumns, "context_summary", "ALTER TABLE chat_sessions ADD COLUMN context_summary TEXT"); + ensureColumn("chat_sessions", chatSessionColumns, "context_summary_message_id", "ALTER TABLE chat_sessions ADD COLUMN context_summary_message_id INTEGER DEFAULT 0"); db.prepare("UPDATE chat_sessions SET is_pinned = 0 WHERE is_pinned IS NULL").run(); + db.prepare("UPDATE chat_sessions SET context_summary_message_id = 0 WHERE context_summary_message_id IS NULL").run(); + + db.prepare(` + CREATE TABLE IF NOT EXISTS chat_memories + ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + username TEXT NOT NULL, + chat_id INTEGER NOT NULL, + assistant_persona_id INTEGER, + user_persona_id INTEGER, + fact TEXT NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (username) REFERENCES users (username) ON DELETE CASCADE, + FOREIGN KEY (chat_id) REFERENCES chat_sessions (id) ON DELETE CASCADE, + FOREIGN KEY (assistant_persona_id) REFERENCES personas (id) ON DELETE SET NULL, + FOREIGN KEY (user_persona_id) REFERENCES personas (id) ON DELETE SET NULL + ) + `).run(); db.prepare(` CREATE TABLE IF NOT EXISTS persona_versions diff --git a/src/middleware/rateLimiters.js b/src/middleware/rateLimiters.js new file mode 100644 index 0000000..41f0d19 --- /dev/null +++ b/src/middleware/rateLimiters.js @@ -0,0 +1,42 @@ +import {createRateLimiter} from "./rateLimit.js"; + +export function createAuthRateLimiters(config) { + return [ + createRateLimiter({ + windowMs: config.authRateLimitWindowMs, + max: config.authRateLimitMaxPerIp, + cleanupIntervalMs: config.rateLimitCleanupIntervalMs, + keyGenerator: (req) => `auth:ip:${req.ip}`, + message: "Too many authentication attempts from this IP. Please try again later." + }), + createRateLimiter({ + windowMs: config.authRateLimitWindowMs, + max: config.authRateLimitMaxPerAccount, + cleanupIntervalMs: config.rateLimitCleanupIntervalMs, + keyGenerator: (req) => { + const username = String(req.body?.username || "").trim().toLowerCase(); + return username ? `auth:user:${username}` : null; + }, + message: "Too many authentication attempts for this account. Please try again later." + }) + ]; +} + +export function createChatRateLimiters(config) { + return [ + createRateLimiter({ + windowMs: config.chatRateLimitWindowMs, + max: config.chatRateLimitMaxPerIp, + cleanupIntervalMs: config.rateLimitCleanupIntervalMs, + keyGenerator: (req) => `chat:ip:${req.ip}`, + message: "Too many chat requests from this IP. Please slow down." + }), + createRateLimiter({ + windowMs: config.chatRateLimitWindowMs, + max: config.chatRateLimitMaxPerAccount, + cleanupIntervalMs: config.rateLimitCleanupIntervalMs, + keyGenerator: (req) => req.session.user ? `chat:user:${req.session.user}` : null, + message: "Too many chat requests for this account. Please slow down." + }) + ]; +} diff --git a/src/routes/chats.js b/src/routes/chats.js index 5143f7c..c9125d1 100644 --- a/src/routes/chats.js +++ b/src/routes/chats.js @@ -1,6 +1,7 @@ import express from "express"; import {requireLogin} from "../middleware/auth.js"; import {validateBody} from "../middleware/validate.js"; +import {validateWorkspaceImport} from "../services/workspaceImportService.js"; const createChatValidator = (body) => ({ value: { @@ -31,6 +32,13 @@ const organizationValidator = (body) => ({ } }); +const memoryValidator = (body) => { + const fact = String(body?.fact || "").replace(/\s+/g, " ").trim(); + if (!fact) return {error: "Memory fact is required"}; + if (fact.length > 800) return {error: "Memory fact must be 800 characters or fewer"}; + return {value: {fact}}; +}; + export function createChatsRouter({repositories, chatService, modelService, config, chatRateLimiters}) { const router = express.Router(); router.use(requireLogin); @@ -202,10 +210,43 @@ export function createChatsRouter({repositories, chatService, modelService, conf const chat = repositories.getChat(chatId, req.session.user); if (!chat) return res.status(404).json({error: "Chat not found"}); repositories.clearChatMessages(chatId); + repositories.updateChatContextSummary(chatId, req.session.user, null, 0); repositories.touchChat(chatId, req.session.user); return res.json({message: "Chat cleared"}); }); + router.get("/chats/:id/memories", (req, res) => { + const chatId = Number(req.params.id); + const chat = repositories.getChat(chatId, req.session.user); + if (!chat) return res.status(404).json({error: "Chat not found"}); + return res.json({memories: repositories.listChatMemories(chatId, req.session.user)}); + }); + + router.post("/chats/:id/memories", validateBody(memoryValidator), (req, res) => { + const chatId = Number(req.params.id); + const chat = repositories.getChat(chatId, req.session.user); + if (!chat) return res.status(404).json({error: "Chat not found"}); + return res.json({memory: repositories.createChatMemory(req.session.user, chat, req.validatedBody.fact)}); + }); + + router.put("/chats/:id/memories/:memoryId", validateBody(memoryValidator), (req, res) => { + const chatId = Number(req.params.id); + const chat = repositories.getChat(chatId, req.session.user); + if (!chat) return res.status(404).json({error: "Chat not found"}); + const result = repositories.updateChatMemory(chatId, Number(req.params.memoryId), req.session.user, req.validatedBody.fact); + if (result.changes === 0) return res.status(404).json({error: "Memory not found"}); + return res.json({memory: repositories.getChatMemory(chatId, Number(req.params.memoryId), req.session.user)}); + }); + + router.delete("/chats/:id/memories/:memoryId", (req, res) => { + const chatId = Number(req.params.id); + const chat = repositories.getChat(chatId, req.session.user); + if (!chat) return res.status(404).json({error: "Chat not found"}); + const result = repositories.deleteChatMemory(chatId, Number(req.params.memoryId), req.session.user); + if (result.changes === 0) return res.status(404).json({error: "Memory not found"}); + return res.json({message: "Memory deleted"}); + }); + router.post("/chat", ...chatRateLimiters, validateBody(sendMessageValidator), async (req, res) => { try { const result = await chatService.sendChatMessage({ @@ -326,7 +367,35 @@ export function createChatsRouter({repositories, chatService, modelService, conf }); router.get("/exports/workspace", (req, res) => res.json({workspace: repositories.exportWorkspace(req.session.user)})); - router.post("/imports/workspace", (req, res) => res.json({imported: repositories.importWorkspace(req.session.user, req.body?.workspace || {})})); + router.post("/imports/workspace/preview", (req, res) => { + const validation = validateWorkspaceImport(req.body?.workspace, repositories.exportWorkspace(req.session.user)); + return res.status(validation.ok ? 200 : 400).json({ + ok: validation.ok, + errors: validation.errors, + warnings: validation.warnings, + preview: validation.preview + }); + }); + router.post("/imports/workspace", (req, res) => { + const validation = validateWorkspaceImport(req.body?.workspace, repositories.exportWorkspace(req.session.user)); + if (!validation.ok) { + return res.status(400).json({ + error: "Workspace import validation failed", + errors: validation.errors, + warnings: validation.warnings, + preview: validation.preview + }); + } + if (validation.preview.duplicates.length && !req.body?.allowDuplicates) { + return res.status(409).json({ + error: "Workspace import contains duplicate records", + errors: [], + warnings: validation.warnings, + preview: validation.preview + }); + } + return res.json({imported: repositories.importWorkspace(req.session.user, validation.workspace), preview: validation.preview}); + }); router.get("/chat/history", (req, res) => res.json({history: repositories.listChats(req.session.user)})); diff --git a/src/routes/system.js b/src/routes/system.js index 3ae027e..95148a8 100644 --- a/src/routes/system.js +++ b/src/routes/system.js @@ -1,6 +1,34 @@ import express from "express"; +import {requireLogin} from "../middleware/auth.js"; -export function createSystemRouter({modelService}) { +function requireAdmin(req, res, next) { + if (req.session.user !== "admin") { + return res.status(403).json({error: "Admin only"}); + } + return next(); +} + +function buildConfigWarnings(config) { + const warnings = []; + if (config.sessionSecret === "change-me-session-secret") { + warnings.push("SESSION_SECRET is using the development default."); + } + if (!config.cookieSecure && !config.testMode) { + warnings.push("COOKIE_SECURE is disabled; use secure cookies behind HTTPS in production."); + } + if (!config.sameOriginProtectionEnabled) { + warnings.push("Same-origin protection is disabled."); + } + if (!config.slowRequestLoggingEnabled) { + warnings.push("Slow request logging is disabled."); + } + if (!config.eventLoopLagMonitorEnabled) { + warnings.push("Event-loop lag monitoring is disabled."); + } + return warnings; +} + +export function createSystemRouter({modelService, config}) { const router = express.Router(); router.get("/health", async (req, res) => { @@ -8,5 +36,13 @@ export function createSystemRouter({modelService}) { return res.status(model.ok ? 200 : 503).json({ok: model.ok, model}); }); + router.get("/admin/diagnostics", requireLogin, requireAdmin, async (req, res) => { + const diagnostics = await modelService.getDiagnostics(); + return res.status(diagnostics.model.ok ? 200 : 503).json({ + model: diagnostics.model, + configWarnings: buildConfigWarnings(config) + }); + }); + return router; } diff --git a/src/services/chatService.js b/src/services/chatService.js index ccd3c20..ddae9c4 100644 --- a/src/services/chatService.js +++ b/src/services/chatService.js @@ -87,6 +87,89 @@ export function createChatService(repositories, modelService, config) { return repositories.getChatMessageByIndex(chatId, safeIndex) || null; }; + const buildMemoryPrompt = (memories) => { + const facts = (memories || []) + .map((memory) => String(memory.fact || "").trim()) + .filter(Boolean) + .slice(0, 30); + if (!facts.length) return null; + return [ + "LONG-TERM MEMORY:", + "These are durable facts the assistant should remember for this chat/persona context.", + "Use them for continuity, but do not mention them unless relevant.", + ...facts.map((fact, index) => `${index + 1}. ${fact}`) + ].join("\n"); + }; + + const clampSummary = (value) => { + const normalized = String(value || "").trim(); + if (normalized.length <= config.chatSummaryMaxChars) return normalized; + return `${normalized.slice(0, Math.max(0, config.chatSummaryMaxChars - 3)).trimEnd()}...`; + }; + + const buildContextSummaryPrompt = (summary) => { + if (!summary) return null; + return [ + "CONVERSATION SUMMARY:", + "This is compressed background context from earlier messages in this chat.", + "Use it for continuity. Recent raw messages below are more authoritative.", + summary + ].join("\n"); + }; + + const formatMessagesForSummary = (messages) => messages.map((message) => { + const speaker = message.role === "bot" ? "Assistant" : message.role === "user" ? "User" : message.role; + return `${speaker}: ${String(message.content || "").trim()}`; + }).join("\n\n"); + + const maybeUpdateContextSummary = async ({user, chatId, selectedModel}) => { + if (config.chatSummaryUpdateEveryMessages <= 0) return; + const session = repositories.getChat(chatId, user); + if (!session) return; + const allMessages = repositories.listChatMessages(chatId); + const keepRecent = Math.max(0, config.chatHistoryLimit); + const cutoffIndex = allMessages.length - keepRecent - 1; + if (cutoffIndex < 0) return; + const cutoffMessage = allMessages[cutoffIndex]; + const summarizable = allMessages + .filter((message) => message.id > Number(session.context_summary_message_id || 0) && message.id <= cutoffMessage.id); + if (summarizable.length < config.chatSummaryUpdateEveryMessages) return; + + const previousSummary = String(session.context_summary || "").trim(); + const summary = await modelService.generateReply(selectedModel, [ + { + role: "system", + content: [ + "You maintain a compact running summary for a chat application.", + "Merge the previous summary and new transcript into one concise continuity summary.", + "Preserve durable facts, decisions, goals, relationships, unresolved threads, locations, preferences, and roleplay continuity.", + "Do not invent details. Do not include generic filler. Keep it under 12 bullet points." + ].join("\n") + }, + { + role: "user", + content: [ + previousSummary ? `Previous summary:\n${previousSummary}` : "Previous summary: none", + "", + "New transcript to compress:", + formatMessagesForSummary(summarizable) + ].join("\n") + } + ]); + + const latestSummarized = summarizable[summarizable.length - 1]; + if (!latestSummarized || !summary) return; + repositories.updateChatContextSummary(chatId, user, clampSummary(summary), latestSummarized.id); + }; + + const updateContextSummaryWithoutBreakingReply = async ({user, chatId, selectedModel}) => { + try { + await maybeUpdateContextSummary({user, chatId, selectedModel}); + } catch (error) { + console.warn("CHAT SUMMARY UPDATE FAILED", chatId, error?.message || error); + } + }; + const getOrCreatePersonaChat = (user, persona, userPersonaId = null, scenarioPrompt = null, scenarioSummary = null) => { const existing = repositories.getChatByParticipants(user, persona.id, userPersonaId); if (existing) return existing; @@ -149,6 +232,7 @@ export function createChatService(repositories, modelService, config) { const activePersona = repositories.getAssistantPersonaForChat(chatId, user); const activeUserPersona = repositories.getUserPersonaForChat(chatId, user) || repositories.getActiveUserPersona(user); const promptMessage = repositories.getPreviousUserMessage(chatId, targetMessage.id); + const memoryPrompt = buildMemoryPrompt(repositories.listChatMemories(chatId, user)); if (!promptMessage) { if (!activePersona || !session.scenario_summary) { @@ -182,6 +266,9 @@ export function createChatService(repositories, modelService, config) { }) }); } + const contextSummaryPrompt = buildContextSummaryPrompt(session.context_summary); + if (contextSummaryPrompt) messagesPayload.push({role: "system", content: contextSummaryPrompt}); + if (memoryPrompt) messagesPayload.push({role: "system", content: memoryPrompt}); messagesPayload.push(...conversation.map((message) => ({ role: message.role === "bot" ? "assistant" : "user", content: message.content @@ -227,6 +314,10 @@ export function createChatService(repositories, modelService, config) { }) }); } + const contextSummaryPrompt = buildContextSummaryPrompt(session.context_summary); + if (contextSummaryPrompt) messagesPayload.push({role: "system", content: contextSummaryPrompt}); + const memoryPrompt = buildMemoryPrompt(repositories.listChatMemories(chatId, user)); + if (memoryPrompt) messagesPayload.push({role: "system", content: memoryPrompt}); messagesPayload.push(...conversation.map((entry) => ({ role: entry.role === "bot" ? "assistant" : "user", content: entry.content @@ -259,6 +350,7 @@ export function createChatService(repositories, modelService, config) { retryPromptMessageId: null }, model); repositories.touchChat(chatId, user); + await updateContextSummaryWithoutBreakingReply({user, chatId, selectedModel: model}); return {reply: fullReply}; }, async streamChatMessage({user, chatId, message, model, onChunk}) { @@ -288,6 +380,7 @@ export function createChatService(repositories, modelService, config) { retryPromptMessageId: null }, model); repositories.touchChat(chatId, user); + await updateContextSummaryWithoutBreakingReply({user, chatId, selectedModel: model}); return {reply: fullReply}; }, async retryMessage({user, chatId, selectedModel, targetMessage, retryStyle = null}) { diff --git a/src/services/modelService.js b/src/services/modelService.js index 41cb799..e550670 100644 --- a/src/services/modelService.js +++ b/src/services/modelService.js @@ -12,7 +12,9 @@ export function createModelService(config, overrides = {}) { const modelState = new Map(); let modelsCache = null; let modelsCacheAt = 0; - let lastHealth = {ok: true, checkedAt: null, error: null}; + let lastHealth = {ok: true, checkedAt: null, error: null, latencyMs: null}; + let lastFailure = null; + let lastListLatencyMs = null; const fetchImpl = overrides.fetch || global.fetch; const ensureModelState = (model) => { @@ -92,6 +94,7 @@ export function createModelService(config, overrides = {}) { state.activeRequests += 1; try { + const startedAt = Date.now(); const fullReply = await withTimeout(config.modelRequestTimeoutMs, async (signal) => { const response = await fetchImpl(`${config.modelApiBaseUrl}/chat`, { method: "POST", @@ -106,14 +109,17 @@ export function createModelService(config, overrides = {}) { return readStreamingReply(response); }); - lastHealth = {ok: true, checkedAt: new Date().toISOString(), error: null}; + lastHealth = {ok: true, checkedAt: new Date().toISOString(), error: null, latencyMs: Date.now() - startedAt}; return fullReply; } catch (error) { + const failure = error instanceof ModelServiceError ? error.message : "Model backend unavailable."; lastHealth = { ok: false, checkedAt: new Date().toISOString(), - error: error instanceof ModelServiceError ? error.message : "Model backend unavailable." + error: failure, + latencyMs: null }; + lastFailure = {at: lastHealth.checkedAt, error: failure}; if (error instanceof ModelServiceError) throw error; throw new ModelServiceError("MODEL_UNAVAILABLE", "Model backend unavailable.", 502); } finally { @@ -126,6 +132,7 @@ export function createModelService(config, overrides = {}) { state.activeRequests += 1; try { + const startedAt = Date.now(); const fullReply = await withTimeout(config.modelRequestTimeoutMs, async (signal) => { const response = await fetchImpl(`${config.modelApiBaseUrl}/chat`, { method: "POST", @@ -140,14 +147,17 @@ export function createModelService(config, overrides = {}) { return readStreamingReply(response, onChunk); }); - lastHealth = {ok: true, checkedAt: new Date().toISOString(), error: null}; + lastHealth = {ok: true, checkedAt: new Date().toISOString(), error: null, latencyMs: Date.now() - startedAt}; return fullReply; } catch (error) { + const failure = error instanceof ModelServiceError ? error.message : "Model backend unavailable."; lastHealth = { ok: false, checkedAt: new Date().toISOString(), - error: error instanceof ModelServiceError ? error.message : "Model backend unavailable." + error: failure, + latencyMs: null }; + lastFailure = {at: lastHealth.checkedAt, error: failure}; if (error instanceof ModelServiceError) throw error; throw new ModelServiceError("MODEL_UNAVAILABLE", "Model backend unavailable.", 502); } finally { @@ -161,6 +171,7 @@ export function createModelService(config, overrides = {}) { } try { + const startedAt = Date.now(); modelsCache = await withTimeout(config.modelListTimeoutMs, async (signal) => { const response = await fetchImpl(`${config.modelApiBaseUrl}/tags`, {signal}); if (!response.ok) { @@ -168,30 +179,59 @@ export function createModelService(config, overrides = {}) { } return response.json(); }); + lastListLatencyMs = Date.now() - startedAt; modelsCacheAt = Date.now(); - lastHealth = {ok: true, checkedAt: new Date().toISOString(), error: null}; + lastHealth = {ok: true, checkedAt: new Date().toISOString(), error: null, latencyMs: lastListLatencyMs}; return modelsCache; } catch (error) { + const failure = error instanceof ModelServiceError ? error.message : "Failed to load models."; lastHealth = { ok: false, checkedAt: new Date().toISOString(), - error: error instanceof ModelServiceError ? error.message : "Failed to load models." + error: failure, + latencyMs: null }; + lastFailure = {at: lastHealth.checkedAt, error: failure}; if (error instanceof ModelServiceError) throw error; throw new ModelServiceError("MODEL_UNAVAILABLE", "Failed to load models.", 502); } }, async checkHealth() { + const startedAt = Date.now(); try { await this.listModels(); - return {ok: true, checkedAt: new Date().toISOString(), error: null}; + const health = {ok: true, checkedAt: new Date().toISOString(), error: null, latencyMs: Date.now() - startedAt}; + lastHealth = health; + return health; } catch (error) { - return {ok: false, checkedAt: new Date().toISOString(), error: error.message}; + const health = {ok: false, checkedAt: new Date().toISOString(), error: error.message, latencyMs: null}; + lastHealth = health; + lastFailure = {at: health.checkedAt, error: error.message}; + return health; } }, getLastHealth() { return lastHealth; }, + async getDiagnostics() { + const model = await this.checkHealth(); + const models = Array.isArray(modelsCache?.models) ? modelsCache.models : []; + return { + model: { + ...model, + backendUrl: config.modelApiBaseUrl, + availableModels: models.map((item) => item.name).filter(Boolean), + modelCount: models.length, + listLatencyMs: lastListLatencyMs, + cacheAgeMs: modelsCacheAt ? Date.now() - modelsCacheAt : null, + lastFailure, + activeRequests: Array.from(modelState.entries()).map(([name, state]) => ({ + name, + activeRequests: state.activeRequests + })) + } + }; + }, mapError(error) { if (error instanceof ModelServiceError) { return {status: error.status, body: {error: error.message, code: error.code}}; diff --git a/src/services/workspaceImportService.js b/src/services/workspaceImportService.js new file mode 100644 index 0000000..86f5530 --- /dev/null +++ b/src/services/workspaceImportService.js @@ -0,0 +1,194 @@ +const VALID_PERSONA_TYPES = new Set(["assistant", "user"]); +const VALID_MESSAGE_ROLES = new Set(["user", "bot", "assistant", "system"]); + +const asArray = (value) => Array.isArray(value) ? value : []; +const normalizeText = (value) => String(value ?? "").trim(); +const optionalText = (value) => { + const normalized = normalizeText(value); + return normalized || null; +}; +const normalizeKey = (value) => normalizeText(value).toLowerCase(); + +function countBy(items, getKey) { + const counts = new Map(); + for (const item of items) { + const key = getKey(item); + if (!key) continue; + counts.set(key, (counts.get(key) || 0) + 1); + } + return counts; +} + +function duplicateNames(items, getKey) { + return Array.from(countBy(items, getKey).entries()) + .filter(([, count]) => count > 1) + .map(([key]) => key); +} + +function validatePersona(persona, index, errors) { + if (!persona || typeof persona !== "object" || Array.isArray(persona)) { + errors.push(`Persona ${index + 1} must be an object.`); + return null; + } + const name = normalizeText(persona.name); + const personaType = normalizeText(persona.persona_type || persona.personaType || "assistant"); + if (!name) errors.push(`Persona ${index + 1} is missing a name.`); + if (!VALID_PERSONA_TYPES.has(personaType)) errors.push(`Persona ${index + 1} has invalid persona_type.`); + return { + name, + pronouns: optionalText(persona.pronouns), + appearance: optionalText(persona.appearance), + background: optionalText(persona.background), + details: optionalText(persona.details), + example_dialogues: optionalText(persona.example_dialogues || persona.exampleDialogues), + persona_type: VALID_PERSONA_TYPES.has(personaType) ? personaType : "assistant" + }; +} + +function validateTemplate(template, index, errors) { + if (!template || typeof template !== "object" || Array.isArray(template)) { + errors.push(`Template ${index + 1} must be an object.`); + return null; + } + const name = normalizeText(template.name); + const promptText = normalizeText(template.prompt_text || template.promptText); + if (!name) errors.push(`Template ${index + 1} is missing a name.`); + if (!promptText) errors.push(`Template ${index + 1} is missing prompt_text.`); + return { + name, + description: optionalText(template.description), + category: optionalText(template.category), + prompt_text: promptText, + starter_text: optionalText(template.starter_text || template.starterText), + is_shared: Boolean(template.is_shared || template.isShared) + }; +} + +function validateMessage(message, chatIndex, messageIndex, errors) { + if (!message || typeof message !== "object" || Array.isArray(message)) { + errors.push(`Chat ${chatIndex + 1}, message ${messageIndex + 1} must be an object.`); + return null; + } + const role = normalizeText(message.role); + const content = normalizeText(message.content); + if (!VALID_MESSAGE_ROLES.has(role)) errors.push(`Chat ${chatIndex + 1}, message ${messageIndex + 1} has invalid role.`); + if (!content) errors.push(`Chat ${chatIndex + 1}, message ${messageIndex + 1} is missing content.`); + const retryVariants = Array.isArray(message.retryVariants) + ? message.retryVariants.map((item) => String(item ?? "")).filter(Boolean) + : Array.isArray(message.retry_variants) + ? message.retry_variants.map((item) => String(item ?? "")).filter(Boolean) + : [content].filter(Boolean); + return { + role: role === "assistant" ? "bot" : role, + content, + model_name: optionalText(message.model_name || message.modelName), + retryVariants, + retryActiveIndex: Number.isInteger(Number(message.retryActiveIndex)) ? Number(message.retryActiveIndex) : 0, + retryRetriesUsed: Number.isInteger(Number(message.retryRetriesUsed)) ? Number(message.retryRetriesUsed) : 0, + retryPromptMessageId: Number.isInteger(Number(message.retryPromptMessageId)) ? Number(message.retryPromptMessageId) : null + }; +} + +function validateMemory(memory, chatIndex, memoryIndex, errors) { + if (typeof memory === "string") { + const fact = normalizeText(memory); + if (!fact) errors.push(`Chat ${chatIndex + 1}, memory ${memoryIndex + 1} is empty.`); + return {fact}; + } + if (!memory || typeof memory !== "object" || Array.isArray(memory)) { + errors.push(`Chat ${chatIndex + 1}, memory ${memoryIndex + 1} must be an object.`); + return null; + } + const fact = normalizeText(memory.fact); + if (!fact) errors.push(`Chat ${chatIndex + 1}, memory ${memoryIndex + 1} is missing fact.`); + return {fact}; +} + +function validateChat(chat, index, errors) { + if (!chat || typeof chat !== "object" || Array.isArray(chat)) { + errors.push(`Chat ${index + 1} must be an object.`); + return null; + } + const title = normalizeText(chat.stored_title || chat.title || "Imported chat") || "Imported chat"; + const messages = asArray(chat.messages).map((message, messageIndex) => validateMessage(message, index, messageIndex, errors)).filter(Boolean); + const memories = asArray(chat.memories).map((memory, memoryIndex) => validateMemory(memory, index, memoryIndex, errors)).filter(Boolean); + return { + title, + stored_title: title, + folder_name: optionalText(chat.folder_name || chat.folderName), + is_pinned: Boolean(chat.is_pinned || chat.isPinned), + archived_at: optionalText(chat.archived_at || chat.archivedAt), + scenario_prompt: optionalText(chat.scenario_prompt || chat.scenarioPrompt), + scenario_summary: optionalText(chat.scenario_summary || chat.scenarioSummary), + context_summary: optionalText(chat.context_summary || chat.contextSummary), + context_summary_message_id: Number.isInteger(Number(chat.context_summary_message_id || chat.contextSummaryMessageId)) + ? Number(chat.context_summary_message_id || chat.contextSummaryMessageId) + : 0, + messages, + memories + }; +} + +export function validateWorkspaceImport(workspace, existingWorkspace = null) { + const errors = []; + const warnings = []; + + if (!workspace || typeof workspace !== "object" || Array.isArray(workspace)) { + return { + ok: false, + errors: ["Workspace import must be a JSON object."], + warnings, + preview: {personas: 0, chats: 0, messages: 0, memories: 0, templates: 0, duplicates: []}, + workspace: {personas: [], chats: [], templates: []} + }; + } + + for (const field of ["personas", "chats", "templates"]) { + if (workspace[field] != null && !Array.isArray(workspace[field])) { + errors.push(`workspace.${field} must be an array.`); + } + } + + const sanitized = { + personas: asArray(workspace.personas).map((persona, index) => validatePersona(persona, index, errors)).filter(Boolean), + chats: asArray(workspace.chats).map((chat, index) => validateChat(chat, index, errors)).filter(Boolean), + templates: asArray(workspace.templates).map((template, index) => validateTemplate(template, index, errors)).filter(Boolean) + }; + + const duplicates = [ + ...duplicateNames(sanitized.personas, (persona) => normalizeKey(`${persona.persona_type}:${persona.name}`)).map((name) => `Duplicate persona in import: ${name}`), + ...duplicateNames(sanitized.chats, (chat) => normalizeKey(chat.title)).map((title) => `Duplicate chat title in import: ${title}`), + ...duplicateNames(sanitized.templates, (template) => normalizeKey(template.name)).map((name) => `Duplicate template in import: ${name}`) + ]; + + if (existingWorkspace) { + const existingPersonaKeys = new Set(asArray(existingWorkspace.personas).map((persona) => normalizeKey(`${persona.persona_type}:${persona.name}`))); + const existingChatTitles = new Set(asArray(existingWorkspace.chats).map((chat) => normalizeKey(chat.stored_title || chat.title))); + const existingTemplateNames = new Set(asArray(existingWorkspace.templates).map((template) => normalizeKey(template.name))); + for (const persona of sanitized.personas) { + if (existingPersonaKeys.has(normalizeKey(`${persona.persona_type}:${persona.name}`))) duplicates.push(`Persona already exists: ${persona.name}`); + } + for (const chat of sanitized.chats) { + if (existingChatTitles.has(normalizeKey(chat.title))) duplicates.push(`Chat title already exists: ${chat.title}`); + } + for (const template of sanitized.templates) { + if (existingTemplateNames.has(normalizeKey(template.name))) duplicates.push(`Template already exists: ${template.name}`); + } + } + + warnings.push(...duplicates); + const preview = { + personas: sanitized.personas.length, + chats: sanitized.chats.length, + messages: sanitized.chats.reduce((sum, chat) => sum + chat.messages.length, 0), + memories: sanitized.chats.reduce((sum, chat) => sum + chat.memories.length, 0), + templates: sanitized.templates.length, + duplicates + }; + + if (!preview.personas && !preview.chats && !preview.templates) { + errors.push("Workspace import does not contain any personas, chats, or templates."); + } + + return {ok: errors.length === 0, errors, warnings, preview, workspace: sanitized}; +} diff --git a/test/app.test.js b/test/app.test.js index 0cf7aaa..b4a7e43 100644 --- a/test/app.test.js +++ b/test/app.test.js @@ -46,6 +46,23 @@ async function startTestServer(overrides = {}) { async checkHealth() { return {ok: true, checkedAt: new Date().toISOString(), error: null}; }, + async getDiagnostics() { + return { + model: { + ok: true, + checkedAt: new Date().toISOString(), + error: null, + latencyMs: 12, + backendUrl: "http://model.test", + availableModels: ["mistral:latest"], + modelCount: 1, + listLatencyMs: 12, + cacheAgeMs: 0, + lastFailure: null, + activeRequests: [] + } + }; + }, mapError(error) { return {status: 500, body: {error: error.message, code: "TEST_MODEL"}}; } @@ -166,6 +183,34 @@ test("createApp can use an injected session store", async () => { await server.close(); }); +test("admin diagnostics require admin and return model status", async () => { + const server = await startTestServer({ + config: { + sessionSecret: "change-me-session-secret", + slowRequestLoggingEnabled: false, + eventLoopLagMonitorEnabled: false + } + }); + const userJar = new CookieJar(); + const adminJar = new CookieJar(); + + await request(server.baseUrl, "/register", {method: "POST", body: {username: "alice", password: "password123"}}); + await request(server.baseUrl, "/login", {method: "POST", body: {username: "alice", password: "password123"}, jar: userJar}); + const userDiagnostics = await request(server.baseUrl, "/admin/diagnostics", {jar: userJar}); + assert.equal(userDiagnostics.status, 403); + assert.equal(userDiagnostics.json.error, "Admin only"); + + await request(server.baseUrl, "/register", {method: "POST", body: {username: "admin", password: "password123"}}); + await request(server.baseUrl, "/login", {method: "POST", body: {username: "admin", password: "password123"}, jar: adminJar}); + const diagnostics = await request(server.baseUrl, "/admin/diagnostics", {jar: adminJar}); + assert.equal(diagnostics.status, 200); + assert.equal(diagnostics.json.model.ok, true); + assert.deepEqual(diagnostics.json.model.availableModels, ["mistral:latest"]); + assert.ok(diagnostics.json.configWarnings.some((warning) => warning.includes("SESSION_SECRET"))); + + await server.close(); +}); + test("invalid JSON returns a clean 400 response", async () => { const server = await startTestServer(); @@ -221,6 +266,9 @@ test("file-backed startup creates data and session directories", async () => { async checkHealth() { return {ok: true, checkedAt: new Date().toISOString(), error: null}; }, + async getDiagnostics() { + return {model: {ok: true, availableModels: [], modelCount: 0, activeRequests: []}}; + }, mapError(error) { return {status: 500, body: {error: error.message}}; } @@ -297,6 +345,135 @@ test("chat creation, send, and retry endpoints work with stubbed model service", await server.close(); }); +test("chat memories are durable facts included in model payload", async () => { + let capturedMessages = []; + const modelService = { + async generateReply(model, messagesPayload) { + capturedMessages = messagesPayload; + return "memory-aware reply"; + }, + async streamReply() { + return "unused"; + }, + async listModels() { + return {models: [{name: "mistral:latest"}]}; + }, + async checkHealth() { + return {ok: true, checkedAt: new Date().toISOString(), error: null}; + }, + async getDiagnostics() { + return {model: {ok: true, availableModels: ["mistral:latest"], modelCount: 1, activeRequests: []}}; + }, + mapError(error) { + return {status: 500, body: {error: error.message, code: "TEST_MODEL"}}; + } + }; + const server = await startTestServer({modelService}); + const jar = new CookieJar(); + + await request(server.baseUrl, "/register", {method: "POST", body: {username: "memoryuser", password: "password123"}}); + await request(server.baseUrl, "/login", {method: "POST", body: {username: "memoryuser", password: "password123"}, jar}); + const createChat = await request(server.baseUrl, "/chats", { + method: "POST", + body: {title: "Memory chat"}, + jar + }); + const chatId = createChat.json.chat.id; + + const memory = await request(server.baseUrl, `/chats/${chatId}/memories`, { + method: "POST", + body: {fact: "The user prefers terse answers."}, + jar + }); + assert.equal(memory.status, 200); + assert.equal(memory.json.memory.fact, "The user prefers terse answers."); + + const memories = await request(server.baseUrl, `/chats/${chatId}/memories`, {jar}); + assert.equal(memories.status, 200); + assert.equal(memories.json.memories.length, 1); + + const sendMessage = await request(server.baseUrl, "/chat", { + method: "POST", + body: {chatId, message: "hello", model: "mistral:latest"}, + jar + }); + assert.equal(sendMessage.status, 200); + assert.ok(capturedMessages.some((message) => message.role === "system" && message.content.includes("The user prefers terse answers."))); + + const deleted = await request(server.baseUrl, `/chats/${chatId}/memories/${memory.json.memory.id}`, { + method: "DELETE", + jar + }); + assert.equal(deleted.status, 200); + + await server.close(); +}); + +test("chat context summary rolls up older messages into future payloads", async () => { + let replyCount = 0; + let latestPayload = []; + const modelService = { + async generateReply(model, messagesPayload) { + if (messagesPayload.some((message) => message.content.includes("Merge the previous summary"))) { + return "Summary: user greeted the assistant and asked for continuity."; + } + latestPayload = messagesPayload; + replyCount += 1; + return `reply-${replyCount}`; + }, + async streamReply() { + return "unused"; + }, + async listModels() { + return {models: [{name: "mistral:latest"}]}; + }, + async checkHealth() { + return {ok: true, checkedAt: new Date().toISOString(), error: null}; + }, + async getDiagnostics() { + return {model: {ok: true, availableModels: ["mistral:latest"], modelCount: 1, activeRequests: []}}; + }, + mapError(error) { + return {status: 500, body: {error: error.message, code: "TEST_MODEL"}}; + } + }; + const server = await startTestServer({ + modelService, + config: {chatHistoryLimit: 2, chatSummaryUpdateEveryMessages: 2} + }); + const jar = new CookieJar(); + + await request(server.baseUrl, "/register", {method: "POST", body: {username: "summarized", password: "password123"}}); + await request(server.baseUrl, "/login", {method: "POST", body: {username: "summarized", password: "password123"}, jar}); + const createChat = await request(server.baseUrl, "/chats", { + method: "POST", + body: {title: "Summary chat"}, + jar + }); + const chatId = createChat.json.chat.id; + + await request(server.baseUrl, "/chat", { + method: "POST", + body: {chatId, message: "hello", model: "mistral:latest"}, + jar + }); + await request(server.baseUrl, "/chat", { + method: "POST", + body: {chatId, message: "remember this", model: "mistral:latest"}, + jar + }); + await request(server.baseUrl, "/chat", { + method: "POST", + body: {chatId, message: "what is next?", model: "mistral:latest"}, + jar + }); + + assert.ok(latestPayload.some((message) => message.role === "system" && message.content.includes("CONVERSATION SUMMARY"))); + assert.ok(latestPayload.some((message) => message.content.includes("user greeted the assistant"))); + + await server.close(); +}); + test("roleplay start keeps the session and roleplay messages work", async () => { const server = await startTestServer(); const jar = new CookieJar(); @@ -458,6 +635,50 @@ test("chat search, organization, and workspace export work", async () => { assert.equal(exported.status, 200); assert.equal(exported.json.workspace.chats.length, 1); + const invalidImport = await request(server.baseUrl, "/imports/workspace/preview", { + method: "POST", + body: {workspace: {personas: [{details: "Missing name"}]}}, + jar + }); + assert.equal(invalidImport.status, 400); + assert.match(invalidImport.json.errors[0], /missing a name/i); + + const duplicateImport = await request(server.baseUrl, "/imports/workspace", { + method: "POST", + body: {workspace: exported.json.workspace}, + jar + }); + assert.equal(duplicateImport.status, 409); + assert.match(duplicateImport.json.error, /duplicate/i); + + const workspace = { + personas: [{name: "Imported Guide", persona_type: "assistant", details: "Imported"}], + templates: [{name: "Imported Template", prompt_text: "Say hello"}], + chats: [{ + title: "Imported Notes", + memories: [{fact: "The imported chat remembers the old station."}], + messages: [ + {role: "user", content: "Imported question"}, + {role: "assistant", content: "Imported answer"} + ] + }] + }; + const preview = await request(server.baseUrl, "/imports/workspace/preview", { + method: "POST", + body: {workspace}, + jar + }); + assert.equal(preview.status, 200); + assert.deepEqual(preview.json.preview, {personas: 1, chats: 1, messages: 2, memories: 1, templates: 1, duplicates: []}); + + const imported = await request(server.baseUrl, "/imports/workspace", { + method: "POST", + body: {workspace}, + jar + }); + assert.equal(imported.status, 200); + assert.deepEqual(imported.json.imported, {personas: 1, chats: 1, templates: 1}); + await server.close(); }); diff --git a/test/config.test.js b/test/config.test.js new file mode 100644 index 0000000..898346a --- /dev/null +++ b/test/config.test.js @@ -0,0 +1,65 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import {createConfig} from "../src/config.js"; + +function withEnv(values, fn) { + const previous = {}; + for (const key of Object.keys(values)) { + previous[key] = process.env[key]; + if (values[key] == null) { + delete process.env[key]; + } else { + process.env[key] = values[key]; + } + } + + try { + return fn(); + } finally { + for (const [key, value] of Object.entries(previous)) { + if (value == null) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } + } +} + +test("test mode disables request and lag logging by default", () => { + withEnv({ + NODE_ENV: "test", + TEST_MODE: null, + EVENT_LOOP_LAG_MONITOR_ENABLED: null, + SLOW_REQUEST_LOGGING_ENABLED: null + }, () => { + const config = createConfig(); + + assert.equal(config.testMode, true); + assert.equal(config.eventLoopLagMonitorEnabled, false); + assert.equal(config.slowRequestLoggingEnabled, false); + }); +}); + +test("production rejects the default session secret", () => { + withEnv({ + NODE_ENV: "production", + TEST_MODE: null, + SESSION_SECRET: null, + COOKIE_SECURE: "true" + }, () => { + assert.throws(() => createConfig(), /SESSION_SECRET/); + }); +}); + +test("production requires secure cookies unless explicitly overridden", () => { + withEnv({ + NODE_ENV: "production", + TEST_MODE: null, + SESSION_SECRET: "production-secret", + COOKIE_SECURE: "false", + ALLOW_INSECURE_COOKIES: null + }, () => { + assert.throws(() => createConfig(), /COOKIE_SECURE/); + }); +});