make huge overhauls to backend

src/services/chatService.js

@@ -87,6 +87,89 @@ export function createChatService(repositories, modelService, config) {
         return repositories.getChatMessageByIndex(chatId, safeIndex) || null;
     };
 
+    const buildMemoryPrompt = (memories) => {
+        const facts = (memories || [])
+            .map((memory) => String(memory.fact || "").trim())
+            .filter(Boolean)
+            .slice(0, 30);
+        if (!facts.length) return null;
+        return [
+            "LONG-TERM MEMORY:",
+            "These are durable facts the assistant should remember for this chat/persona context.",
+            "Use them for continuity, but do not mention them unless relevant.",
+            ...facts.map((fact, index) => `${index + 1}. ${fact}`)
+        ].join("\n");
+    };
+
+    const clampSummary = (value) => {
+        const normalized = String(value || "").trim();
+        if (normalized.length <= config.chatSummaryMaxChars) return normalized;
+        return `${normalized.slice(0, Math.max(0, config.chatSummaryMaxChars - 3)).trimEnd()}...`;
+    };
+
+    const buildContextSummaryPrompt = (summary) => {
+        if (!summary) return null;
+        return [
+            "CONVERSATION SUMMARY:",
+            "This is compressed background context from earlier messages in this chat.",
+            "Use it for continuity. Recent raw messages below are more authoritative.",
+            summary
+        ].join("\n");
+    };
+
+    const formatMessagesForSummary = (messages) => messages.map((message) => {
+        const speaker = message.role === "bot" ? "Assistant" : message.role === "user" ? "User" : message.role;
+        return `${speaker}: ${String(message.content || "").trim()}`;
+    }).join("\n\n");
+
+    const maybeUpdateContextSummary = async ({user, chatId, selectedModel}) => {
+        if (config.chatSummaryUpdateEveryMessages <= 0) return;
+        const session = repositories.getChat(chatId, user);
+        if (!session) return;
+        const allMessages = repositories.listChatMessages(chatId);
+        const keepRecent = Math.max(0, config.chatHistoryLimit);
+        const cutoffIndex = allMessages.length - keepRecent - 1;
+        if (cutoffIndex < 0) return;
+        const cutoffMessage = allMessages[cutoffIndex];
+        const summarizable = allMessages
+            .filter((message) => message.id > Number(session.context_summary_message_id || 0) && message.id <= cutoffMessage.id);
+        if (summarizable.length < config.chatSummaryUpdateEveryMessages) return;
+
+        const previousSummary = String(session.context_summary || "").trim();
+        const summary = await modelService.generateReply(selectedModel, [
+            {
+                role: "system",
+                content: [
+                    "You maintain a compact running summary for a chat application.",
+                    "Merge the previous summary and new transcript into one concise continuity summary.",
+                    "Preserve durable facts, decisions, goals, relationships, unresolved threads, locations, preferences, and roleplay continuity.",
+                    "Do not invent details. Do not include generic filler. Keep it under 12 bullet points."
+                ].join("\n")
+            },
+            {
+                role: "user",
+                content: [
+                    previousSummary ? `Previous summary:\n${previousSummary}` : "Previous summary: none",
+                    "",
+                    "New transcript to compress:",
+                    formatMessagesForSummary(summarizable)
+                ].join("\n")
+            }
+        ]);
+
+        const latestSummarized = summarizable[summarizable.length - 1];
+        if (!latestSummarized || !summary) return;
+        repositories.updateChatContextSummary(chatId, user, clampSummary(summary), latestSummarized.id);
+    };
+
+    const updateContextSummaryWithoutBreakingReply = async ({user, chatId, selectedModel}) => {
+        try {
+            await maybeUpdateContextSummary({user, chatId, selectedModel});
+        } catch (error) {
+            console.warn("CHAT SUMMARY UPDATE FAILED", chatId, error?.message || error);
+        }
+    };
+
     const getOrCreatePersonaChat = (user, persona, userPersonaId = null, scenarioPrompt = null, scenarioSummary = null) => {
         const existing = repositories.getChatByParticipants(user, persona.id, userPersonaId);
         if (existing) return existing;
@@ -149,6 +232,7 @@ export function createChatService(repositories, modelService, config) {
         const activePersona = repositories.getAssistantPersonaForChat(chatId, user);
         const activeUserPersona = repositories.getUserPersonaForChat(chatId, user) || repositories.getActiveUserPersona(user);
         const promptMessage = repositories.getPreviousUserMessage(chatId, targetMessage.id);
+        const memoryPrompt = buildMemoryPrompt(repositories.listChatMemories(chatId, user));
 
         if (!promptMessage) {
             if (!activePersona || !session.scenario_summary) {
@@ -182,6 +266,9 @@ export function createChatService(repositories, modelService, config) {
                 })
             });
         }
+        const contextSummaryPrompt = buildContextSummaryPrompt(session.context_summary);
+        if (contextSummaryPrompt) messagesPayload.push({role: "system", content: contextSummaryPrompt});
+        if (memoryPrompt) messagesPayload.push({role: "system", content: memoryPrompt});
         messagesPayload.push(...conversation.map((message) => ({
             role: message.role === "bot" ? "assistant" : "user",
             content: message.content
@@ -227,6 +314,10 @@ export function createChatService(repositories, modelService, config) {
                     })
                 });
             }
+            const contextSummaryPrompt = buildContextSummaryPrompt(session.context_summary);
+            if (contextSummaryPrompt) messagesPayload.push({role: "system", content: contextSummaryPrompt});
+            const memoryPrompt = buildMemoryPrompt(repositories.listChatMemories(chatId, user));
+            if (memoryPrompt) messagesPayload.push({role: "system", content: memoryPrompt});
             messagesPayload.push(...conversation.map((entry) => ({
                 role: entry.role === "bot" ? "assistant" : "user",
                 content: entry.content
@@ -259,6 +350,7 @@ export function createChatService(repositories, modelService, config) {
                 retryPromptMessageId: null
             }, model);
             repositories.touchChat(chatId, user);
+            await updateContextSummaryWithoutBreakingReply({user, chatId, selectedModel: model});
             return {reply: fullReply};
         },
         async streamChatMessage({user, chatId, message, model, onChunk}) {
@@ -288,6 +380,7 @@ export function createChatService(repositories, modelService, config) {
                 retryPromptMessageId: null
             }, model);
             repositories.touchChat(chatId, user);
+            await updateContextSummaryWithoutBreakingReply({user, chatId, selectedModel: model});
             return {reply: fullReply};
         },
         async retryMessage({user, chatId, selectedModel, targetMessage, retryStyle = null}) {

src/services/modelService.js

@@ -12,7 +12,9 @@ export function createModelService(config, overrides = {}) {
     const modelState = new Map();
     let modelsCache = null;
     let modelsCacheAt = 0;
-    let lastHealth = {ok: true, checkedAt: null, error: null};
+    let lastHealth = {ok: true, checkedAt: null, error: null, latencyMs: null};
+    let lastFailure = null;
+    let lastListLatencyMs = null;
     const fetchImpl = overrides.fetch || global.fetch;
 
     const ensureModelState = (model) => {
@@ -92,6 +94,7 @@ export function createModelService(config, overrides = {}) {
             state.activeRequests += 1;
 
             try {
+                const startedAt = Date.now();
                 const fullReply = await withTimeout(config.modelRequestTimeoutMs, async (signal) => {
                     const response = await fetchImpl(`${config.modelApiBaseUrl}/chat`, {
                         method: "POST",
@@ -106,14 +109,17 @@ export function createModelService(config, overrides = {}) {
 
                     return readStreamingReply(response);
                 });
-                lastHealth = {ok: true, checkedAt: new Date().toISOString(), error: null};
+                lastHealth = {ok: true, checkedAt: new Date().toISOString(), error: null, latencyMs: Date.now() - startedAt};
                 return fullReply;
             } catch (error) {
+                const failure = error instanceof ModelServiceError ? error.message : "Model backend unavailable.";
                 lastHealth = {
                     ok: false,
                     checkedAt: new Date().toISOString(),
-                    error: error instanceof ModelServiceError ? error.message : "Model backend unavailable."
+                    error: failure,
+                    latencyMs: null
                 };
+                lastFailure = {at: lastHealth.checkedAt, error: failure};
                 if (error instanceof ModelServiceError) throw error;
                 throw new ModelServiceError("MODEL_UNAVAILABLE", "Model backend unavailable.", 502);
             } finally {
@@ -126,6 +132,7 @@ export function createModelService(config, overrides = {}) {
             state.activeRequests += 1;
 
             try {
+                const startedAt = Date.now();
                 const fullReply = await withTimeout(config.modelRequestTimeoutMs, async (signal) => {
                     const response = await fetchImpl(`${config.modelApiBaseUrl}/chat`, {
                         method: "POST",
@@ -140,14 +147,17 @@ export function createModelService(config, overrides = {}) {
 
                     return readStreamingReply(response, onChunk);
                 });
-                lastHealth = {ok: true, checkedAt: new Date().toISOString(), error: null};
+                lastHealth = {ok: true, checkedAt: new Date().toISOString(), error: null, latencyMs: Date.now() - startedAt};
                 return fullReply;
             } catch (error) {
+                const failure = error instanceof ModelServiceError ? error.message : "Model backend unavailable.";
                 lastHealth = {
                     ok: false,
                     checkedAt: new Date().toISOString(),
-                    error: error instanceof ModelServiceError ? error.message : "Model backend unavailable."
+                    error: failure,
+                    latencyMs: null
                 };
+                lastFailure = {at: lastHealth.checkedAt, error: failure};
                 if (error instanceof ModelServiceError) throw error;
                 throw new ModelServiceError("MODEL_UNAVAILABLE", "Model backend unavailable.", 502);
             } finally {
@@ -161,6 +171,7 @@ export function createModelService(config, overrides = {}) {
             }
 
             try {
+                const startedAt = Date.now();
                 modelsCache = await withTimeout(config.modelListTimeoutMs, async (signal) => {
                     const response = await fetchImpl(`${config.modelApiBaseUrl}/tags`, {signal});
                     if (!response.ok) {
@@ -168,30 +179,59 @@ export function createModelService(config, overrides = {}) {
                     }
                     return response.json();
                 });
+                lastListLatencyMs = Date.now() - startedAt;
                 modelsCacheAt = Date.now();
-                lastHealth = {ok: true, checkedAt: new Date().toISOString(), error: null};
+                lastHealth = {ok: true, checkedAt: new Date().toISOString(), error: null, latencyMs: lastListLatencyMs};
                 return modelsCache;
             } catch (error) {
+                const failure = error instanceof ModelServiceError ? error.message : "Failed to load models.";
                 lastHealth = {
                     ok: false,
                     checkedAt: new Date().toISOString(),
-                    error: error instanceof ModelServiceError ? error.message : "Failed to load models."
+                    error: failure,
+                    latencyMs: null
                 };
+                lastFailure = {at: lastHealth.checkedAt, error: failure};
                 if (error instanceof ModelServiceError) throw error;
                 throw new ModelServiceError("MODEL_UNAVAILABLE", "Failed to load models.", 502);
             }
         },
         async checkHealth() {
+            const startedAt = Date.now();
             try {
                 await this.listModels();
-                return {ok: true, checkedAt: new Date().toISOString(), error: null};
+                const health = {ok: true, checkedAt: new Date().toISOString(), error: null, latencyMs: Date.now() - startedAt};
+                lastHealth = health;
+                return health;
             } catch (error) {
-                return {ok: false, checkedAt: new Date().toISOString(), error: error.message};
+                const health = {ok: false, checkedAt: new Date().toISOString(), error: error.message, latencyMs: null};
+                lastHealth = health;
+                lastFailure = {at: health.checkedAt, error: error.message};
+                return health;
             }
         },
         getLastHealth() {
             return lastHealth;
         },
+        async getDiagnostics() {
+            const model = await this.checkHealth();
+            const models = Array.isArray(modelsCache?.models) ? modelsCache.models : [];
+            return {
+                model: {
+                    ...model,
+                    backendUrl: config.modelApiBaseUrl,
+                    availableModels: models.map((item) => item.name).filter(Boolean),
+                    modelCount: models.length,
+                    listLatencyMs: lastListLatencyMs,
+                    cacheAgeMs: modelsCacheAt ? Date.now() - modelsCacheAt : null,
+                    lastFailure,
+                    activeRequests: Array.from(modelState.entries()).map(([name, state]) => ({
+                        name,
+                        activeRequests: state.activeRequests
+                    }))
+                }
+            };
+        },
         mapError(error) {
             if (error instanceof ModelServiceError) {
                 return {status: error.status, body: {error: error.message, code: error.code}};

src/services/workspaceImportService.js

@@ -0,0 +1,194 @@
+const VALID_PERSONA_TYPES = new Set(["assistant", "user"]);
+const VALID_MESSAGE_ROLES = new Set(["user", "bot", "assistant", "system"]);
+
+const asArray = (value) => Array.isArray(value) ? value : [];
+const normalizeText = (value) => String(value ?? "").trim();
+const optionalText = (value) => {
+    const normalized = normalizeText(value);
+    return normalized || null;
+};
+const normalizeKey = (value) => normalizeText(value).toLowerCase();
+
+function countBy(items, getKey) {
+    const counts = new Map();
+    for (const item of items) {
+        const key = getKey(item);
+        if (!key) continue;
+        counts.set(key, (counts.get(key) || 0) + 1);
+    }
+    return counts;
+}
+
+function duplicateNames(items, getKey) {
+    return Array.from(countBy(items, getKey).entries())
+        .filter(([, count]) => count > 1)
+        .map(([key]) => key);
+}
+
+function validatePersona(persona, index, errors) {
+    if (!persona || typeof persona !== "object" || Array.isArray(persona)) {
+        errors.push(`Persona ${index + 1} must be an object.`);
+        return null;
+    }
+    const name = normalizeText(persona.name);
+    const personaType = normalizeText(persona.persona_type || persona.personaType || "assistant");
+    if (!name) errors.push(`Persona ${index + 1} is missing a name.`);
+    if (!VALID_PERSONA_TYPES.has(personaType)) errors.push(`Persona ${index + 1} has invalid persona_type.`);
+    return {
+        name,
+        pronouns: optionalText(persona.pronouns),
+        appearance: optionalText(persona.appearance),
+        background: optionalText(persona.background),
+        details: optionalText(persona.details),
+        example_dialogues: optionalText(persona.example_dialogues || persona.exampleDialogues),
+        persona_type: VALID_PERSONA_TYPES.has(personaType) ? personaType : "assistant"
+    };
+}
+
+function validateTemplate(template, index, errors) {
+    if (!template || typeof template !== "object" || Array.isArray(template)) {
+        errors.push(`Template ${index + 1} must be an object.`);
+        return null;
+    }
+    const name = normalizeText(template.name);
+    const promptText = normalizeText(template.prompt_text || template.promptText);
+    if (!name) errors.push(`Template ${index + 1} is missing a name.`);
+    if (!promptText) errors.push(`Template ${index + 1} is missing prompt_text.`);
+    return {
+        name,
+        description: optionalText(template.description),
+        category: optionalText(template.category),
+        prompt_text: promptText,
+        starter_text: optionalText(template.starter_text || template.starterText),
+        is_shared: Boolean(template.is_shared || template.isShared)
+    };
+}
+
+function validateMessage(message, chatIndex, messageIndex, errors) {
+    if (!message || typeof message !== "object" || Array.isArray(message)) {
+        errors.push(`Chat ${chatIndex + 1}, message ${messageIndex + 1} must be an object.`);
+        return null;
+    }
+    const role = normalizeText(message.role);
+    const content = normalizeText(message.content);
+    if (!VALID_MESSAGE_ROLES.has(role)) errors.push(`Chat ${chatIndex + 1}, message ${messageIndex + 1} has invalid role.`);
+    if (!content) errors.push(`Chat ${chatIndex + 1}, message ${messageIndex + 1} is missing content.`);
+    const retryVariants = Array.isArray(message.retryVariants)
+        ? message.retryVariants.map((item) => String(item ?? "")).filter(Boolean)
+        : Array.isArray(message.retry_variants)
+            ? message.retry_variants.map((item) => String(item ?? "")).filter(Boolean)
+            : [content].filter(Boolean);
+    return {
+        role: role === "assistant" ? "bot" : role,
+        content,
+        model_name: optionalText(message.model_name || message.modelName),
+        retryVariants,
+        retryActiveIndex: Number.isInteger(Number(message.retryActiveIndex)) ? Number(message.retryActiveIndex) : 0,
+        retryRetriesUsed: Number.isInteger(Number(message.retryRetriesUsed)) ? Number(message.retryRetriesUsed) : 0,
+        retryPromptMessageId: Number.isInteger(Number(message.retryPromptMessageId)) ? Number(message.retryPromptMessageId) : null
+    };
+}
+
+function validateMemory(memory, chatIndex, memoryIndex, errors) {
+    if (typeof memory === "string") {
+        const fact = normalizeText(memory);
+        if (!fact) errors.push(`Chat ${chatIndex + 1}, memory ${memoryIndex + 1} is empty.`);
+        return {fact};
+    }
+    if (!memory || typeof memory !== "object" || Array.isArray(memory)) {
+        errors.push(`Chat ${chatIndex + 1}, memory ${memoryIndex + 1} must be an object.`);
+        return null;
+    }
+    const fact = normalizeText(memory.fact);
+    if (!fact) errors.push(`Chat ${chatIndex + 1}, memory ${memoryIndex + 1} is missing fact.`);
+    return {fact};
+}
+
+function validateChat(chat, index, errors) {
+    if (!chat || typeof chat !== "object" || Array.isArray(chat)) {
+        errors.push(`Chat ${index + 1} must be an object.`);
+        return null;
+    }
+    const title = normalizeText(chat.stored_title || chat.title || "Imported chat") || "Imported chat";
+    const messages = asArray(chat.messages).map((message, messageIndex) => validateMessage(message, index, messageIndex, errors)).filter(Boolean);
+    const memories = asArray(chat.memories).map((memory, memoryIndex) => validateMemory(memory, index, memoryIndex, errors)).filter(Boolean);
+    return {
+        title,
+        stored_title: title,
+        folder_name: optionalText(chat.folder_name || chat.folderName),
+        is_pinned: Boolean(chat.is_pinned || chat.isPinned),
+        archived_at: optionalText(chat.archived_at || chat.archivedAt),
+        scenario_prompt: optionalText(chat.scenario_prompt || chat.scenarioPrompt),
+        scenario_summary: optionalText(chat.scenario_summary || chat.scenarioSummary),
+        context_summary: optionalText(chat.context_summary || chat.contextSummary),
+        context_summary_message_id: Number.isInteger(Number(chat.context_summary_message_id || chat.contextSummaryMessageId))
+            ? Number(chat.context_summary_message_id || chat.contextSummaryMessageId)
+            : 0,
+        messages,
+        memories
+    };
+}
+
+export function validateWorkspaceImport(workspace, existingWorkspace = null) {
+    const errors = [];
+    const warnings = [];
+
+    if (!workspace || typeof workspace !== "object" || Array.isArray(workspace)) {
+        return {
+            ok: false,
+            errors: ["Workspace import must be a JSON object."],
+            warnings,
+            preview: {personas: 0, chats: 0, messages: 0, memories: 0, templates: 0, duplicates: []},
+            workspace: {personas: [], chats: [], templates: []}
+        };
+    }
+
+    for (const field of ["personas", "chats", "templates"]) {
+        if (workspace[field] != null && !Array.isArray(workspace[field])) {
+            errors.push(`workspace.${field} must be an array.`);
+        }
+    }
+
+    const sanitized = {
+        personas: asArray(workspace.personas).map((persona, index) => validatePersona(persona, index, errors)).filter(Boolean),
+        chats: asArray(workspace.chats).map((chat, index) => validateChat(chat, index, errors)).filter(Boolean),
+        templates: asArray(workspace.templates).map((template, index) => validateTemplate(template, index, errors)).filter(Boolean)
+    };
+
+    const duplicates = [
+        ...duplicateNames(sanitized.personas, (persona) => normalizeKey(`${persona.persona_type}:${persona.name}`)).map((name) => `Duplicate persona in import: ${name}`),
+        ...duplicateNames(sanitized.chats, (chat) => normalizeKey(chat.title)).map((title) => `Duplicate chat title in import: ${title}`),
+        ...duplicateNames(sanitized.templates, (template) => normalizeKey(template.name)).map((name) => `Duplicate template in import: ${name}`)
+    ];
+
+    if (existingWorkspace) {
+        const existingPersonaKeys = new Set(asArray(existingWorkspace.personas).map((persona) => normalizeKey(`${persona.persona_type}:${persona.name}`)));
+        const existingChatTitles = new Set(asArray(existingWorkspace.chats).map((chat) => normalizeKey(chat.stored_title || chat.title)));
+        const existingTemplateNames = new Set(asArray(existingWorkspace.templates).map((template) => normalizeKey(template.name)));
+        for (const persona of sanitized.personas) {
+            if (existingPersonaKeys.has(normalizeKey(`${persona.persona_type}:${persona.name}`))) duplicates.push(`Persona already exists: ${persona.name}`);
+        }
+        for (const chat of sanitized.chats) {
+            if (existingChatTitles.has(normalizeKey(chat.title))) duplicates.push(`Chat title already exists: ${chat.title}`);
+        }
+        for (const template of sanitized.templates) {
+            if (existingTemplateNames.has(normalizeKey(template.name))) duplicates.push(`Template already exists: ${template.name}`);
+        }
+    }
+
+    warnings.push(...duplicates);
+    const preview = {
+        personas: sanitized.personas.length,
+        chats: sanitized.chats.length,
+        messages: sanitized.chats.reduce((sum, chat) => sum + chat.messages.length, 0),
+        memories: sanitized.chats.reduce((sum, chat) => sum + chat.memories.length, 0),
+        templates: sanitized.templates.length,
+        duplicates
+    };
+
+    if (!preview.personas && !preview.chats && !preview.templates) {
+        errors.push("Workspace import does not contain any personas, chats, or templates.");
+    }
+
+    return {ok: errors.length === 0, errors, warnings, preview, workspace: sanitized};
+}