add vision analysis service to dev branch?

src/routes/visionDebug.js

@@ -0,0 +1,16 @@
+import express from "express";
+import {requireLogin} from "../middleware/auth.js";
+
+export function createVisionDebugRouter({visionDebugService, repositories}) {
+    const router = express.Router();
+    router.use(requireLogin);
+    router.get("/debug/vision/latest", (req, res) => {
+        const diagnostic = visionDebugService.latest(req.session.user, repositories);
+        return diagnostic ? res.json({diagnostic}) : res.status(404).json({error: "Vision diagnostic not found"});
+    });
+    router.get("/debug/vision/:requestId", (req, res) => {
+        const diagnostic = visionDebugService.get(req.params.requestId, req.session.user, repositories);
+        return diagnostic ? res.json({diagnostic}) : res.status(404).json({error: "Vision diagnostic not found"});
+    });
+    return router;
+}

src/services/visionDebugService.js

@@ -0,0 +1,180 @@
+const RAW_VISION_LIMIT = 12000;
+const EVIDENCE_LIMIT = 100000;
+const ANSWER_LIMIT = 30000;
+const REASONING_LIMIT = 30000;
+const MAX_RECORDS = 100;
+const MAX_AGE_MS = 60 * 60 * 1000;
+
+const bounded = (value, limit) => typeof value === "string" ? value.slice(0, limit) : value;
+const clone = value => value == null ? value : structuredClone(value);
+const exposed = record => {
+    const value = clone(record);
+    delete value.user;
+    return value;
+};
+
+export function createVisionDebugService(config) {
+    const enabled = config.visionDebug === true;
+    const records = new Map();
+
+    const prune = () => {
+        const cutoff = Date.now() - MAX_AGE_MS;
+        for (const [id, record] of records) {
+            if (Date.parse(record.updatedAt) < cutoff) records.delete(id);
+        }
+        while (records.size > MAX_RECORDS) records.delete(records.keys().next().value);
+    };
+    const update = (requestId, operation) => {
+        if (!enabled || !requestId) return null;
+        const record = records.get(requestId);
+        if (!record) return null;
+        operation(record);
+        record.updatedAt = new Date().toISOString();
+        return record;
+    };
+    const image = (record, attachmentId) => record.images.find(item => item.attachmentId === attachmentId);
+
+    return {
+        enabled,
+        begin({requestId, user, chatId, triggeringMessageId, userQuestion, selectedModel, reasoningRequested, webSearchRequested}) {
+            if (!enabled || !requestId) return;
+            prune();
+            const timestamp = new Date().toISOString();
+            records.delete(requestId);
+            records.set(requestId, {
+                requestId, user, chatId, triggeringMessageId,
+                createdAt: timestamp, updatedAt: timestamp,
+                request: {userQuestion, selectedModel, reasoningRequested: reasoningRequested === true, webSearchRequested: webSearchRequested === true},
+                images: [],
+                qwen: null,
+                search: {
+                    requested: webSearchRequested === true,
+                    needed: null,
+                    explicitSearchIntent: null,
+                    state: webSearchRequested === true ? "pending" : "disabled",
+                    visualReference: "",
+                    queries: [],
+                    candidateCount: 0,
+                    selectedCount: 0,
+                    selectedSources: []
+                },
+                visibleReasoning: null,
+                finalAnswer: null,
+                error: null,
+                experimentNote: "For controlled Web Search/Reasoning comparisons, reuse this attachment and exact question. Re-uploading identical bytes creates a new attachment ID and vision cache entry."
+            });
+            prune();
+        },
+        recordImage(requestId, value) {
+            update(requestId, record => {
+                const existing = image(record, value.attachmentId);
+                const next = {
+                    requestId,
+                    chatId: record.chatId,
+                    triggeringMessageId: record.triggeringMessageId,
+                    attachmentId: value.attachmentId,
+                    sourceMessageId: value.sourceMessageId,
+                    position: value.position,
+                    scope: value.scope,
+                    label: value.label,
+                    originalFilename: value.originalFilename,
+                    mimeType: value.mimeType,
+                    width: value.width,
+                    height: value.height,
+                    sizeBytes: value.sizeBytes,
+                    imageSha256: null,
+                    visionModel: value.visionModel,
+                    visionPromptVersion: value.visionPromptVersion,
+                    normalizedQuestion: value.normalizedQuestion,
+                    cacheHit: null,
+                    cacheIdentity: null,
+                    rawVisionResponse: null,
+                    normalizedVisionAnalysis: null
+                };
+                if (existing) Object.assign(existing, next); else record.images.push(next);
+            });
+        },
+        recordCache(requestId, attachmentId, {cacheHit, cacheIdentity, normalizedVisionAnalysis}) {
+            update(requestId, record => {
+                const entry = image(record, attachmentId);
+                if (!entry) return;
+                entry.cacheHit = cacheHit === true;
+                entry.cacheIdentity = clone(cacheIdentity);
+                entry.rawVisionResponse = null;
+                if (normalizedVisionAnalysis !== undefined) entry.normalizedVisionAnalysis = clone(normalizedVisionAnalysis);
+            });
+        },
+        recordVisionRun(requestId, attachmentId, {imageSha256, rawVisionResponse, normalizedVisionAnalysis}) {
+            update(requestId, record => {
+                const entry = image(record, attachmentId);
+                if (!entry) return;
+                entry.cacheHit = false;
+                entry.imageSha256 = imageSha256 || null;
+                entry.rawVisionResponse = bounded(String(rawVisionResponse ?? ""), RAW_VISION_LIMIT);
+                entry.normalizedVisionAnalysis = clone(normalizedVisionAnalysis);
+            });
+        },
+        recordQwen(requestId, value) {
+            update(requestId, record => {
+                record.qwen = {
+                    imageEvidence: bounded(value.imageEvidence, EVIDENCE_LIMIT),
+                    visionGuidanceSha256: value.visionGuidanceSha256,
+                    visionEvidenceIncluded: value.visionEvidenceIncluded === true,
+                    imageCount: value.imageCount,
+                    currentImageLabels: clone(value.currentImageLabels),
+                    model: value.model,
+                    think: value.think === true,
+                    stream: value.stream === true,
+                    temperature: value.temperature ?? null,
+                    numCtx: value.numCtx ?? null,
+                    numPredict: value.numPredict ?? null,
+                    keepAlive: value.keepAlive
+                };
+            });
+        },
+        recordSearch(requestId, value) {
+            update(requestId, record => {
+                record.search = {
+                    ...record.search,
+                    requested: value.requested === true,
+                    needed: value.needed ?? record.search.needed,
+                    explicitSearchIntent: value.explicitSearchIntent ?? record.search.explicitSearchIntent,
+                    state: value.state ?? record.search.state,
+                    visualReference: bounded(value.visualReference ?? record.search.visualReference, 1200),
+                    queries: Array.isArray(value.queries) ? value.queries.map(item => bounded(String(item), 500)) : record.search.queries,
+                    candidateCount: value.candidateCount ?? record.search.candidateCount,
+                    selectedCount: value.selectedCount ?? record.search.selectedCount,
+                    selectedSources: Array.isArray(value.selectedSources) ? value.selectedSources.slice(0, 5).map((source, index) => ({
+                        index: index + 1,
+                        title: bounded(String(source.title || ""), 300),
+                        url: bounded(String(source.url || ""), 2048),
+                        snippet: bounded(String(source.snippet || ""), 500)
+                    })) : record.search.selectedSources
+                };
+            });
+        },
+        complete(requestId, {finalAnswer, visibleReasoning}) {
+            update(requestId, record => {
+                record.finalAnswer = bounded(String(finalAnswer ?? ""), ANSWER_LIMIT);
+                record.visibleReasoning = visibleReasoning ? bounded(String(visibleReasoning), REASONING_LIMIT) : null;
+            });
+        },
+        fail(requestId, error) {
+            update(requestId, record => { record.error = bounded(String(error?.message || error || "Request failed"), 1000); });
+        },
+        get(requestId, user, repositories) {
+            if (!enabled) return null;
+            prune();
+            const record = records.get(requestId);
+            if (!record || record.user !== user || !repositories.getChat(record.chatId, user)) return null;
+            return exposed(record);
+        },
+        latest(user, repositories) {
+            if (!enabled) return null;
+            prune();
+            const values = [...records.values()].reverse();
+            const record = values.find(item => item.user === user && repositories.getChat(item.chatId, user));
+            return record ? exposed(record) : null;
+        }
+    };
+}

test/visionDiagnostics.test.js

@@ -0,0 +1,206 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import {createServer} from "node:http";
+import sharp from "sharp";
+import {createApp} from "../src/app.js";
+import {VISION_PROMPT_VERSION} from "../src/services/visionAnalysis.js";
+
+async function listen(t, handler) {
+    const server = createServer(handler);
+    await new Promise(resolve => server.listen(0, "127.0.0.1", resolve));
+    t.after(() => new Promise(resolve => { server.closeAllConnections(); server.close(resolve); }));
+    return {server, url: `http://127.0.0.1:${server.address().port}`};
+}
+
+async function setup(t, visionDebug = true) {
+    const directory = await fs.mkdtemp(path.join(os.tmpdir(), "elsewhere-vision-debug-"));
+    const requests = [];
+    const searchQueries = [];
+    const rawVisionResponse = ` \n${JSON.stringify({
+        scene: "A penguin stands on dark rocks.",
+        subjects: [{type: "bird", description: "Black crown and white throat", location: "center"}],
+        visualDetails: ["A thin black line runs directly beneath the chin."],
+        visibleText: [], spatialRelationships: [], colorsAndAppearance: ["black and white plumage"],
+        recognizedElements: ["penguin"], notableFeatures: ["thin black line beneath chin"], uncertainties: []
+    })}\n `;
+    const search = await listen(t, (req, res) => {
+        searchQueries.push(new URL(req.url, "http://localhost").searchParams.get("q"));
+        res.setHeader("Content-Type", "application/json");
+        res.end(JSON.stringify({results: [{title: "Penguin identification guide", url: "https://example.com/penguins",
+            content: "A penguin identification guide compares crown, throat, bill, and chin markings."}]}));
+    });
+    const app = createApp({config: {
+        dbPath: ":memory:", uploadDir: directory, testMode: true, sessionSecret: "vision-debug-tests",
+        visionDebug, visionModel: "qwen2.5vl:7b", chatSummaryUpdateEveryMessages: 0,
+        searxngUrl: search.url, eventLoopLagMonitorEnabled: false,
+        slowRequestLoggingEnabled: false, structuredRequestLoggingEnabled: false
+    }, modelOverrides: {fetch: async (_url, options) => {
+        const body = JSON.parse(options.body);
+        requests.push(body);
+        const chunks = body.model === "qwen2.5vl:7b"
+            ? [{message: {content: rawVisionResponse}, done: true}]
+            : [...(body.think ? [{message: {thinking: "Compare the supplied visible evidence."}}] : []),
+                {message: {content: "Final penguin answer."}, done: true}];
+        return new Response(chunks.map(chunk => JSON.stringify(chunk)).join("\n"));
+    }}});
+    const backend = await listen(t, app);
+    t.after(async () => { await app.locals.close(); await fs.rm(directory, {recursive: true, force: true}); });
+    const register = async username => {
+        const response = await fetch(backend.url + "/register", {method: "POST", headers: {"Content-Type": "application/json"},
+            body: JSON.stringify({username, password: "test-password"})});
+        assert.equal(response.status, 200);
+        return response.headers.get("set-cookie").split(";")[0];
+    };
+    const request = (url, {cookie = "", method = "GET", body, headers = {}} = {}) => fetch(backend.url + url, {
+        method, headers: {...headers, ...(cookie ? {Cookie: cookie} : {}), ...(body ? {"Content-Type": "application/json"} : {})},
+        body: body ? JSON.stringify(body) : undefined
+    });
+    const upload = async (cookie, bytes, filename = "penguin.png") => {
+        const form = new FormData();
+        form.append("image", new Blob([bytes], {type: "image/png"}), filename);
+        const response = await fetch(backend.url + "/uploads", {method: "POST", headers: {Cookie: cookie}, body: form});
+        assert.equal(response.status, 201);
+        return response.json();
+    };
+    const createChat = async cookie => {
+        const response = await request("/chats", {cookie, method: "POST", body: {title: "Vision debug"}});
+        return (await response.json()).chat.id;
+    };
+    return {app, backend, register, request, upload, createChat, requests, searchQueries, rawVisionResponse};
+}
+
+const getDiagnostic = async (s, cookie, requestId) => {
+    const response = await s.request(`/debug/vision/${requestId}`, {cookie});
+    assert.equal(response.status, 200);
+    return (await response.json()).diagnostic;
+};
+
+test("vision diagnostics routes are absent when disabled and protected when enabled", async t => {
+    const disabled = await setup(t, false);
+    assert.equal(disabled.app.locals.config.visionDebug, false);
+    const disabledUser = await disabled.register("debug-disabled");
+    assert.equal((await disabled.request("/debug/vision/latest", {cookie: disabledUser})).status, 404);
+
+    const enabled = await setup(t, true);
+    assert.equal((await enabled.request("/debug/vision/latest")).status, 403);
+});
+
+test("vision diagnostics capture misses, hits, Qwen evidence, search, reasoning and ownership safely", async t => {
+    const s = await setup(t, true);
+    const owner = await s.register("debug-owner");
+    const other = await s.register("debug-other");
+    const chatId = await s.createChat(owner);
+    const bytes = await sharp({create: {width: 24, height: 12, channels: 3, background: "black"}}).png().toBuffer();
+    const attachment = await s.upload(owner, bytes);
+    const question = "Search the web to identify this penguin.";
+
+    const first = await s.request("/chat", {cookie: owner, method: "POST", body: {
+        chatId, message: question, attachments: [attachment.id], model: "qwen3:14b", webSearch: false, reasoning: false
+    }});
+    assert.equal(first.status, 200);
+    const firstRequestId = first.headers.get("x-request-id");
+    const miss = await getDiagnostic(s, owner, firstRequestId);
+    assert.equal((await s.request(`/debug/vision/${firstRequestId}`, {cookie: other})).status, 404);
+    assert.equal(miss.requestId, firstRequestId);
+    assert.equal(miss.chatId, chatId);
+    assert.equal(miss.user, undefined);
+    assert.equal(miss.images.length, 1);
+    const image = miss.images[0];
+    assert.equal(image.attachmentId, attachment.id);
+    assert.equal(image.position, 0);
+    assert.equal(image.scope, "current");
+    assert.equal(image.label, "Current image 1");
+    assert.equal(image.originalFilename, "penguin.png");
+    assert.equal(image.mimeType, "image/png");
+    assert.equal(image.width, 24); assert.equal(image.height, 12); assert.equal(image.sizeBytes, bytes.length);
+    assert.equal(image.visionModel, "qwen2.5vl:7b");
+    assert.equal(image.visionPromptVersion, VISION_PROMPT_VERSION);
+    assert.equal(image.normalizedQuestion, question);
+    assert.equal(image.cacheHit, false);
+    assert.match(image.imageSha256, /^[a-f0-9]{64}$/);
+    assert.equal(image.rawVisionResponse, s.rawVisionResponse);
+    assert.equal(image.normalizedVisionAnalysis.scene, "A penguin stands on dark rocks.");
+    assert.equal(miss.qwen.visionEvidenceIncluded, true);
+    assert.equal(miss.qwen.imageCount, 1);
+    assert.deepEqual(miss.qwen.currentImageLabels, ["Current image 1"]);
+    assert.equal(miss.qwen.model, "qwen3:14b");
+    assert.equal(miss.qwen.think, false);
+    assert.equal(miss.qwen.stream, true);
+    assert.equal(miss.qwen.keepAlive, "30m");
+    const finalRequest = s.requests.findLast(request => request.model === "qwen3:14b");
+    assert.equal(miss.qwen.imageEvidence, finalRequest.messages.find(message => message.content.startsWith("<image_evidence>")).content);
+    assert.match(miss.qwen.visionGuidanceSha256, /^[a-f0-9]{64}$/);
+    assert.equal(miss.search.state, "disabled");
+    assert.equal(miss.visibleReasoning, null);
+    assert.equal(miss.finalAnswer, "Final penguin answer.");
+    const uploadedBase64 = s.requests.find(request => request.model === "qwen2.5vl:7b").messages.at(-1).images[0];
+    const diagnosticText = JSON.stringify(miss);
+    assert.equal(diagnosticText.includes(uploadedBase64), false);
+    assert.equal(diagnosticText.includes(bytes.toString("latin1")), false);
+
+    const messagesResponse = await s.request(`/chats/${chatId}/messages`, {cookie: owner});
+    const bot = (await messagesResponse.json()).messages.at(-1);
+    const retry = await s.request(`/chats/${chatId}/messages/${bot.id}/retry`, {cookie: owner, method: "POST", body: {
+        model: "qwen3:14b", webSearch: true, reasoning: true
+    }});
+    assert.equal(retry.status, 200);
+    const hit = await getDiagnostic(s, owner, retry.headers.get("x-request-id"));
+    assert.equal(hit.images[0].cacheHit, true);
+    assert.deepEqual(hit.images[0].cacheIdentity, {attachmentId: attachment.id, visionModel: "qwen2.5vl:7b",
+        visionPromptVersion: VISION_PROMPT_VERSION, normalizedQuestion: question});
+    assert.equal(hit.images[0].rawVisionResponse, null);
+    assert.equal(hit.images[0].imageSha256, null);
+    assert.deepEqual(hit.images[0].normalizedVisionAnalysis, image.normalizedVisionAnalysis);
+    assert.equal(hit.qwen.imageEvidence, miss.qwen.imageEvidence);
+    assert.equal(hit.qwen.think, true);
+    assert.equal(hit.visibleReasoning, "Compare the supplied visible evidence.");
+    assert.equal(hit.search.requested, true);
+    assert.equal(hit.search.needed, true);
+    assert.equal(hit.search.explicitSearchIntent, true);
+    assert.equal(hit.search.state, "succeeded");
+    assert.match(hit.search.visualReference, /thin black line beneath chin/i);
+    assert.deepEqual(hit.search.queries, s.searchQueries);
+    assert.equal(hit.search.candidateCount, 1);
+    assert.equal(hit.search.selectedCount, 1);
+    assert.deepEqual(hit.search.selectedSources[0], {index: 1, title: "Penguin identification guide",
+        url: "https://example.com/penguins", snippet: "A penguin identification guide compares crown, throat, bill, and chin markings."});
+    assert.equal(hit.finalAnswer, "Final penguin answer.");
+    const latest = await s.request("/debug/vision/latest", {cookie: owner});
+    assert.equal((await latest.json()).diagnostic.requestId, hit.requestId);
+});
+
+test("diagnostics distinguish skipped search and preserve multiple and historical image provenance", async t => {
+    const s = await setup(t, true);
+    const cookie = await s.register("debug-provenance");
+    const chatId = await s.createChat(cookie);
+    const red = await sharp({create: {width: 20, height: 10, channels: 3, background: "red"}}).png().toBuffer();
+    const blue = await sharp({create: {width: 30, height: 15, channels: 3, background: "blue"}}).png().toBuffer();
+    const first = await s.upload(cookie, red, "first.png");
+    const second = await s.upload(cookie, blue, "second.png");
+    const comparison = await s.request("/chat", {cookie, method: "POST", body: {
+        chatId, message: "What species are these?", attachments: [first.id, second.id], webSearch: true
+    }});
+    const current = await getDiagnostic(s, cookie, comparison.headers.get("x-request-id"));
+    assert.deepEqual(current.images.map(image => [image.attachmentId, image.position, image.scope, image.label]), [
+        [first.id, 0, "current", "Current image 1"], [second.id, 1, "current", "Current image 2"]
+    ]);
+    assert.notEqual(current.images[0].imageSha256, current.images[1].imageSha256);
+    assert.equal(current.search.requested, true);
+    assert.equal(current.search.needed, false);
+    assert.equal(current.search.state, "skipped");
+    assert.deepEqual(current.search.queries, []);
+    assert.equal(current.search.candidateCount, 0);
+
+    const historicalResponse = await s.request("/chat", {cookie, method: "POST", body: {
+        chatId, message: "What did the previous image show?"
+    }});
+    const historical = await getDiagnostic(s, cookie, historicalResponse.headers.get("x-request-id"));
+    assert.equal(historical.images.length, 1);
+    assert.equal(historical.images[0].attachmentId, second.id);
+    assert.equal(historical.images[0].position, 1);
+    assert.equal(historical.images[0].scope, "historical");
+    assert.match(historical.images[0].label, /Earlier image from message/);
+});