diff --git a/src/routes/visionDebug.js b/src/routes/visionDebug.js new file mode 100644 index 0000000..4135e16 --- /dev/null +++ b/src/routes/visionDebug.js @@ -0,0 +1,16 @@ +import express from "express"; +import {requireLogin} from "../middleware/auth.js"; + +export function createVisionDebugRouter({visionDebugService, repositories}) { + const router = express.Router(); + router.use(requireLogin); + router.get("/debug/vision/latest", (req, res) => { + const diagnostic = visionDebugService.latest(req.session.user, repositories); + return diagnostic ? res.json({diagnostic}) : res.status(404).json({error: "Vision diagnostic not found"}); + }); + router.get("/debug/vision/:requestId", (req, res) => { + const diagnostic = visionDebugService.get(req.params.requestId, req.session.user, repositories); + return diagnostic ? res.json({diagnostic}) : res.status(404).json({error: "Vision diagnostic not found"}); + }); + return router; +} diff --git a/src/services/visionDebugService.js b/src/services/visionDebugService.js new file mode 100644 index 0000000..2a702d3 --- /dev/null +++ b/src/services/visionDebugService.js @@ -0,0 +1,180 @@ +const RAW_VISION_LIMIT = 12000; +const EVIDENCE_LIMIT = 100000; +const ANSWER_LIMIT = 30000; +const REASONING_LIMIT = 30000; +const MAX_RECORDS = 100; +const MAX_AGE_MS = 60 * 60 * 1000; + +const bounded = (value, limit) => typeof value === "string" ? value.slice(0, limit) : value; +const clone = value => value == null ? value : structuredClone(value); +const exposed = record => { + const value = clone(record); + delete value.user; + return value; +}; + +export function createVisionDebugService(config) { + const enabled = config.visionDebug === true; + const records = new Map(); + + const prune = () => { + const cutoff = Date.now() - MAX_AGE_MS; + for (const [id, record] of records) { + if (Date.parse(record.updatedAt) < cutoff) records.delete(id); + } + while (records.size > MAX_RECORDS) records.delete(records.keys().next().value); + }; + const update = (requestId, operation) => { + if (!enabled || !requestId) return null; + const record = records.get(requestId); + if (!record) return null; + operation(record); + record.updatedAt = new Date().toISOString(); + return record; + }; + const image = (record, attachmentId) => record.images.find(item => item.attachmentId === attachmentId); + + return { + enabled, + begin({requestId, user, chatId, triggeringMessageId, userQuestion, selectedModel, reasoningRequested, webSearchRequested}) { + if (!enabled || !requestId) return; + prune(); + const timestamp = new Date().toISOString(); + records.delete(requestId); + records.set(requestId, { + requestId, user, chatId, triggeringMessageId, + createdAt: timestamp, updatedAt: timestamp, + request: {userQuestion, selectedModel, reasoningRequested: reasoningRequested === true, webSearchRequested: webSearchRequested === true}, + images: [], + qwen: null, + search: { + requested: webSearchRequested === true, + needed: null, + explicitSearchIntent: null, + state: webSearchRequested === true ? "pending" : "disabled", + visualReference: "", + queries: [], + candidateCount: 0, + selectedCount: 0, + selectedSources: [] + }, + visibleReasoning: null, + finalAnswer: null, + error: null, + experimentNote: "For controlled Web Search/Reasoning comparisons, reuse this attachment and exact question. Re-uploading identical bytes creates a new attachment ID and vision cache entry." + }); + prune(); + }, + recordImage(requestId, value) { + update(requestId, record => { + const existing = image(record, value.attachmentId); + const next = { + requestId, + chatId: record.chatId, + triggeringMessageId: record.triggeringMessageId, + attachmentId: value.attachmentId, + sourceMessageId: value.sourceMessageId, + position: value.position, + scope: value.scope, + label: value.label, + originalFilename: value.originalFilename, + mimeType: value.mimeType, + width: value.width, + height: value.height, + sizeBytes: value.sizeBytes, + imageSha256: null, + visionModel: value.visionModel, + visionPromptVersion: value.visionPromptVersion, + normalizedQuestion: value.normalizedQuestion, + cacheHit: null, + cacheIdentity: null, + rawVisionResponse: null, + normalizedVisionAnalysis: null + }; + if (existing) Object.assign(existing, next); else record.images.push(next); + }); + }, + recordCache(requestId, attachmentId, {cacheHit, cacheIdentity, normalizedVisionAnalysis}) { + update(requestId, record => { + const entry = image(record, attachmentId); + if (!entry) return; + entry.cacheHit = cacheHit === true; + entry.cacheIdentity = clone(cacheIdentity); + entry.rawVisionResponse = null; + if (normalizedVisionAnalysis !== undefined) entry.normalizedVisionAnalysis = clone(normalizedVisionAnalysis); + }); + }, + recordVisionRun(requestId, attachmentId, {imageSha256, rawVisionResponse, normalizedVisionAnalysis}) { + update(requestId, record => { + const entry = image(record, attachmentId); + if (!entry) return; + entry.cacheHit = false; + entry.imageSha256 = imageSha256 || null; + entry.rawVisionResponse = bounded(String(rawVisionResponse ?? ""), RAW_VISION_LIMIT); + entry.normalizedVisionAnalysis = clone(normalizedVisionAnalysis); + }); + }, + recordQwen(requestId, value) { + update(requestId, record => { + record.qwen = { + imageEvidence: bounded(value.imageEvidence, EVIDENCE_LIMIT), + visionGuidanceSha256: value.visionGuidanceSha256, + visionEvidenceIncluded: value.visionEvidenceIncluded === true, + imageCount: value.imageCount, + currentImageLabels: clone(value.currentImageLabels), + model: value.model, + think: value.think === true, + stream: value.stream === true, + temperature: value.temperature ?? null, + numCtx: value.numCtx ?? null, + numPredict: value.numPredict ?? null, + keepAlive: value.keepAlive + }; + }); + }, + recordSearch(requestId, value) { + update(requestId, record => { + record.search = { + ...record.search, + requested: value.requested === true, + needed: value.needed ?? record.search.needed, + explicitSearchIntent: value.explicitSearchIntent ?? record.search.explicitSearchIntent, + state: value.state ?? record.search.state, + visualReference: bounded(value.visualReference ?? record.search.visualReference, 1200), + queries: Array.isArray(value.queries) ? value.queries.map(item => bounded(String(item), 500)) : record.search.queries, + candidateCount: value.candidateCount ?? record.search.candidateCount, + selectedCount: value.selectedCount ?? record.search.selectedCount, + selectedSources: Array.isArray(value.selectedSources) ? value.selectedSources.slice(0, 5).map((source, index) => ({ + index: index + 1, + title: bounded(String(source.title || ""), 300), + url: bounded(String(source.url || ""), 2048), + snippet: bounded(String(source.snippet || ""), 500) + })) : record.search.selectedSources + }; + }); + }, + complete(requestId, {finalAnswer, visibleReasoning}) { + update(requestId, record => { + record.finalAnswer = bounded(String(finalAnswer ?? ""), ANSWER_LIMIT); + record.visibleReasoning = visibleReasoning ? bounded(String(visibleReasoning), REASONING_LIMIT) : null; + }); + }, + fail(requestId, error) { + update(requestId, record => { record.error = bounded(String(error?.message || error || "Request failed"), 1000); }); + }, + get(requestId, user, repositories) { + if (!enabled) return null; + prune(); + const record = records.get(requestId); + if (!record || record.user !== user || !repositories.getChat(record.chatId, user)) return null; + return exposed(record); + }, + latest(user, repositories) { + if (!enabled) return null; + prune(); + const values = [...records.values()].reverse(); + const record = values.find(item => item.user === user && repositories.getChat(item.chatId, user)); + return record ? exposed(record) : null; + } + }; +} diff --git a/test/visionDiagnostics.test.js b/test/visionDiagnostics.test.js new file mode 100644 index 0000000..91574e3 --- /dev/null +++ b/test/visionDiagnostics.test.js @@ -0,0 +1,206 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import {createServer} from "node:http"; +import sharp from "sharp"; +import {createApp} from "../src/app.js"; +import {VISION_PROMPT_VERSION} from "../src/services/visionAnalysis.js"; + +async function listen(t, handler) { + const server = createServer(handler); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + t.after(() => new Promise(resolve => { server.closeAllConnections(); server.close(resolve); })); + return {server, url: `http://127.0.0.1:${server.address().port}`}; +} + +async function setup(t, visionDebug = true) { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "elsewhere-vision-debug-")); + const requests = []; + const searchQueries = []; + const rawVisionResponse = ` \n${JSON.stringify({ + scene: "A penguin stands on dark rocks.", + subjects: [{type: "bird", description: "Black crown and white throat", location: "center"}], + visualDetails: ["A thin black line runs directly beneath the chin."], + visibleText: [], spatialRelationships: [], colorsAndAppearance: ["black and white plumage"], + recognizedElements: ["penguin"], notableFeatures: ["thin black line beneath chin"], uncertainties: [] + })}\n `; + const search = await listen(t, (req, res) => { + searchQueries.push(new URL(req.url, "http://localhost").searchParams.get("q")); + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify({results: [{title: "Penguin identification guide", url: "https://example.com/penguins", + content: "A penguin identification guide compares crown, throat, bill, and chin markings."}]})); + }); + const app = createApp({config: { + dbPath: ":memory:", uploadDir: directory, testMode: true, sessionSecret: "vision-debug-tests", + visionDebug, visionModel: "qwen2.5vl:7b", chatSummaryUpdateEveryMessages: 0, + searxngUrl: search.url, eventLoopLagMonitorEnabled: false, + slowRequestLoggingEnabled: false, structuredRequestLoggingEnabled: false + }, modelOverrides: {fetch: async (_url, options) => { + const body = JSON.parse(options.body); + requests.push(body); + const chunks = body.model === "qwen2.5vl:7b" + ? [{message: {content: rawVisionResponse}, done: true}] + : [...(body.think ? [{message: {thinking: "Compare the supplied visible evidence."}}] : []), + {message: {content: "Final penguin answer."}, done: true}]; + return new Response(chunks.map(chunk => JSON.stringify(chunk)).join("\n")); + }}}); + const backend = await listen(t, app); + t.after(async () => { await app.locals.close(); await fs.rm(directory, {recursive: true, force: true}); }); + const register = async username => { + const response = await fetch(backend.url + "/register", {method: "POST", headers: {"Content-Type": "application/json"}, + body: JSON.stringify({username, password: "test-password"})}); + assert.equal(response.status, 200); + return response.headers.get("set-cookie").split(";")[0]; + }; + const request = (url, {cookie = "", method = "GET", body, headers = {}} = {}) => fetch(backend.url + url, { + method, headers: {...headers, ...(cookie ? {Cookie: cookie} : {}), ...(body ? {"Content-Type": "application/json"} : {})}, + body: body ? JSON.stringify(body) : undefined + }); + const upload = async (cookie, bytes, filename = "penguin.png") => { + const form = new FormData(); + form.append("image", new Blob([bytes], {type: "image/png"}), filename); + const response = await fetch(backend.url + "/uploads", {method: "POST", headers: {Cookie: cookie}, body: form}); + assert.equal(response.status, 201); + return response.json(); + }; + const createChat = async cookie => { + const response = await request("/chats", {cookie, method: "POST", body: {title: "Vision debug"}}); + return (await response.json()).chat.id; + }; + return {app, backend, register, request, upload, createChat, requests, searchQueries, rawVisionResponse}; +} + +const getDiagnostic = async (s, cookie, requestId) => { + const response = await s.request(`/debug/vision/${requestId}`, {cookie}); + assert.equal(response.status, 200); + return (await response.json()).diagnostic; +}; + +test("vision diagnostics routes are absent when disabled and protected when enabled", async t => { + const disabled = await setup(t, false); + assert.equal(disabled.app.locals.config.visionDebug, false); + const disabledUser = await disabled.register("debug-disabled"); + assert.equal((await disabled.request("/debug/vision/latest", {cookie: disabledUser})).status, 404); + + const enabled = await setup(t, true); + assert.equal((await enabled.request("/debug/vision/latest")).status, 403); +}); + +test("vision diagnostics capture misses, hits, Qwen evidence, search, reasoning and ownership safely", async t => { + const s = await setup(t, true); + const owner = await s.register("debug-owner"); + const other = await s.register("debug-other"); + const chatId = await s.createChat(owner); + const bytes = await sharp({create: {width: 24, height: 12, channels: 3, background: "black"}}).png().toBuffer(); + const attachment = await s.upload(owner, bytes); + const question = "Search the web to identify this penguin."; + + const first = await s.request("/chat", {cookie: owner, method: "POST", body: { + chatId, message: question, attachments: [attachment.id], model: "qwen3:14b", webSearch: false, reasoning: false + }}); + assert.equal(first.status, 200); + const firstRequestId = first.headers.get("x-request-id"); + const miss = await getDiagnostic(s, owner, firstRequestId); + assert.equal((await s.request(`/debug/vision/${firstRequestId}`, {cookie: other})).status, 404); + assert.equal(miss.requestId, firstRequestId); + assert.equal(miss.chatId, chatId); + assert.equal(miss.user, undefined); + assert.equal(miss.images.length, 1); + const image = miss.images[0]; + assert.equal(image.attachmentId, attachment.id); + assert.equal(image.position, 0); + assert.equal(image.scope, "current"); + assert.equal(image.label, "Current image 1"); + assert.equal(image.originalFilename, "penguin.png"); + assert.equal(image.mimeType, "image/png"); + assert.equal(image.width, 24); assert.equal(image.height, 12); assert.equal(image.sizeBytes, bytes.length); + assert.equal(image.visionModel, "qwen2.5vl:7b"); + assert.equal(image.visionPromptVersion, VISION_PROMPT_VERSION); + assert.equal(image.normalizedQuestion, question); + assert.equal(image.cacheHit, false); + assert.match(image.imageSha256, /^[a-f0-9]{64}$/); + assert.equal(image.rawVisionResponse, s.rawVisionResponse); + assert.equal(image.normalizedVisionAnalysis.scene, "A penguin stands on dark rocks."); + assert.equal(miss.qwen.visionEvidenceIncluded, true); + assert.equal(miss.qwen.imageCount, 1); + assert.deepEqual(miss.qwen.currentImageLabels, ["Current image 1"]); + assert.equal(miss.qwen.model, "qwen3:14b"); + assert.equal(miss.qwen.think, false); + assert.equal(miss.qwen.stream, true); + assert.equal(miss.qwen.keepAlive, "30m"); + const finalRequest = s.requests.findLast(request => request.model === "qwen3:14b"); + assert.equal(miss.qwen.imageEvidence, finalRequest.messages.find(message => message.content.startsWith("")).content); + assert.match(miss.qwen.visionGuidanceSha256, /^[a-f0-9]{64}$/); + assert.equal(miss.search.state, "disabled"); + assert.equal(miss.visibleReasoning, null); + assert.equal(miss.finalAnswer, "Final penguin answer."); + const uploadedBase64 = s.requests.find(request => request.model === "qwen2.5vl:7b").messages.at(-1).images[0]; + const diagnosticText = JSON.stringify(miss); + assert.equal(diagnosticText.includes(uploadedBase64), false); + assert.equal(diagnosticText.includes(bytes.toString("latin1")), false); + + const messagesResponse = await s.request(`/chats/${chatId}/messages`, {cookie: owner}); + const bot = (await messagesResponse.json()).messages.at(-1); + const retry = await s.request(`/chats/${chatId}/messages/${bot.id}/retry`, {cookie: owner, method: "POST", body: { + model: "qwen3:14b", webSearch: true, reasoning: true + }}); + assert.equal(retry.status, 200); + const hit = await getDiagnostic(s, owner, retry.headers.get("x-request-id")); + assert.equal(hit.images[0].cacheHit, true); + assert.deepEqual(hit.images[0].cacheIdentity, {attachmentId: attachment.id, visionModel: "qwen2.5vl:7b", + visionPromptVersion: VISION_PROMPT_VERSION, normalizedQuestion: question}); + assert.equal(hit.images[0].rawVisionResponse, null); + assert.equal(hit.images[0].imageSha256, null); + assert.deepEqual(hit.images[0].normalizedVisionAnalysis, image.normalizedVisionAnalysis); + assert.equal(hit.qwen.imageEvidence, miss.qwen.imageEvidence); + assert.equal(hit.qwen.think, true); + assert.equal(hit.visibleReasoning, "Compare the supplied visible evidence."); + assert.equal(hit.search.requested, true); + assert.equal(hit.search.needed, true); + assert.equal(hit.search.explicitSearchIntent, true); + assert.equal(hit.search.state, "succeeded"); + assert.match(hit.search.visualReference, /thin black line beneath chin/i); + assert.deepEqual(hit.search.queries, s.searchQueries); + assert.equal(hit.search.candidateCount, 1); + assert.equal(hit.search.selectedCount, 1); + assert.deepEqual(hit.search.selectedSources[0], {index: 1, title: "Penguin identification guide", + url: "https://example.com/penguins", snippet: "A penguin identification guide compares crown, throat, bill, and chin markings."}); + assert.equal(hit.finalAnswer, "Final penguin answer."); + const latest = await s.request("/debug/vision/latest", {cookie: owner}); + assert.equal((await latest.json()).diagnostic.requestId, hit.requestId); +}); + +test("diagnostics distinguish skipped search and preserve multiple and historical image provenance", async t => { + const s = await setup(t, true); + const cookie = await s.register("debug-provenance"); + const chatId = await s.createChat(cookie); + const red = await sharp({create: {width: 20, height: 10, channels: 3, background: "red"}}).png().toBuffer(); + const blue = await sharp({create: {width: 30, height: 15, channels: 3, background: "blue"}}).png().toBuffer(); + const first = await s.upload(cookie, red, "first.png"); + const second = await s.upload(cookie, blue, "second.png"); + const comparison = await s.request("/chat", {cookie, method: "POST", body: { + chatId, message: "What species are these?", attachments: [first.id, second.id], webSearch: true + }}); + const current = await getDiagnostic(s, cookie, comparison.headers.get("x-request-id")); + assert.deepEqual(current.images.map(image => [image.attachmentId, image.position, image.scope, image.label]), [ + [first.id, 0, "current", "Current image 1"], [second.id, 1, "current", "Current image 2"] + ]); + assert.notEqual(current.images[0].imageSha256, current.images[1].imageSha256); + assert.equal(current.search.requested, true); + assert.equal(current.search.needed, false); + assert.equal(current.search.state, "skipped"); + assert.deepEqual(current.search.queries, []); + assert.equal(current.search.candidateCount, 0); + + const historicalResponse = await s.request("/chat", {cookie, method: "POST", body: { + chatId, message: "What did the previous image show?" + }}); + const historical = await getDiagnostic(s, cookie, historicalResponse.headers.get("x-request-id")); + assert.equal(historical.images.length, 1); + assert.equal(historical.images[0].attachmentId, second.id); + assert.equal(historical.images[0].position, 1); + assert.equal(historical.images[0].scope, "historical"); + assert.match(historical.images[0].label, /Earlier image from message/); +});