/** * @file Prüft Zugriffsschutz und Diagnoseaufzeichnungen für Bildanalysen, Caches und Modellanfragen. */ import test from "node:test"; import assert from "node:assert/strict"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import {createServer} from "node:http"; import sharp from "sharp"; import {createApp} from "../src/app.js"; import {VISION_PROMPT_VERSION} from "../src/services/visionAnalysis.js"; async function listen(t, handler) { const server = createServer(handler); await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); t.after(() => new Promise(resolve => { server.closeAllConnections(); server.close(resolve); })); return {server, url: `http://127.0.0.1:${server.address().port}`}; } async function setup(t, visionDebug = true) { const directory = await fs.mkdtemp(path.join(os.tmpdir(), "elsewhere-vision-debug-")); const requests = []; const searchQueries = []; const rawVisionResponse = ` \n${JSON.stringify({ scene: "A penguin stands on dark rocks.", subjects: [{type: "bird", description: "Black crown and white throat", location: "center"}], visualDetails: ["A thin black line runs directly beneath the chin."], visibleText: [], spatialRelationships: [], colorsAndAppearance: ["black and white plumage"], recognizedElements: ["penguin"], notableFeatures: ["thin black line beneath chin"], uncertainties: [] })}\n `; const search = await listen(t, (req, res) => { searchQueries.push(new URL(req.url, "http://localhost").searchParams.get("q")); res.setHeader("Content-Type", "application/json"); res.end(JSON.stringify({results: [{title: "Penguin identification guide", url: "https://example.com/penguins", content: "A penguin identification guide compares crown, throat, bill, and chin markings."}]})); }); const app = createApp({config: { dbPath: ":memory:", uploadDir: directory, testMode: true, sessionSecret: "vision-debug-tests", visionDebug, visionModel: "qwen2.5vl:7b", chatSummaryUpdateEveryMessages: 0, searxngUrl: search.url, eventLoopLagMonitorEnabled: false, slowRequestLoggingEnabled: false, structuredRequestLoggingEnabled: false }, modelOverrides: {fetch: async (_url, options) => { const body = JSON.parse(options.body); requests.push(body); const chunks = body.model === "qwen2.5vl:7b" ? [{message: {content: rawVisionResponse}, done: true}] : [...(body.think ? [{message: {thinking: "Compare the supplied visible evidence."}}] : []), {message: {content: "Final penguin answer."}, done: true}]; return new Response(chunks.map(chunk => JSON.stringify(chunk)).join("\n")); }}}); const backend = await listen(t, app); t.after(async () => { await app.locals.close(); await fs.rm(directory, {recursive: true, force: true}); }); const register = async username => { const response = await fetch(backend.url + "/register", {method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify({username, password: "test-password"})}); assert.equal(response.status, 200); return response.headers.get("set-cookie").split(";")[0]; }; const request = (url, {cookie = "", method = "GET", body, headers = {}} = {}) => fetch(backend.url + url, { method, headers: {...headers, ...(cookie ? {Cookie: cookie} : {}), ...(body ? {"Content-Type": "application/json"} : {})}, body: body ? JSON.stringify(body) : undefined }); const upload = async (cookie, bytes, filename = "penguin.png") => { const form = new FormData(); form.append("image", new Blob([bytes], {type: "image/png"}), filename); const response = await fetch(backend.url + "/uploads", {method: "POST", headers: {Cookie: cookie}, body: form}); assert.equal(response.status, 201); return response.json(); }; const createChat = async cookie => { const response = await request("/chats", {cookie, method: "POST", body: {title: "Vision debug"}}); return (await response.json()).chat.id; }; return {app, backend, register, request, upload, createChat, requests, searchQueries, rawVisionResponse}; } const getDiagnostic = async (s, cookie, requestId) => { const response = await s.request(`/debug/vision/${requestId}`, {cookie}); assert.equal(response.status, 200); return (await response.json()).diagnostic; }; test("vision diagnostics routes are absent when disabled and protected when enabled", async t => { const disabled = await setup(t, false); assert.equal(disabled.app.locals.config.visionDebug, false); const disabledUser = await disabled.register("debug-disabled"); assert.equal((await disabled.request("/debug/vision/latest", {cookie: disabledUser})).status, 404); const enabled = await setup(t, true); assert.equal((await enabled.request("/debug/vision/latest")).status, 403); }); test("vision diagnostics capture misses, hits, Qwen evidence, search, reasoning and ownership safely", async t => { const s = await setup(t, true); const owner = await s.register("debug-owner"); const other = await s.register("debug-other"); const chatId = await s.createChat(owner); const bytes = await sharp({create: {width: 24, height: 12, channels: 3, background: "black"}}).png().toBuffer(); const attachment = await s.upload(owner, bytes); const question = "Search the web to identify this penguin."; const first = await s.request("/chat", {cookie: owner, method: "POST", body: { chatId, message: question, attachments: [attachment.id], model: "qwen3:14b", webSearch: false, reasoning: false }}); assert.equal(first.status, 200); const firstRequestId = first.headers.get("x-request-id"); const miss = await getDiagnostic(s, owner, firstRequestId); assert.equal((await s.request(`/debug/vision/${firstRequestId}`, {cookie: other})).status, 404); assert.equal(miss.requestId, firstRequestId); assert.equal(miss.chatId, chatId); assert.equal(miss.user, undefined); assert.equal(miss.images.length, 1); const image = miss.images[0]; assert.equal(image.attachmentId, attachment.id); assert.equal(image.position, 0); assert.equal(image.scope, "current"); assert.equal(image.label, "Current image 1"); assert.equal(image.originalFilename, "penguin.png"); assert.equal(image.mimeType, "image/png"); assert.equal(image.width, 24); assert.equal(image.height, 12); assert.equal(image.sizeBytes, bytes.length); assert.equal(image.visionModel, "qwen2.5vl:7b"); assert.equal(image.visionPromptVersion, VISION_PROMPT_VERSION); assert.equal(image.normalizedQuestion, question); assert.equal(image.cacheHit, false); assert.match(image.imageSha256, /^[a-f0-9]{64}$/); assert.equal(image.rawVisionResponse, s.rawVisionResponse); assert.equal(image.normalizedVisionAnalysis.scene, "A penguin stands on dark rocks."); assert.equal(miss.qwen.visionEvidenceIncluded, true); assert.equal(miss.qwen.imageCount, 1); assert.deepEqual(miss.qwen.currentImageLabels, ["Current image 1"]); assert.equal(miss.qwen.model, "qwen3:14b"); assert.equal(miss.qwen.think, false); assert.equal(miss.qwen.stream, true); assert.equal(miss.qwen.keepAlive, "30m"); const finalRequest = s.requests.findLast(request => request.model === "qwen3:14b"); assert.equal(miss.qwen.imageEvidence, finalRequest.messages.find(message => message.content.startsWith("")).content); assert.match(miss.qwen.visionGuidanceSha256, /^[a-f0-9]{64}$/); assert.equal(miss.search.state, "disabled"); assert.equal(miss.visibleReasoning, null); assert.equal(miss.finalAnswer, "Final penguin answer."); const uploadedBase64 = s.requests.find(request => request.model === "qwen2.5vl:7b").messages.at(-1).images[0]; const diagnosticText = JSON.stringify(miss); assert.equal(diagnosticText.includes(uploadedBase64), false); assert.equal(diagnosticText.includes(bytes.toString("latin1")), false); const messagesResponse = await s.request(`/chats/${chatId}/messages`, {cookie: owner}); const bot = (await messagesResponse.json()).messages.at(-1); const retry = await s.request(`/chats/${chatId}/messages/${bot.id}/retry`, {cookie: owner, method: "POST", body: { model: "qwen3:14b", webSearch: true, reasoning: true }}); assert.equal(retry.status, 200); const hit = await getDiagnostic(s, owner, retry.headers.get("x-request-id")); assert.equal(hit.images[0].cacheHit, true); assert.deepEqual(hit.images[0].cacheIdentity, {attachmentId: attachment.id, visionModel: "qwen2.5vl:7b", visionPromptVersion: VISION_PROMPT_VERSION, normalizedQuestion: question}); assert.equal(hit.images[0].rawVisionResponse, null); assert.equal(hit.images[0].imageSha256, null); assert.deepEqual(hit.images[0].normalizedVisionAnalysis, image.normalizedVisionAnalysis); assert.equal(hit.qwen.imageEvidence, miss.qwen.imageEvidence); assert.equal(hit.qwen.think, true); assert.equal(hit.visibleReasoning, "Compare the supplied visible evidence."); assert.equal(hit.search.requested, true); assert.equal(hit.search.needed, true); assert.equal(hit.search.explicitSearchIntent, true); assert.equal(hit.search.state, "succeeded"); assert.match(hit.search.visualReference, /thin black line beneath chin/i); assert.deepEqual(hit.search.queries, s.searchQueries); assert.equal(hit.search.candidateCount, 1); assert.equal(hit.search.selectedCount, 1); assert.deepEqual(hit.search.selectedSources[0], {index: 1, title: "Penguin identification guide", url: "https://example.com/penguins", snippet: "A penguin identification guide compares crown, throat, bill, and chin markings."}); assert.equal(hit.finalAnswer, "Final penguin answer."); const latest = await s.request("/debug/vision/latest", {cookie: owner}); assert.equal((await latest.json()).diagnostic.requestId, hit.requestId); }); test("diagnostics distinguish skipped search and preserve multiple and historical image provenance", async t => { const s = await setup(t, true); const cookie = await s.register("debug-provenance"); const chatId = await s.createChat(cookie); const red = await sharp({create: {width: 20, height: 10, channels: 3, background: "red"}}).png().toBuffer(); const blue = await sharp({create: {width: 30, height: 15, channels: 3, background: "blue"}}).png().toBuffer(); const first = await s.upload(cookie, red, "first.png"); const second = await s.upload(cookie, blue, "second.png"); const comparison = await s.request("/chat", {cookie, method: "POST", body: { chatId, message: "What species are these?", attachments: [first.id, second.id], webSearch: true }}); const current = await getDiagnostic(s, cookie, comparison.headers.get("x-request-id")); assert.deepEqual(current.images.map(image => [image.attachmentId, image.position, image.scope, image.label]), [ [first.id, 0, "current", "Current image 1"], [second.id, 1, "current", "Current image 2"] ]); assert.notEqual(current.images[0].imageSha256, current.images[1].imageSha256); assert.equal(current.search.requested, true); assert.equal(current.search.needed, false); assert.equal(current.search.state, "skipped"); assert.deepEqual(current.search.queries, []); assert.equal(current.search.candidateCount, 0); const historicalResponse = await s.request("/chat", {cookie, method: "POST", body: { chatId, message: "What did the previous image show?" }}); const historical = await getDiagnostic(s, cookie, historicalResponse.headers.get("x-request-id")); assert.equal(historical.images.length, 1); assert.equal(historical.images[0].attachmentId, second.id); assert.equal(historical.images[0].position, 1); assert.equal(historical.images[0].scope, "historical"); assert.match(historical.images[0].label, /Earlier image from message/); });