211 lines
12 KiB
JavaScript
211 lines
12 KiB
JavaScript
/**
|
|
* @file Prüft Zugriffsschutz und Diagnoseaufzeichnungen für Bildanalysen, Caches und Modellanfragen.
|
|
*/
|
|
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import fs from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import {createServer} from "node:http";
|
|
import sharp from "sharp";
|
|
import {createApp} from "../src/app.js";
|
|
import {VISION_PROMPT_VERSION} from "../src/services/visionAnalysis.js";
|
|
|
|
async function listen(t, handler) {
|
|
const server = createServer(handler);
|
|
await new Promise(resolve => server.listen(0, "127.0.0.1", resolve));
|
|
t.after(() => new Promise(resolve => { server.closeAllConnections(); server.close(resolve); }));
|
|
return {server, url: `http://127.0.0.1:${server.address().port}`};
|
|
}
|
|
|
|
async function setup(t, visionDebug = true) {
|
|
const directory = await fs.mkdtemp(path.join(os.tmpdir(), "elsewhere-vision-debug-"));
|
|
const requests = [];
|
|
const searchQueries = [];
|
|
const rawVisionResponse = ` \n${JSON.stringify({
|
|
scene: "A penguin stands on dark rocks.",
|
|
subjects: [{type: "bird", description: "Black crown and white throat", location: "center"}],
|
|
visualDetails: ["A thin black line runs directly beneath the chin."],
|
|
visibleText: [], spatialRelationships: [], colorsAndAppearance: ["black and white plumage"],
|
|
recognizedElements: ["penguin"], notableFeatures: ["thin black line beneath chin"], uncertainties: []
|
|
})}\n `;
|
|
const search = await listen(t, (req, res) => {
|
|
searchQueries.push(new URL(req.url, "http://localhost").searchParams.get("q"));
|
|
res.setHeader("Content-Type", "application/json");
|
|
res.end(JSON.stringify({results: [{title: "Penguin identification guide", url: "https://example.com/penguins",
|
|
content: "A penguin identification guide compares crown, throat, bill, and chin markings."}]}));
|
|
});
|
|
const app = createApp({config: {
|
|
dbPath: ":memory:", uploadDir: directory, testMode: true, sessionSecret: "vision-debug-tests",
|
|
visionDebug, visionModel: "qwen2.5vl:7b", chatSummaryUpdateEveryMessages: 0,
|
|
searxngUrl: search.url, eventLoopLagMonitorEnabled: false,
|
|
slowRequestLoggingEnabled: false, structuredRequestLoggingEnabled: false
|
|
}, modelOverrides: {fetch: async (_url, options) => {
|
|
const body = JSON.parse(options.body);
|
|
if (body.format?.properties?.search) return new Response(JSON.stringify({message: {content: '{"search":false,"query":""}'}, done: true}));
|
|
requests.push(body);
|
|
const chunks = body.model === "qwen2.5vl:7b"
|
|
? [{message: {content: rawVisionResponse}, done: true}]
|
|
: [...(body.think ? [{message: {thinking: "Compare the supplied visible evidence."}}] : []),
|
|
{message: {content: "Final penguin answer."}, done: true}];
|
|
return new Response(chunks.map(chunk => JSON.stringify(chunk)).join("\n"));
|
|
}}});
|
|
const backend = await listen(t, app);
|
|
t.after(async () => { await app.locals.close(); await fs.rm(directory, {recursive: true, force: true}); });
|
|
const register = async username => {
|
|
const response = await fetch(backend.url + "/register", {method: "POST", headers: {"Content-Type": "application/json"},
|
|
body: JSON.stringify({username, password: "test-password"})});
|
|
assert.equal(response.status, 200);
|
|
return response.headers.get("set-cookie").split(";")[0];
|
|
};
|
|
const request = (url, {cookie = "", method = "GET", body, headers = {}} = {}) => fetch(backend.url + url, {
|
|
method, headers: {...headers, ...(cookie ? {Cookie: cookie} : {}), ...(body ? {"Content-Type": "application/json"} : {})},
|
|
body: body ? JSON.stringify(body) : undefined
|
|
});
|
|
const upload = async (cookie, bytes, filename = "penguin.png") => {
|
|
const form = new FormData();
|
|
form.append("image", new Blob([bytes], {type: "image/png"}), filename);
|
|
const response = await fetch(backend.url + "/uploads", {method: "POST", headers: {Cookie: cookie}, body: form});
|
|
assert.equal(response.status, 201);
|
|
return response.json();
|
|
};
|
|
const createChat = async cookie => {
|
|
const response = await request("/chats", {cookie, method: "POST", body: {title: "Vision debug"}});
|
|
return (await response.json()).chat.id;
|
|
};
|
|
return {app, backend, register, request, upload, createChat, requests, searchQueries, rawVisionResponse};
|
|
}
|
|
|
|
const getDiagnostic = async (s, cookie, requestId) => {
|
|
const response = await s.request(`/debug/vision/${requestId}`, {cookie});
|
|
assert.equal(response.status, 200);
|
|
return (await response.json()).diagnostic;
|
|
};
|
|
|
|
test("vision diagnostics routes are absent when disabled and protected when enabled", async t => {
|
|
const disabled = await setup(t, false);
|
|
assert.equal(disabled.app.locals.config.visionDebug, false);
|
|
const disabledUser = await disabled.register("debug-disabled");
|
|
assert.equal((await disabled.request("/debug/vision/latest", {cookie: disabledUser})).status, 404);
|
|
|
|
const enabled = await setup(t, true);
|
|
assert.equal((await enabled.request("/debug/vision/latest")).status, 403);
|
|
});
|
|
|
|
test("vision diagnostics capture misses, hits, Qwen evidence, search, reasoning and ownership safely", async t => {
|
|
const s = await setup(t, true);
|
|
const owner = await s.register("debug-owner");
|
|
const other = await s.register("debug-other");
|
|
const chatId = await s.createChat(owner);
|
|
const bytes = await sharp({create: {width: 24, height: 12, channels: 3, background: "black"}}).png().toBuffer();
|
|
const attachment = await s.upload(owner, bytes);
|
|
const question = "Search the web to identify this penguin.";
|
|
|
|
const first = await s.request("/chat", {cookie: owner, method: "POST", body: {
|
|
chatId, message: question, attachments: [attachment.id], model: "qwen3:14b", webSearch: false, reasoning: false
|
|
}});
|
|
assert.equal(first.status, 200);
|
|
const firstRequestId = first.headers.get("x-request-id");
|
|
const miss = await getDiagnostic(s, owner, firstRequestId);
|
|
assert.equal((await s.request(`/debug/vision/${firstRequestId}`, {cookie: other})).status, 404);
|
|
assert.equal(miss.requestId, firstRequestId);
|
|
assert.equal(miss.chatId, chatId);
|
|
assert.equal(miss.user, undefined);
|
|
assert.equal(miss.images.length, 1);
|
|
const image = miss.images[0];
|
|
assert.equal(image.attachmentId, attachment.id);
|
|
assert.equal(image.position, 0);
|
|
assert.equal(image.scope, "current");
|
|
assert.equal(image.label, "Current image 1");
|
|
assert.equal(image.originalFilename, "penguin.png");
|
|
assert.equal(image.mimeType, "image/png");
|
|
assert.equal(image.width, 24); assert.equal(image.height, 12); assert.equal(image.sizeBytes, bytes.length);
|
|
assert.equal(image.visionModel, "qwen2.5vl:7b");
|
|
assert.equal(image.visionPromptVersion, VISION_PROMPT_VERSION);
|
|
assert.equal(image.normalizedQuestion, question);
|
|
assert.equal(image.cacheHit, false);
|
|
assert.match(image.imageSha256, /^[a-f0-9]{64}$/);
|
|
assert.equal(image.rawVisionResponse, s.rawVisionResponse);
|
|
assert.equal(image.normalizedVisionAnalysis.scene, "A penguin stands on dark rocks.");
|
|
assert.equal(miss.qwen.visionEvidenceIncluded, true);
|
|
assert.equal(miss.qwen.imageCount, 1);
|
|
assert.deepEqual(miss.qwen.currentImageLabels, ["Current image 1"]);
|
|
assert.equal(miss.qwen.model, "qwen3:14b");
|
|
assert.equal(miss.qwen.think, false);
|
|
assert.equal(miss.qwen.stream, true);
|
|
assert.equal(miss.qwen.keepAlive, "30m");
|
|
const finalRequest = s.requests.findLast(request => request.model === "qwen3:14b");
|
|
assert.equal(miss.qwen.imageEvidence, finalRequest.messages.find(message => message.content.startsWith("<image_evidence>")).content);
|
|
assert.match(miss.qwen.visionGuidanceSha256, /^[a-f0-9]{64}$/);
|
|
assert.equal(miss.search.state, "disabled");
|
|
assert.equal(miss.visibleReasoning, null);
|
|
assert.equal(miss.finalAnswer, "Final penguin answer.");
|
|
const uploadedBase64 = s.requests.find(request => request.model === "qwen2.5vl:7b").messages.at(-1).images[0];
|
|
const diagnosticText = JSON.stringify(miss);
|
|
assert.equal(diagnosticText.includes(uploadedBase64), false);
|
|
assert.equal(diagnosticText.includes(bytes.toString("latin1")), false);
|
|
|
|
const messagesResponse = await s.request(`/chats/${chatId}/messages`, {cookie: owner});
|
|
const bot = (await messagesResponse.json()).messages.at(-1);
|
|
const retry = await s.request(`/chats/${chatId}/messages/${bot.id}/retry`, {cookie: owner, method: "POST", body: {
|
|
model: "qwen3:14b", webSearch: true, reasoning: true
|
|
}});
|
|
assert.equal(retry.status, 200);
|
|
const hit = await getDiagnostic(s, owner, retry.headers.get("x-request-id"));
|
|
assert.equal(hit.images[0].cacheHit, true);
|
|
assert.deepEqual(hit.images[0].cacheIdentity, {attachmentId: attachment.id, visionModel: "qwen2.5vl:7b",
|
|
visionPromptVersion: VISION_PROMPT_VERSION, normalizedQuestion: question});
|
|
assert.equal(hit.images[0].rawVisionResponse, null);
|
|
assert.equal(hit.images[0].imageSha256, null);
|
|
assert.deepEqual(hit.images[0].normalizedVisionAnalysis, image.normalizedVisionAnalysis);
|
|
assert.equal(hit.qwen.imageEvidence, miss.qwen.imageEvidence);
|
|
assert.equal(hit.qwen.think, true);
|
|
assert.equal(hit.visibleReasoning, "Compare the supplied visible evidence.");
|
|
assert.equal(hit.search.requested, true);
|
|
assert.equal(hit.search.needed, true);
|
|
assert.equal(hit.search.explicitSearchIntent, true);
|
|
assert.equal(hit.search.state, "succeeded");
|
|
assert.match(hit.search.visualReference, /thin black line beneath chin/i);
|
|
assert.deepEqual(hit.search.queries, s.searchQueries);
|
|
assert.equal(hit.search.candidateCount, 1);
|
|
assert.equal(hit.search.selectedCount, 1);
|
|
assert.deepEqual(hit.search.selectedSources[0], {index: 1, title: "Penguin identification guide",
|
|
url: "https://example.com/penguins", snippet: "A penguin identification guide compares crown, throat, bill, and chin markings."});
|
|
assert.equal(hit.finalAnswer, "Final penguin answer.");
|
|
const latest = await s.request("/debug/vision/latest", {cookie: owner});
|
|
assert.equal((await latest.json()).diagnostic.requestId, hit.requestId);
|
|
});
|
|
|
|
test("diagnostics distinguish skipped search and preserve multiple and historical image provenance", async t => {
|
|
const s = await setup(t, true);
|
|
const cookie = await s.register("debug-provenance");
|
|
const chatId = await s.createChat(cookie);
|
|
const red = await sharp({create: {width: 20, height: 10, channels: 3, background: "red"}}).png().toBuffer();
|
|
const blue = await sharp({create: {width: 30, height: 15, channels: 3, background: "blue"}}).png().toBuffer();
|
|
const first = await s.upload(cookie, red, "first.png");
|
|
const second = await s.upload(cookie, blue, "second.png");
|
|
const comparison = await s.request("/chat", {cookie, method: "POST", body: {
|
|
chatId, message: "What species are these?", attachments: [first.id, second.id], webSearch: true
|
|
}});
|
|
const current = await getDiagnostic(s, cookie, comparison.headers.get("x-request-id"));
|
|
assert.deepEqual(current.images.map(image => [image.attachmentId, image.position, image.scope, image.label]), [
|
|
[first.id, 0, "current", "Current image 1"], [second.id, 1, "current", "Current image 2"]
|
|
]);
|
|
assert.notEqual(current.images[0].imageSha256, current.images[1].imageSha256);
|
|
assert.equal(current.search.requested, true);
|
|
assert.equal(current.search.needed, false);
|
|
assert.equal(current.search.state, "skipped");
|
|
assert.deepEqual(current.search.queries, []);
|
|
assert.equal(current.search.candidateCount, 0);
|
|
|
|
const historicalResponse = await s.request("/chat", {cookie, method: "POST", body: {
|
|
chatId, message: "What did the previous image show?"
|
|
}});
|
|
const historical = await getDiagnostic(s, cookie, historicalResponse.headers.get("x-request-id"));
|
|
assert.equal(historical.images.length, 1);
|
|
assert.equal(historical.images[0].attachmentId, second.id);
|
|
assert.equal(historical.images[0].position, 1);
|
|
assert.equal(historical.images[0].scope, "historical");
|
|
assert.match(historical.images[0].label, /Earlier image from message/);
|
|
});
|