adjust attachments to hopefully fix them?
This commit is contained in:
Frederik Spirgi
+141
-11
@@ -11,12 +11,14 @@ import {createDatabase} from "../src/db/index.js";
|
||||
import {createConfig} from "../src/config.js";
|
||||
import {VISION_PROMPT_VERSION, VISION_RESPONSE_SCHEMA} from "../src/services/visionAnalysis.js";
|
||||
|
||||
async function setup(t, {visionResult = null} = {}) {
|
||||
async function setup(t, {visionResult = null, replyResult = null} = {}) {
|
||||
const directory = await fs.mkdtemp(path.join(os.tmpdir(), "elsewhere-images-"));
|
||||
const requests = [];
|
||||
const order = [];
|
||||
const searchQueries = [];
|
||||
const searchServer = createServer((req, res) => {
|
||||
order.push("search");
|
||||
searchQueries.push(new URL(req.url, "http://localhost").searchParams.get("q"));
|
||||
res.setHeader("Content-Type", "application/json");
|
||||
res.end(JSON.stringify({results: [{title: "Reasoning button documentation", url: "https://example.com/reasoning", content: "The Reasoning button is disabled when the selected model does not support reasoning."}]}));
|
||||
});
|
||||
@@ -39,8 +41,8 @@ async function setup(t, {visionResult = null} = {}) {
|
||||
});
|
||||
if (visionFailure && body.model === "gemma3:4b") return new Response("Unavailable", {status: 503});
|
||||
const content = body.model === "gemma3:4b"
|
||||
? visionResult ? visionResult(body) : JSON.stringify({summary: "A blue screenshot", visibleText: ["Reasoning"], observations: ["A disabled button"], uncertainties: []})
|
||||
: "The button is disabled.";
|
||||
? visionResult ? await visionResult(body) : JSON.stringify({summary: "A blue screenshot", visibleText: ["Reasoning"], observations: ["A disabled button"], uncertainties: []})
|
||||
: replyResult ? await replyResult(body) : "The button is disabled.";
|
||||
return new Response([
|
||||
...(body.think ? [{message: {thinking: "Inspect evidence"}}] : []),
|
||||
{message: {content: content.slice(0, 5)}}, {message: {content: content.slice(5)}, done: true}
|
||||
@@ -67,7 +69,7 @@ async function setup(t, {visionResult = null} = {}) {
|
||||
};
|
||||
const image = format => sharp({create: {width: 20, height: 10, channels: 3, background: "blue"}}).toFormat(format).toBuffer();
|
||||
const chat = async () => (await (await api("/chats", {title: "Image test"})).json()).chat.id;
|
||||
return {app, api, upload, image, register, directory, requests, order, chat,
|
||||
return {app, api, upload, image, register, directory, requests, order, searchQueries, chat,
|
||||
failVision: () => { visionFailure = true; }, holdVision: () => { holdVision = true; return visionStarted; },
|
||||
get visionAborted() { return visionAborted; }};
|
||||
}
|
||||
@@ -250,6 +252,134 @@ const imageEvidence = request => {
|
||||
return JSON.parse(message.content.split("\n")[1]);
|
||||
};
|
||||
|
||||
test("new Paris image wins over earlier penguin image, including regeneration and branch copies", async t => {
|
||||
const penguinBytes = await sharp({create: {width: 20, height: 10, channels: 3, background: "black"}}).png().toBuffer();
|
||||
const parisBytes = await sharp({create: {width: 20, height: 10, channels: 3, background: "green"}}).png().toBuffer();
|
||||
assert.notDeepEqual(penguinBytes, parisBytes);
|
||||
const s = await setup(t, {
|
||||
visionResult: body => {
|
||||
const bytes = Buffer.from(body.messages.at(-1).images[0], "base64");
|
||||
if (bytes.equals(penguinBytes)) return JSON.stringify({summary: "Penguin chick on ice", answer: "Penguin chick"});
|
||||
if (bytes.equals(parisBytes)) return JSON.stringify({summary: "Paris street with a person", answer: "Paris street"});
|
||||
throw new Error("Unexpected image bytes");
|
||||
},
|
||||
replyResult: body => imageEvidence(body).images.find(image => image.scope === "current")?.analysis.summary || "No current image"
|
||||
});
|
||||
const penguin = await (await s.upload(penguinBytes, "penguin.png")).json();
|
||||
const paris = await (await s.upload(parisBytes, "paris.png")).json();
|
||||
const chatId = await s.chat();
|
||||
const first = await (await s.api("/chat", {chatId, message: "What kind of penguin is this?", attachments: [penguin.id]})).json();
|
||||
assert.equal(first.reply, "Penguin chick on ice");
|
||||
const firstBot = (await (await s.api(`/chats/${chatId}/messages`)).json()).messages.at(-1);
|
||||
const second = await (await s.api("/chat", {chatId, message: "What does this image mean?", attachments: [paris.id]})).json();
|
||||
assert.equal(second.reply, "Paris street with a person");
|
||||
const secondMessages = (await (await s.api(`/chats/${chatId}/messages`)).json()).messages;
|
||||
const currentUser = secondMessages.at(-2);
|
||||
const secondBot = secondMessages.at(-1);
|
||||
const visions = s.requests.filter(request => request.model === "gemma3:4b");
|
||||
assert.equal(visions.length, 2, "The previous penguin is not reanalyzed for the Paris question");
|
||||
assert.deepEqual(Buffer.from(visions[1].messages.at(-1).images[0], "base64"), parisBytes);
|
||||
assert.equal(s.app.locals.db.prepare("SELECT COUNT(*) AS n FROM attachment_vision_analyses WHERE attachment_id=? AND question=?")
|
||||
.get(penguin.id, "What does this image mean?").n, 0);
|
||||
const final = s.requests.at(-1);
|
||||
assert.deepEqual(imageEvidence(final).images.map(({id, messageId, position, scope, label}) => ({id, messageId, position, scope, label})),
|
||||
[{id: paris.id, messageId: currentUser.id, position: 0, scope: "current", label: "Current image 1"}]);
|
||||
assert.ok(final.messages.some(message => message.content.includes("Penguin chick on ice")), "Text history remains available");
|
||||
assert.match(final.messages.filter(message => message.role === "system").map(message => message.content).join("\n"),
|
||||
/The following images are attached to the user's current message: Current image 1/);
|
||||
assert.equal(final.messages.at(-1).content, "What does this image mean?");
|
||||
|
||||
assert.equal((await s.api(`/chats/${chatId}/messages/${firstBot.id}/retry`, {})).status, 200);
|
||||
assert.deepEqual(imageEvidence(s.requests.at(-1)).images.map(image => image.id), [penguin.id]);
|
||||
assert.ok(s.requests.at(-1).messages.every(message => !message.content.includes("What does this image mean?")));
|
||||
assert.equal((await s.api(`/chats/${chatId}/messages/${secondBot.id}/retry`, {})).status, 200);
|
||||
assert.deepEqual(imageEvidence(s.requests.at(-1)).images.map(image => image.id), [paris.id]);
|
||||
assert.equal(s.requests.filter(request => request.model === "gemma3:4b").length, 2, "Regeneration reuses per-question caches");
|
||||
|
||||
const branched = await (await s.api(`/chats/${chatId}/messages/${firstBot.id}/branch`, {confirmed: true})).json();
|
||||
assert.equal(branched.messages[0].attachments[0].id, penguin.id, "Branch shares the original attachment ID");
|
||||
const branchParis = await (await s.upload(parisBytes, "branch-paris.png")).json();
|
||||
const branchReply = await (await s.api("/chat", {chatId: branched.chat.id, message: "What does this image mean?", attachments: [branchParis.id]})).json();
|
||||
assert.equal(branchReply.reply, "Paris street with a person");
|
||||
assert.deepEqual(imageEvidence(s.requests.at(-1)).images.map(image => image.id), [branchParis.id]);
|
||||
});
|
||||
|
||||
test("explicit previous-image follow-up and ordered current-image comparison remain available", async t => {
|
||||
const red = await sharp({create: {width: 20, height: 10, channels: 3, background: "red"}}).png().toBuffer();
|
||||
const blue = await sharp({create: {width: 20, height: 10, channels: 3, background: "blue"}}).png().toBuffer();
|
||||
const s = await setup(t, {visionResult: body => JSON.stringify({summary:
|
||||
Buffer.from(body.messages.at(-1).images[0], "base64").equals(red) ? "Red image" : "Blue image"})});
|
||||
const first = await (await s.upload(red, "first.png")).json();
|
||||
const second = await (await s.upload(blue, "second.png")).json();
|
||||
const chatId = await s.chat();
|
||||
await s.api("/chat", {chatId, message: "What is this?", attachments: [first.id]});
|
||||
await s.api("/chat", {chatId, message: "What did the previous image show?"});
|
||||
const historical = imageEvidence(s.requests.at(-1)).images;
|
||||
assert.equal(historical.length, 1);
|
||||
assert.equal(historical[0].id, first.id);
|
||||
assert.equal(historical[0].scope, "historical");
|
||||
assert.match(historical[0].label, /Earlier image from message/);
|
||||
await s.api("/chat", {chatId, message: "How are you?"});
|
||||
assert.equal(s.requests.at(-1).messages.some(message => message.content.startsWith("<image_evidence>")), false);
|
||||
await s.api("/chat", {chatId, message: "Compare this image to the previous one.", attachments: [second.id]});
|
||||
const compared = imageEvidence(s.requests.at(-1)).images;
|
||||
assert.deepEqual(compared.map(image => [image.id, image.scope]), [[second.id, "current"], [first.id, "historical"]]);
|
||||
await s.api("/chat", {chatId, message: "Compare these images.", attachments: [first.id, second.id]});
|
||||
const current = imageEvidence(s.requests.at(-1)).images;
|
||||
assert.deepEqual(current.map(image => image.id), [first.id, second.id]);
|
||||
assert.deepEqual(current.map(image => [image.position, image.scope, image.label]),
|
||||
[[0, "current", "Current image 1"], [1, "current", "Current image 2"]]);
|
||||
});
|
||||
|
||||
test("Web Search uses the current image description instead of an earlier image", async t => {
|
||||
const penguinBytes = await sharp({create: {width: 20, height: 10, channels: 3, background: "black"}}).png().toBuffer();
|
||||
const dashboardBytes = await sharp({create: {width: 20, height: 10, channels: 3, background: "yellow"}}).png().toBuffer();
|
||||
const s = await setup(t, {visionResult: body => JSON.stringify({summary:
|
||||
Buffer.from(body.messages.at(-1).images[0], "base64").equals(penguinBytes) ? "Penguin on ice" : "Dashboard warning lamp"})});
|
||||
const penguin = await (await s.upload(penguinBytes, "penguin.png")).json();
|
||||
const dashboard = await (await s.upload(dashboardBytes, "dashboard.png")).json();
|
||||
const chatId = await s.chat();
|
||||
await s.api("/chat", {chatId, message: "Describe this animal", attachments: [penguin.id]});
|
||||
const wire = await (await s.api("/chat/stream", {chatId, message: "Search the web for whether this dashboard warning is serious.", attachments: [dashboard.id], webSearch: true, reasoning: true})).text();
|
||||
assert.match(wire, /event: done/);
|
||||
assert.match(wire, /event: thinking/);
|
||||
assert.ok(s.searchQueries.length);
|
||||
assert.match(s.searchQueries.join(" "), /Dashboard warning lamp/i);
|
||||
assert.doesNotMatch(s.searchQueries.join(" "), /Penguin on ice/i);
|
||||
assert.deepEqual(imageEvidence(s.requests.at(-1)).images.map(image => image.id), [dashboard.id]);
|
||||
assert.equal(s.requests.at(-1).think, true);
|
||||
});
|
||||
|
||||
test("final Qwen history stays on the triggering message when the active leaf changes during vision", async t => {
|
||||
const penguinBytes = await sharp({create: {width: 20, height: 10, channels: 3, background: "black"}}).png().toBuffer();
|
||||
const parisBytes = await sharp({create: {width: 20, height: 10, channels: 3, background: "green"}}).png().toBuffer();
|
||||
let signalVisionStarted;
|
||||
let releaseVision;
|
||||
const visionStarted = new Promise(resolve => { signalVisionStarted = resolve; });
|
||||
const visionGate = new Promise(resolve => { releaseVision = resolve; });
|
||||
const s = await setup(t, {visionResult: async body => {
|
||||
if (Buffer.from(body.messages.at(-1).images[0], "base64").equals(parisBytes)) {
|
||||
signalVisionStarted();
|
||||
await visionGate;
|
||||
return JSON.stringify({summary: "Paris street scene"});
|
||||
}
|
||||
return JSON.stringify({summary: "Penguin on ice"});
|
||||
}});
|
||||
const penguin = await (await s.upload(penguinBytes, "penguin.png")).json();
|
||||
const paris = await (await s.upload(parisBytes, "paris.png")).json();
|
||||
const chatId = await s.chat();
|
||||
await s.api("/chat", {chatId, message: "What is this penguin?", attachments: [penguin.id]});
|
||||
const firstBot = (await (await s.api(`/chats/${chatId}/messages`)).json()).messages.at(-1);
|
||||
const response = await s.api("/chat/stream", {chatId, message: "What does this Paris image mean?", attachments: [paris.id]});
|
||||
await visionStarted;
|
||||
s.app.locals.repositories.updateChatActiveLeaf(chatId, "images", firstBot.id);
|
||||
releaseVision();
|
||||
assert.match(await response.text(), /event: done/);
|
||||
const final = s.requests.at(-1);
|
||||
assert.equal(final.messages.at(-1).content, "What does this Paris image mean?");
|
||||
assert.deepEqual(imageEvidence(final).images.map(image => image.id), [paris.id]);
|
||||
});
|
||||
|
||||
test("question-aware analysis reaches Qwen intact, separates tasks, and reuses the original task for older regeneration", async t => {
|
||||
const penguin = {summary: "Gray down, black head, large white facial patches", answer: "Emperor penguin chick", confidence: 0.94,
|
||||
evidence: ["Dense gray juvenile down", "Black head", "Large white facial patches"], visibleText: [], observations: [], alternatives: [], uncertainties: []};
|
||||
@@ -273,7 +403,7 @@ test("question-aware analysis reaches Qwen intact, separates tasks, and reuses t
|
||||
const firstBot = (await (await s.api(`/chats/${chatId}/messages`)).json()).messages.at(-1);
|
||||
|
||||
// A follow-up without re-uploading must not reuse the penguin-specific answer.
|
||||
await s.api("/chat", {chatId, message: "Why is the Reasoning button disabled?"});
|
||||
await s.api("/chat", {chatId, message: "In the previous screenshot, why is the Reasoning button disabled?"});
|
||||
assert.equal(s.requests.filter(request => request.model === "gemma3:4b").length, 2);
|
||||
assert.deepEqual(imageEvidence(s.requests.at(-1)).images[0].analysis, ui);
|
||||
assert.equal(s.app.locals.db.prepare("SELECT COUNT(*) n FROM attachment_vision_analyses").get().n, 2);
|
||||
@@ -281,7 +411,7 @@ test("question-aware analysis reaches Qwen intact, separates tasks, and reuses t
|
||||
assert.match(retry, /event: done/);
|
||||
assert.deepEqual(imageEvidence(s.requests.at(-1)).images[0].analysis, penguin);
|
||||
assert.equal(s.requests.filter(request => request.model === "gemma3:4b").length, 2, "Original question remains cached independently");
|
||||
assert.ok(s.requests.at(-1).messages.every(message => !message.content.includes("Why is the Reasoning button disabled?")));
|
||||
assert.ok(s.requests.at(-1).messages.every(message => !message.content.includes("In the previous screenshot, why is the Reasoning button disabled?")));
|
||||
});
|
||||
|
||||
test("legacy, prompt-version, model and corrupt caches invalidate; matching task cache survives and context never falls back to stale data", async t => {
|
||||
@@ -296,18 +426,18 @@ test("legacy, prompt-version, model and corrupt caches invalidate; matching task
|
||||
const calls = () => s.requests.filter(request => request.model === "gemma3:4b").length;
|
||||
assert.equal(calls(), 1, "Legacy model-only cache is invalid");
|
||||
await service.analyze(user.id, " Identify this "); assert.equal(calls(), 1);
|
||||
assert.equal(service.context([user], "Unrelated task"), null);
|
||||
assert.equal(service.context(service.selectImages([user], "Unrelated task"), "Unrelated task"), null);
|
||||
db.prepare("UPDATE attachment_vision_analyses SET prompt_version=?").run(VISION_PROMPT_VERSION - 1);
|
||||
assert.equal(service.context([user], "Identify this"), null);
|
||||
assert.equal(service.context(service.selectImages([user], "Identify this"), "Identify this"), null);
|
||||
await service.analyze(user.id, "Identify this"); assert.equal(calls(), 2);
|
||||
db.prepare("UPDATE attachment_vision_analyses SET vision_model='different-model' WHERE prompt_version=?").run(VISION_PROMPT_VERSION);
|
||||
assert.equal(service.context([user], "Identify this"), null);
|
||||
assert.equal(service.context(service.selectImages([user], "Identify this"), "Identify this"), null);
|
||||
await service.analyze(user.id, "Identify this"); assert.equal(calls(), 3);
|
||||
db.prepare("UPDATE attachment_vision_analyses SET analysis='broken' WHERE vision_model='gemma3:4b' AND prompt_version=?").run(VISION_PROMPT_VERSION);
|
||||
await service.analyze(user.id, "Identify this"); assert.equal(calls(), 4);
|
||||
await service.analyze(user.id, "Read the visible text"); assert.equal(calls(), 5);
|
||||
await service.analyze(user.id, "Identify this"); assert.equal(calls(), 5);
|
||||
const evidence = service.context([user], "Identify this");
|
||||
const evidence = service.context(service.selectImages([user], "Identify this"), "Identify this");
|
||||
assert.ok(evidence.every(message => !message.content.includes("Old vague result")));
|
||||
const row = db.prepare("SELECT * FROM attachment_vision_analyses WHERE vision_model='gemma3:4b' AND prompt_version=? AND question=?").get(VISION_PROMPT_VERSION, "Identify this");
|
||||
assert.ok(row.analyzed_at);
|
||||
@@ -320,7 +450,7 @@ test("malformed visual output remains raw evidence and image instruction text st
|
||||
: JSON.stringify({summary: "Text screenshot", answer: "A visible instruction", confidence: 0.8, visibleText: [attack], evidence: [], alternatives: [], uncertainties: []})});
|
||||
const attachment = await (await s.upload(await s.image("png"))).json();
|
||||
const chatId = await s.chat();
|
||||
for (const question of ["Describe the screenshot", "Read the visible text"]) {
|
||||
for (const question of ["Describe the screenshot", "Read the visible text in the previous screenshot"]) {
|
||||
const wire = await (await s.api("/chat/stream", {chatId, message: question, ...(plain ? {attachments: [attachment.id]} : {})})).text();
|
||||
assert.match(wire, /event: done/); assert.doesNotMatch(wire, /event: error/);
|
||||
const final = s.requests.at(-1);
|
||||
|
||||
Reference in New Issue
Block a user