improve reasoning toggle tests and refactor model capability checks
This commit is contained in:
@@ -0,0 +1,49 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {appendAttachments, createAttachmentComposer} from "../public/app/attachments.js";
|
||||
|
||||
class Element {
|
||||
constructor() { this.children = []; this.listeners = {}; this.classList = {add() {}, remove() {}}; }
|
||||
append(...children) { this.children.push(...children); }
|
||||
replaceChildren(...children) { this.children = children; }
|
||||
setAttribute() {}
|
||||
addEventListener(name, listener) { this.listeners[name] = listener; }
|
||||
click() { this.clicked = true; }
|
||||
}
|
||||
test("composer preview, pending state, upload failure/retry, removal, paste, and history rendering", async t => {
|
||||
const originalDocument = globalThis.document, originalFetch = globalThis.fetch;
|
||||
t.after(() => { globalThis.document = originalDocument; globalThis.fetch = originalFetch; });
|
||||
const elements = Object.fromEntries(["attachImage", "imagePicker", "uploadPreviews", "composer"].map(id => [id, new Element()]));
|
||||
globalThis.document = {getElementById: id => elements[id], querySelector: () => elements.composer, createElement: () => new Element()};
|
||||
let completeUpload;
|
||||
let changes = 0;
|
||||
globalThis.fetch = async (url) => {
|
||||
if (url === "/uploads/config") return Response.json({maxFiles: 5, maxSizeMb: 10});
|
||||
if (url === "/uploads") return new Promise(resolve => { completeUpload = resolve; });
|
||||
return Response.json({success: true});
|
||||
};
|
||||
const composer = createAttachmentComposer({onChange: () => changes++, onError: message => assert.fail(message)});
|
||||
elements.attachImage.onclick(); assert.equal(elements.imagePicker.clicked, true);
|
||||
elements.imagePicker.files = [new File(["image fixture"], "screenshot.png", {type: "image/png"})];
|
||||
elements.imagePicker.onchange();
|
||||
assert.equal(composer.pending, true); assert.equal(composer.attachments.length, 0);
|
||||
assert.equal(elements.uploadPreviews.children.length, 1);
|
||||
const tick = () => new Promise(resolve => setTimeout(resolve, 0));
|
||||
completeUpload(Response.json({error: "Malformed image"}, {status: 400})); await tick();
|
||||
assert.match(elements.uploadPreviews.children[0].children[1].textContent, /Malformed image/);
|
||||
elements.uploadPreviews.children[0].children[2].onclick();
|
||||
const metadata = {id: "12345678-1234-1234-1234-123456789abc", originalName: "screenshot.png"};
|
||||
completeUpload(Response.json(metadata, {status: 201})); await tick();
|
||||
assert.equal(composer.pending, false); assert.deepEqual(composer.attachments, [metadata]);
|
||||
composer.setBusy(true); assert.equal(elements.attachImage.disabled, true);
|
||||
composer.setBusy(false);
|
||||
elements.uploadPreviews.children[0].children.at(-1).onclick();
|
||||
assert.equal(composer.attachments.length, 0); assert.equal(elements.uploadPreviews.hidden, true);
|
||||
let prevented = false;
|
||||
elements.composer.listeners.paste({clipboardData: {files: [], getData: () => "text"}, preventDefault: () => { prevented = true; }});
|
||||
assert.equal(prevented, false);
|
||||
const history = new Element(); appendAttachments(history); assert.equal(history.children.length, 0);
|
||||
appendAttachments(history, [metadata]);
|
||||
assert.equal(history.children[0].children[0].href, `/attachments/${metadata.id}`);
|
||||
assert.ok(changes > 4);
|
||||
});
|
||||
@@ -0,0 +1,236 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import sharp from "sharp";
|
||||
import {createServer} from "node:http";
|
||||
import {createApp} from "../src/app.js";
|
||||
import {createAttachmentService, parseVisionAnalysis} from "../src/services/attachmentService.js";
|
||||
import {createDatabase} from "../src/db/index.js";
|
||||
import {createConfig} from "../src/config.js";
|
||||
|
||||
async function setup(t) {
|
||||
const directory = await fs.mkdtemp(path.join(os.tmpdir(), "elsewhere-images-"));
|
||||
const requests = [];
|
||||
const order = [];
|
||||
const searchServer = createServer((req, res) => {
|
||||
order.push("search");
|
||||
res.setHeader("Content-Type", "application/json");
|
||||
res.end(JSON.stringify({results: [{title: "Reasoning button documentation", url: "https://example.com/reasoning", content: "The Reasoning button is disabled when the selected model does not support reasoning."}]}));
|
||||
});
|
||||
await new Promise(resolve => searchServer.listen(0, "127.0.0.1", resolve));
|
||||
let visionFailure = false;
|
||||
let holdVision = false;
|
||||
let visionAborted = false;
|
||||
let startedVision;
|
||||
const visionStarted = new Promise(resolve => { startedVision = resolve; });
|
||||
const app = createApp({config: {dbPath: ":memory:", uploadDir: directory, uploadMaxSizeMb: 1, uploadMaxPixels: 10000,
|
||||
testMode: true, sessionSecret: "attachment-tests", chatSummaryUpdateEveryMessages: 0,
|
||||
searxngUrl: `http://127.0.0.1:${searchServer.address().port}`,
|
||||
eventLoopLagMonitorEnabled: false, slowRequestLoggingEnabled: false, structuredRequestLoggingEnabled: false},
|
||||
modelOverrides: {fetch: async (_url, options) => {
|
||||
const body = JSON.parse(options.body); requests.push(body);
|
||||
order.push(body.model === "gemma3:4b" ? "vision" : "answer");
|
||||
if (holdVision && body.model === "gemma3:4b") return new Promise((_resolve, reject) => {
|
||||
startedVision();
|
||||
options.signal.addEventListener("abort", () => { visionAborted = true; reject(new DOMException("Aborted", "AbortError")); }, {once: true});
|
||||
});
|
||||
if (visionFailure && body.model === "gemma3:4b") return new Response("Unavailable", {status: 503});
|
||||
const content = body.model === "gemma3:4b" ? JSON.stringify({summary: "A blue screenshot", visibleText: ["Reasoning"], observations: ["A disabled button"], uncertainties: []}) : "The button is disabled.";
|
||||
return new Response([
|
||||
...(body.think ? [{message: {thinking: "Inspect evidence"}}] : []),
|
||||
{message: {content: content.slice(0, 5)}}, {message: {content: content.slice(5)}, done: true}
|
||||
].map(v => JSON.stringify(v)).join("\n"));
|
||||
}}});
|
||||
const server = app.listen(0, "127.0.0.1");
|
||||
await new Promise(resolve => server.once("listening", resolve));
|
||||
t.after(async () => {
|
||||
await new Promise(resolve => { server.closeAllConnections(); server.close(resolve); });
|
||||
await new Promise(resolve => { searchServer.closeAllConnections(); searchServer.close(resolve); });
|
||||
await app.locals.close(); await fs.rm(directory, {recursive: true, force: true});
|
||||
});
|
||||
const base = `http://127.0.0.1:${server.address().port}`;
|
||||
const register = async username => {
|
||||
const response = await fetch(base + "/register", {method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify({username, password: "test-password"})});
|
||||
assert.equal(response.status, 200); return response.headers.get("set-cookie").split(";")[0];
|
||||
};
|
||||
const cookie = await register("images");
|
||||
const api = (url, body, method = body ? "POST" : "GET", auth = cookie) => fetch(base + url, {method,
|
||||
headers: {Cookie: auth, "Content-Type": "application/json"}, body: body ? JSON.stringify(body) : undefined});
|
||||
const upload = (buffer, filename = "image.png", auth = cookie) => {
|
||||
const form = new FormData(); form.append("image", new Blob([buffer], {type: "image/png"}), filename);
|
||||
return fetch(base + "/uploads", {method: "POST", headers: auth ? {Cookie: auth} : {}, body: form});
|
||||
};
|
||||
const image = format => sharp({create: {width: 20, height: 10, channels: 3, background: "blue"}}).toFormat(format).toBuffer();
|
||||
const chat = async () => (await (await api("/chats", {title: "Image test"})).json()).chat.id;
|
||||
return {app, api, upload, image, register, directory, requests, order, chat,
|
||||
failVision: () => { visionFailure = true; }, holdVision: () => { holdVision = true; return visionStarted; },
|
||||
get visionAborted() { return visionAborted; }};
|
||||
}
|
||||
|
||||
test("validated local uploads, ownership, limits and temporary cleanup", async t => {
|
||||
const s = await setup(t);
|
||||
for (const format of ["png", "jpeg", "webp"]) {
|
||||
const bytes = await s.image(format);
|
||||
const response = await s.upload(bytes, "../../untrusted.png");
|
||||
assert.equal(response.status, 201);
|
||||
const metadata = await response.json();
|
||||
assert.equal(metadata.mimeType, `image/${format}`);
|
||||
assert.equal(metadata.width, 20); assert.equal(metadata.height, 10);
|
||||
assert.equal(metadata.path, undefined); assert.equal(metadata.storageKey, undefined);
|
||||
const row = s.app.locals.db.prepare("SELECT * FROM attachments WHERE id=?").get(metadata.id);
|
||||
assert.equal(row.status, "temporary");
|
||||
assert.match(row.storage_key, /^\d{4}\/\d{2}\/\d{2}\/[a-f0-9-]{36}\.(png|jpeg|webp)$/);
|
||||
assert.deepEqual(await fs.readFile(path.join(s.directory, row.storage_key)), bytes);
|
||||
const download = await s.api(`/attachments/${metadata.id}`);
|
||||
assert.equal(download.status, 200); assert.equal(download.headers.get("x-content-type-options"), "nosniff");
|
||||
assert.deepEqual(Buffer.from(await download.arrayBuffer()), bytes);
|
||||
}
|
||||
assert.equal((await s.upload(await s.image("png"), "image.png", "")).status, 403);
|
||||
for (const buffer of [Buffer.from("MZ executable"), Buffer.from("<svg></svg>"), Buffer.alloc(0), (await s.image("png")).subarray(0, 40)]) {
|
||||
assert.equal((await s.upload(buffer)).status, 400);
|
||||
}
|
||||
assert.equal((await s.upload(Buffer.alloc(1024 * 1024 + 1))).status, 413);
|
||||
assert.equal((await s.upload(await sharp({create: {width: 101, height: 101, channels: 3, background: "red"}}).png().toBuffer())).status, 400);
|
||||
const other = await s.register("other-images");
|
||||
const row = s.app.locals.db.prepare("SELECT * FROM attachments LIMIT 1").get();
|
||||
assert.equal((await s.api(`/attachments/${row.id}`, null, "GET", other)).status, 404);
|
||||
assert.equal((await s.api("/attachments/%2Fetc%2Fpasswd")).status, 404);
|
||||
assert.throws(() => s.app.locals.attachmentService.storage.resolve("../../secret"));
|
||||
const chatId = await s.chat();
|
||||
assert.equal((await s.api("/chat", {chatId, message: "Image", attachments: [row.id]}, "POST", other)).status, 404);
|
||||
const otherChat = (await (await s.api("/chats", {title: "Other"}, "POST", other)).json()).chat.id;
|
||||
assert.equal((await s.api("/chat", {chatId: otherChat, message: "Image", attachments: [row.id]}, "POST", other)).status, 404);
|
||||
assert.equal((await s.api("/chat", {chatId, message: "Image", attachments: ["/etc/passwd"]})).status, 400);
|
||||
assert.equal((await s.api("/chat", {chatId, message: "Image", attachments: Array(6).fill(row.id)})).status, 400);
|
||||
s.app.locals.db.prepare("UPDATE attachments SET created_at=datetime('now','-25 hours')").run();
|
||||
await s.app.locals.attachmentService.cleanup();
|
||||
assert.equal(s.app.locals.db.prepare("SELECT COUNT(*) n FROM attachments").get().n, 0);
|
||||
await assert.rejects(fs.access(path.join(s.directory, row.storage_key)));
|
||||
});
|
||||
|
||||
test("account rename retains ownership and account deletion removes pending files", async t => {
|
||||
const s = await setup(t);
|
||||
const image = await (await s.upload(await s.image("png"))).json();
|
||||
const db = s.app.locals.db;
|
||||
const key = db.prepare("SELECT storage_key FROM attachments WHERE id=?").get(image.id).storage_key;
|
||||
// The attachment FK follows the existing username-based account model.
|
||||
db.prepare("UPDATE users SET username=? WHERE username=?").run("renamed-images", "images");
|
||||
assert.equal(s.app.locals.attachmentService.owned(image.id, "renamed-images").username, "renamed-images");
|
||||
s.app.locals.repositories.deleteUser("renamed-images");
|
||||
await s.app.locals.attachmentService.cleanup();
|
||||
assert.equal(db.prepare("SELECT COUNT(*) n FROM attachments").get().n, 0);
|
||||
await assert.rejects(fs.access(path.join(s.directory, key)));
|
||||
});
|
||||
|
||||
test("images feed dedicated vision, cache, streamed reasoning, reload and shared branch lifecycle", async t => {
|
||||
const s = await setup(t);
|
||||
const uploads = [];
|
||||
for (const format of ["png", "jpeg"]) uploads.push(await (await s.upload(await s.image(format))).json());
|
||||
const chatId = await s.chat();
|
||||
const response = await s.api("/chat/stream", {chatId, message: "Why is the button disabled?", attachments: uploads.map(v => v.id), model: "qwen3:14b", reasoning: true});
|
||||
const wire = await response.text();
|
||||
assert.match(wire, /"status":"analyzing"/); assert.match(wire, /event: thinking/); assert.match(wire, /event: chunk/); assert.match(wire, /event: done/);
|
||||
const vision = s.requests.filter(v => v.model === "gemma3:4b");
|
||||
assert.equal(vision.length, 2);
|
||||
for (const request of vision) {
|
||||
assert.equal(request.think, false);
|
||||
assert.equal(request.format, "json");
|
||||
assert.match(request.messages.at(-1).content, /Why is the button disabled/);
|
||||
assert.ok(Buffer.from(request.messages.at(-1).images[0], "base64").length);
|
||||
}
|
||||
const final = s.requests.at(-1);
|
||||
assert.equal(final.model, "qwen3:14b"); assert.equal(final.think, true); assert.equal(final.stream, true);
|
||||
assert.ok(final.messages.some(v => v.content.includes("A blue screenshot")));
|
||||
assert.ok(final.messages.every(v => !v.images)); assert.ok(!JSON.stringify(final).includes(s.directory));
|
||||
const db = s.app.locals.db;
|
||||
assert.equal(db.prepare("SELECT COUNT(*) n FROM attachments WHERE status='attached' AND vision_analysis IS NOT NULL").get().n, 2);
|
||||
const history = await (await s.api(`/chats/${chatId}/messages`)).json();
|
||||
assert.deepEqual(history.messages[0].attachments, uploads);
|
||||
const exported = await (await s.api("/exports/workspace")).json();
|
||||
assert.deepEqual(exported.workspace.chats[0].messages[0].attachments, uploads);
|
||||
const botId = history.messages.at(-1).id;
|
||||
assert.equal((await s.api(`/chats/${chatId}/messages/${botId}/retry`, {model: "qwen3:14b"})).status, 200);
|
||||
assert.equal(s.requests.filter(v => v.model === "gemma3:4b").length, 2, "Retry reuses vision cache");
|
||||
const branchResponse = await s.api(`/chats/${chatId}/messages/${botId}/branch`, {confirmed: true});
|
||||
assert.equal(branchResponse.status, 200);
|
||||
const branch = await branchResponse.json();
|
||||
await s.api(`/chats/${chatId}`, null, "DELETE"); await s.app.locals.attachmentService.cleanup();
|
||||
assert.equal((await s.api(`/attachments/${uploads[0].id}`)).status, 200, "Branch retains shared file");
|
||||
await s.api(`/chats/${branch.chat.id}`, null, "DELETE"); await s.app.locals.attachmentService.cleanup();
|
||||
assert.equal(db.prepare("SELECT COUNT(*) n FROM attachments").get().n, 0);
|
||||
assert.equal((await s.api(`/attachments/${uploads[0].id}`)).status, 404);
|
||||
assert.equal(s.app.locals.db.prepare("SELECT COUNT(*) n FROM upload_deletions").get().n, 0);
|
||||
});
|
||||
|
||||
test("images compose with all Web Search and Reasoning combinations in order", async t => {
|
||||
const s = await setup(t);
|
||||
const chatId = await s.chat();
|
||||
for (const webSearch of [false, true]) for (const reasoning of [false, true]) {
|
||||
const attachment = await (await s.upload(await s.image("png"))).json();
|
||||
s.order.length = 0;
|
||||
const wire = await (await s.api("/chat/stream", {chatId, message: "Search the web for the Reasoning button documentation", attachments: [attachment.id], webSearch, reasoning})).text();
|
||||
assert.deepEqual(s.order, webSearch ? ["vision", "search", "answer"] : ["vision", "answer"]);
|
||||
const final = s.requests.at(-1);
|
||||
assert.equal(final.think, reasoning); assert.equal(final.stream, true);
|
||||
assert.ok(final.messages.some(v => v.content.includes("IMAGE OBSERVATIONS")));
|
||||
assert.equal(final.messages.some(v => v.content.startsWith("<web_search_results>")), webSearch);
|
||||
assert.equal(wire.includes("event: thinking"), reasoning);
|
||||
assert.equal(wire.includes('"status":"searching"'), webSearch);
|
||||
assert.equal(wire.includes('"sources":[{'), webSearch);
|
||||
}
|
||||
});
|
||||
|
||||
test("vision failure prevents an ungrounded answer and malformed JSON falls back", async t => {
|
||||
const s = await setup(t);
|
||||
const attachment = await (await s.upload(await s.image("png"))).json();
|
||||
s.failVision();
|
||||
const wire = await (await s.api("/chat/stream", {chatId: await s.chat(), message: "Explain", attachments: [attachment.id]})).text();
|
||||
assert.match(wire, /event: error/); assert.match(wire, /Image analysis failed/);
|
||||
assert.ok(!s.requests.some(v => v.model === "qwen3:14b"));
|
||||
assert.equal(parseVisionAnalysis("Plain visual description").summary, "Plain visual description");
|
||||
assert.equal(parseVisionAnalysis('{"summary":"UI","visibleText":[42,"Hello"]}').visibleText.length, 1);
|
||||
assert.throws(() => parseVisionAnalysis(""));
|
||||
});
|
||||
|
||||
test("browser disconnect cancels in-progress vision and never starts the final model", async t => {
|
||||
const s = await setup(t);
|
||||
const attachment = await (await s.upload(await s.image("png"))).json();
|
||||
const started = s.holdVision();
|
||||
const response = await s.api("/chat/stream", {chatId: await s.chat(), message: "Describe", attachments: [attachment.id]});
|
||||
await started;
|
||||
await response.body.cancel();
|
||||
for (let attempt = 0; attempt < 100 && !s.visionAborted; attempt++) await new Promise(resolve => setTimeout(resolve, 5));
|
||||
assert.equal(s.visionAborted, true);
|
||||
assert.equal(s.requests.length, 1);
|
||||
assert.equal(s.app.locals.db.prepare("SELECT vision_analysis FROM attachments WHERE id=?").get(attachment.id).vision_analysis, null);
|
||||
});
|
||||
|
||||
test("additive schema, files and cached vision survive database reopen; message deletion releases them", async t => {
|
||||
const directory = await fs.mkdtemp(path.join(os.tmpdir(), "elsewhere-image-persistence-"));
|
||||
const config = createConfig({dbPath: path.join(directory, "app.sqlite"), dataDir: directory, uploadDir: path.join(directory, "uploads"), structuredRequestLoggingEnabled: false});
|
||||
let calls = 0;
|
||||
const model = {async generateReply() { calls++; return '{"summary":"Persisted evidence"}'; }};
|
||||
let db = createDatabase(config);
|
||||
let service = createAttachmentService(db, config, model);
|
||||
t.after(async () => { await service.close(); db.close(); await fs.rm(directory, {recursive: true, force: true}); });
|
||||
db.prepare("INSERT INTO users(username,password) VALUES('persist','test')").run();
|
||||
const chatId = Number(db.prepare("INSERT INTO chat_sessions(username,title) VALUES('persist','Images')").run().lastInsertRowid);
|
||||
const buffer = await sharp({create: {width: 2, height: 2, channels: 3, background: "blue"}}).png().toBuffer();
|
||||
const attachment = await service.upload({buffer, size: buffer.length, originalname: "persistent.png"}, "persist");
|
||||
const messageId = Number(service.createMessage([attachment.id], "persist", chatId, () => db.prepare("INSERT INTO chat_messages(chat_id,role,content) VALUES(?,'user','Describe')").run(chatId)).lastInsertRowid);
|
||||
await service.analyze(messageId, "Describe");
|
||||
await service.close(); db.close();
|
||||
db = createDatabase(config); service = createAttachmentService(db, config, model);
|
||||
assert.deepEqual(service.list(messageId), [attachment]);
|
||||
const row = service.owned(attachment.id, "persist");
|
||||
assert.deepEqual(await service.storage.read(row.storage_key), buffer);
|
||||
await service.analyze(messageId, "Describe"); assert.equal(calls, 1);
|
||||
config.visionModel = "gemma3:updated";
|
||||
await service.analyze(messageId, "Describe"); assert.equal(calls, 2);
|
||||
db.prepare("DELETE FROM chat_messages WHERE id=?").run(messageId);
|
||||
await service.cleanup();
|
||||
assert.equal(db.prepare("SELECT COUNT(*) n FROM attachments").get().n, 0);
|
||||
await assert.rejects(service.storage.read(row.storage_key));
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {createServer} from "node:http";
|
||||
import {createApp} from "../src/app.js";
|
||||
|
||||
// Controlled protocol fixtures, not claims about real-world events or statistics.
|
||||
const lifespan = {title: "Life expectancy estimates 2026", url: "https://health.example/2026", content: "The 2026 life expectancy estimate in this test dataset is 74 years."};
|
||||
const events = [
|
||||
{title: "Satellite launched", url: "https://space.example/launch", content: "On September 10, 2025, researchers launched a satellite into orbit."},
|
||||
{title: "Cooperation agreement", url: "https://policy.example/deal", content: "Officials signed a cooperation agreement on September 10, 2025."}
|
||||
];
|
||||
async function listen(t, handler) {
|
||||
const server = createServer(handler);
|
||||
await new Promise(resolve => server.listen(0, "127.0.0.1", resolve));
|
||||
t.after(() => new Promise(resolve => { server.closeAllConnections(); server.close(resolve); }));
|
||||
return `http://127.0.0.1:${server.address().port}`;
|
||||
}
|
||||
|
||||
test("explicit/date searches ground the actual Ollama stream before any generation", async (t) => {
|
||||
const queries = [];
|
||||
const modelRequests = [];
|
||||
let retrieved = false;
|
||||
let expectedSearch = false;
|
||||
let searchMode = "normal";
|
||||
const searxngUrl = await listen(t, (req, res) => {
|
||||
const query = new URL(req.url, "http://localhost").searchParams.get("q");
|
||||
queries.push(query);
|
||||
res.setHeader("Content-Type", "application/json");
|
||||
const results = searchMode === "empty" ? [] : searchMode === "irrelevant"
|
||||
? [{title: "Unrelated document", url: "https://other.example", content: "This document describes medieval pottery and ceramics."}]
|
||||
: query.includes("2026") ? [lifespan] : events;
|
||||
setTimeout(() => {
|
||||
retrieved = true;
|
||||
res.statusCode = searchMode === "failed" ? 503 : 200;
|
||||
res.end(JSON.stringify({results}));
|
||||
}, 10);
|
||||
});
|
||||
const app = createApp({config: {
|
||||
dbPath: ":memory:", testMode: true, sessionSecret: "test-only", searxngUrl,
|
||||
webSearchTimeoutMs: 1000, chatSummaryUpdateEveryMessages: 0,
|
||||
eventLoopLagMonitorEnabled: false, structuredRequestLoggingEnabled: false, slowRequestLoggingEnabled: false
|
||||
}, modelOverrides: {fetch: async (_url, options) => {
|
||||
const body = JSON.parse(options.body);
|
||||
modelRequests.push(body);
|
||||
if (expectedSearch) assert.equal(retrieved, true, "Search must finish before Ollama starts");
|
||||
const chunks = [
|
||||
...(body.think ? [{message: {thinking: "Evaluate the supplied evidence."}}] : []),
|
||||
{message: {content: "Supported "}}, {message: {content: "answer [1]."}, done: true}
|
||||
];
|
||||
return new Response(chunks.map(chunk => JSON.stringify(chunk)).join("\n"));
|
||||
}}});
|
||||
t.after(() => app.locals.close());
|
||||
const base = await listen(t, app);
|
||||
let cookie;
|
||||
const api = async (path, body) => {
|
||||
const response = await fetch(base + path, {
|
||||
method: body ? "POST" : "GET",
|
||||
headers: {"Content-Type": "application/json", ...(cookie ? {Cookie: cookie} : {})},
|
||||
body: body ? JSON.stringify(body) : undefined
|
||||
});
|
||||
if (response.headers.get("set-cookie")) cookie = response.headers.get("set-cookie").split(";")[0];
|
||||
assert.equal(response.status, 200);
|
||||
return response.json();
|
||||
};
|
||||
await api("/register", {username: "grounding", password: "test-password"});
|
||||
const chatId = (await api("/chats", {title: "Grounding regressions"})).chat.id;
|
||||
const send = async (message, webSearch, reasoning) => {
|
||||
retrieved = false;
|
||||
const response = await fetch(base + "/chat/stream", {
|
||||
method: "POST", headers: {"Content-Type": "application/json", Cookie: cookie},
|
||||
body: JSON.stringify({chatId, model: "qwen3:14b", message, webSearch, reasoning})
|
||||
});
|
||||
assert.equal(response.status, 200);
|
||||
return (await response.text()).trim().split("\n\n").map(block => {
|
||||
const [event, data] = block.split("\n");
|
||||
return {event: event.slice(7), data: JSON.parse(data.slice(6))};
|
||||
});
|
||||
};
|
||||
for (const message of ["Search the internet for the average lifespan in 2026", "what happened on 10.09.2025?"]) {
|
||||
for (const webSearch of [false, true]) for (const reasoning of [false, true]) {
|
||||
expectedSearch = webSearch;
|
||||
const before = queries.length;
|
||||
const wire = await send(message, webSearch, reasoning);
|
||||
const request = modelRequests.at(-1);
|
||||
const done = wire.at(-1);
|
||||
assert.equal(done.event, "done");
|
||||
assert.equal(request.think, reasoning);
|
||||
assert.equal(request.stream, true);
|
||||
assert.equal(queries.length - before, webSearch ? 1 : 0);
|
||||
assert.equal(wire.some(event => event.data.status === "searching"), webSearch);
|
||||
assert.equal(wire.some(event => event.event === "thinking"), reasoning);
|
||||
assert.equal(wire.filter(event => event.event === "chunk").length, 2);
|
||||
assert.match(request.messages[0].content, /Current date and time:/);
|
||||
assert.equal(request.messages.at(-1).content, message);
|
||||
const context = request.messages.find(entry => entry.content.startsWith("<web_search_results>"));
|
||||
assert.equal(Boolean(context), webSearch);
|
||||
assert.equal(done.data.searchState, webSearch ? "succeeded" : "skipped");
|
||||
if (webSearch) {
|
||||
const expected = (message.includes("2026") ? [lifespan] : events).map(({title, url, content}) => ({title, url, snippet: content}));
|
||||
assert.deepEqual(done.data.sources, expected);
|
||||
assert.equal(request.messages.at(-2), context);
|
||||
assert.match(request.messages.at(-3).content, /Evaluate the numbered titles and snippets/);
|
||||
const numbered = expected.map((source, index) => `[${index + 1}]\nTitle: ${source.title}\nURL: ${source.url}\nSnippet: ${source.snippet}`).join("\n\n");
|
||||
assert.equal(context.content, `<web_search_results>\n${numbered}\n</web_search_results>`);
|
||||
const stored = (await api(`/chats/${chatId}/messages`)).messages.at(-1);
|
||||
assert.deepEqual(stored.sources, expected);
|
||||
} else assert.equal(done.data.sources, undefined);
|
||||
}
|
||||
}
|
||||
expectedSearch = false;
|
||||
const before = queries.length;
|
||||
const arithmetic = await send("What is 2 + 2?", true, false);
|
||||
assert.equal(queries.length, before);
|
||||
assert.equal(arithmetic.at(-1).data.searchState, "skipped");
|
||||
expectedSearch = true;
|
||||
await send("Search the web for what is 2 + 2", true, false);
|
||||
assert.equal(queries.length, before + 1);
|
||||
for (const [mode, state] of [["empty", "no_results"], ["irrelevant", "no_relevant_results"]]) {
|
||||
searchMode = mode;
|
||||
const wire = await send("Search the internet for the average lifespan in 2026", true, false);
|
||||
assert.equal(wire.at(-1).data.searchState, state);
|
||||
assert.deepEqual(wire.at(-1).data.sources, []);
|
||||
assert.ok(modelRequests.at(-1).messages.some(message => message.content.includes("Tell the user you searched")));
|
||||
}
|
||||
searchMode = "failed";
|
||||
const requestsBefore = modelRequests.length;
|
||||
const failure = await send("Search the internet for the average lifespan in 2026", true, false);
|
||||
assert.equal(failure.at(-1).event, "error");
|
||||
assert.equal(failure.at(-1).data.searchState, "failed");
|
||||
assert.equal(modelRequests.length, requestsBefore);
|
||||
});
|
||||
Reference in New Issue
Block a user