improve reasoning toggle tests and refactor model capability checks
This commit is contained in:
@@ -0,0 +1,131 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {createServer} from "node:http";
|
||||
import {createApp} from "../src/app.js";
|
||||
|
||||
// Controlled protocol fixtures, not claims about real-world events or statistics.
|
||||
const lifespan = {title: "Life expectancy estimates 2026", url: "https://health.example/2026", content: "The 2026 life expectancy estimate in this test dataset is 74 years."};
|
||||
const events = [
|
||||
{title: "Satellite launched", url: "https://space.example/launch", content: "On September 10, 2025, researchers launched a satellite into orbit."},
|
||||
{title: "Cooperation agreement", url: "https://policy.example/deal", content: "Officials signed a cooperation agreement on September 10, 2025."}
|
||||
];
|
||||
async function listen(t, handler) {
|
||||
const server = createServer(handler);
|
||||
await new Promise(resolve => server.listen(0, "127.0.0.1", resolve));
|
||||
t.after(() => new Promise(resolve => { server.closeAllConnections(); server.close(resolve); }));
|
||||
return `http://127.0.0.1:${server.address().port}`;
|
||||
}
|
||||
|
||||
test("explicit/date searches ground the actual Ollama stream before any generation", async (t) => {
|
||||
const queries = [];
|
||||
const modelRequests = [];
|
||||
let retrieved = false;
|
||||
let expectedSearch = false;
|
||||
let searchMode = "normal";
|
||||
const searxngUrl = await listen(t, (req, res) => {
|
||||
const query = new URL(req.url, "http://localhost").searchParams.get("q");
|
||||
queries.push(query);
|
||||
res.setHeader("Content-Type", "application/json");
|
||||
const results = searchMode === "empty" ? [] : searchMode === "irrelevant"
|
||||
? [{title: "Unrelated document", url: "https://other.example", content: "This document describes medieval pottery and ceramics."}]
|
||||
: query.includes("2026") ? [lifespan] : events;
|
||||
setTimeout(() => {
|
||||
retrieved = true;
|
||||
res.statusCode = searchMode === "failed" ? 503 : 200;
|
||||
res.end(JSON.stringify({results}));
|
||||
}, 10);
|
||||
});
|
||||
const app = createApp({config: {
|
||||
dbPath: ":memory:", testMode: true, sessionSecret: "test-only", searxngUrl,
|
||||
webSearchTimeoutMs: 1000, chatSummaryUpdateEveryMessages: 0,
|
||||
eventLoopLagMonitorEnabled: false, structuredRequestLoggingEnabled: false, slowRequestLoggingEnabled: false
|
||||
}, modelOverrides: {fetch: async (_url, options) => {
|
||||
const body = JSON.parse(options.body);
|
||||
modelRequests.push(body);
|
||||
if (expectedSearch) assert.equal(retrieved, true, "Search must finish before Ollama starts");
|
||||
const chunks = [
|
||||
...(body.think ? [{message: {thinking: "Evaluate the supplied evidence."}}] : []),
|
||||
{message: {content: "Supported "}}, {message: {content: "answer [1]."}, done: true}
|
||||
];
|
||||
return new Response(chunks.map(chunk => JSON.stringify(chunk)).join("\n"));
|
||||
}}});
|
||||
t.after(() => app.locals.close());
|
||||
const base = await listen(t, app);
|
||||
let cookie;
|
||||
const api = async (path, body) => {
|
||||
const response = await fetch(base + path, {
|
||||
method: body ? "POST" : "GET",
|
||||
headers: {"Content-Type": "application/json", ...(cookie ? {Cookie: cookie} : {})},
|
||||
body: body ? JSON.stringify(body) : undefined
|
||||
});
|
||||
if (response.headers.get("set-cookie")) cookie = response.headers.get("set-cookie").split(";")[0];
|
||||
assert.equal(response.status, 200);
|
||||
return response.json();
|
||||
};
|
||||
await api("/register", {username: "grounding", password: "test-password"});
|
||||
const chatId = (await api("/chats", {title: "Grounding regressions"})).chat.id;
|
||||
const send = async (message, webSearch, reasoning) => {
|
||||
retrieved = false;
|
||||
const response = await fetch(base + "/chat/stream", {
|
||||
method: "POST", headers: {"Content-Type": "application/json", Cookie: cookie},
|
||||
body: JSON.stringify({chatId, model: "qwen3:14b", message, webSearch, reasoning})
|
||||
});
|
||||
assert.equal(response.status, 200);
|
||||
return (await response.text()).trim().split("\n\n").map(block => {
|
||||
const [event, data] = block.split("\n");
|
||||
return {event: event.slice(7), data: JSON.parse(data.slice(6))};
|
||||
});
|
||||
};
|
||||
for (const message of ["Search the internet for the average lifespan in 2026", "what happened on 10.09.2025?"]) {
|
||||
for (const webSearch of [false, true]) for (const reasoning of [false, true]) {
|
||||
expectedSearch = webSearch;
|
||||
const before = queries.length;
|
||||
const wire = await send(message, webSearch, reasoning);
|
||||
const request = modelRequests.at(-1);
|
||||
const done = wire.at(-1);
|
||||
assert.equal(done.event, "done");
|
||||
assert.equal(request.think, reasoning);
|
||||
assert.equal(request.stream, true);
|
||||
assert.equal(queries.length - before, webSearch ? 1 : 0);
|
||||
assert.equal(wire.some(event => event.data.status === "searching"), webSearch);
|
||||
assert.equal(wire.some(event => event.event === "thinking"), reasoning);
|
||||
assert.equal(wire.filter(event => event.event === "chunk").length, 2);
|
||||
assert.match(request.messages[0].content, /Current date and time:/);
|
||||
assert.equal(request.messages.at(-1).content, message);
|
||||
const context = request.messages.find(entry => entry.content.startsWith("<web_search_results>"));
|
||||
assert.equal(Boolean(context), webSearch);
|
||||
assert.equal(done.data.searchState, webSearch ? "succeeded" : "skipped");
|
||||
if (webSearch) {
|
||||
const expected = (message.includes("2026") ? [lifespan] : events).map(({title, url, content}) => ({title, url, snippet: content}));
|
||||
assert.deepEqual(done.data.sources, expected);
|
||||
assert.equal(request.messages.at(-2), context);
|
||||
assert.match(request.messages.at(-3).content, /Evaluate the numbered titles and snippets/);
|
||||
const numbered = expected.map((source, index) => `[${index + 1}]\nTitle: ${source.title}\nURL: ${source.url}\nSnippet: ${source.snippet}`).join("\n\n");
|
||||
assert.equal(context.content, `<web_search_results>\n${numbered}\n</web_search_results>`);
|
||||
const stored = (await api(`/chats/${chatId}/messages`)).messages.at(-1);
|
||||
assert.deepEqual(stored.sources, expected);
|
||||
} else assert.equal(done.data.sources, undefined);
|
||||
}
|
||||
}
|
||||
expectedSearch = false;
|
||||
const before = queries.length;
|
||||
const arithmetic = await send("What is 2 + 2?", true, false);
|
||||
assert.equal(queries.length, before);
|
||||
assert.equal(arithmetic.at(-1).data.searchState, "skipped");
|
||||
expectedSearch = true;
|
||||
await send("Search the web for what is 2 + 2", true, false);
|
||||
assert.equal(queries.length, before + 1);
|
||||
for (const [mode, state] of [["empty", "no_results"], ["irrelevant", "no_relevant_results"]]) {
|
||||
searchMode = mode;
|
||||
const wire = await send("Search the internet for the average lifespan in 2026", true, false);
|
||||
assert.equal(wire.at(-1).data.searchState, state);
|
||||
assert.deepEqual(wire.at(-1).data.sources, []);
|
||||
assert.ok(modelRequests.at(-1).messages.some(message => message.content.includes("Tell the user you searched")));
|
||||
}
|
||||
searchMode = "failed";
|
||||
const requestsBefore = modelRequests.length;
|
||||
const failure = await send("Search the internet for the average lifespan in 2026", true, false);
|
||||
assert.equal(failure.at(-1).event, "error");
|
||||
assert.equal(failure.at(-1).data.searchState, "failed");
|
||||
assert.equal(modelRequests.length, requestsBefore);
|
||||
});
|
||||
Reference in New Issue
Block a user