Fretux/KrisHD_AI
3
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

show reasoning in frontend and some other stuff

Browse Source
This commit is contained in:
fretuks
2026-09-21 20:24:40 +02:00
parent ceb3c752ff
commit fd76a79aaf
12 changed files with 220 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/services/modelService.js
+84 -28
View File
@@ -1,4 +1,4 @@
import {execFile} from "child_process";
import {supportsReasoning} from "../../public/app/modelCapabilities.js";
class ModelServiceError extends Error {
constructor(code, message, status = 500) {
@@ -41,9 +41,8 @@ export function createModelService(config, overrides = {}) {
if (state.timer) clearTimeout(state.timer);
state.timer = setTimeout(() => {
if (state.activeRequests > 0) return;
execFile("ollama", ["stop", model], () => {
modelState.delete(model);
});
// Ollama owns unloading through keep_alive; only clear local bookkeeping.
modelState.delete(model);
}, config.modelUnloadAfterMs);
state.timer.unref?.();
};
@@ -74,7 +73,37 @@ export function createModelService(config, overrides = {}) {
}
};
const readStreamingReply = async (response, onChunk = null, onThinking = null) => {
const withGenerationTimeout = async (callback, externalSignal) => {
const controller = new AbortController();
let timedOut = false;
let idleTimer;
const expire = () => { timedOut = true; controller.abort(); };
const touch = (ms = config.modelInactivityTimeoutMs ?? config.modelRequestTimeoutMs ?? 180000) => {
clearTimeout(idleTimer);
idleTimer = setTimeout(expire, ms);
};
const abort = () => controller.abort();
if (externalSignal?.aborted) abort();
else externalSignal?.addEventListener("abort", abort, {once: true});
touch(config.modelConnectTimeoutMs ?? 30000);
const maxTimer = setTimeout(expire, config.modelMaxGenerationMs ?? 600000);
try {
controller.signal.throwIfAborted();
return await callback(controller.signal, touch);
} catch (error) {
if (controller.signal.aborted) {
throw new ModelServiceError(timedOut ? "MODEL_TIMEOUT" : "MODEL_CANCELLED",
timedOut ? "Model backend timed out." : "Model request was cancelled.", timedOut ? 504 : 499);
}
throw error;
} finally {
clearTimeout(idleTimer);
clearTimeout(maxTimer);
externalSignal?.removeEventListener("abort", abort);
}
};
const readStreamingReply = async (response, onChunk = null, onThinking = null, touch = () => {}, signal = null) => {
if (!response.body) {
throw new ModelServiceError("MODEL_BAD_RESPONSE", "Model backend returned no body.", 502);
}
@@ -84,9 +113,18 @@ export function createModelService(config, overrides = {}) {
let buffer = "";
let fullReply = "";
let fullThinking = "";
let completed = false;
const consume = (line) => {
const data = JSON.parse(line);
if (data.message?.thinking) {
if (!data || typeof data !== "object" || data.error) {
throw new ModelServiceError("MODEL_BAD_RESPONSE", "Model backend returned an invalid stream.", 502);
}
for (const field of ["content", "thinking"]) {
if (data.message?.[field] != null && typeof data.message[field] !== "string") {
throw new ModelServiceError("MODEL_BAD_RESPONSE", "Model backend returned an invalid stream.", 502);
}
}
if (data.message?.thinking && onThinking) {
fullThinking += data.message.thinking;
onThinking?.(data.message.thinking, fullThinking);
}
@@ -94,29 +132,43 @@ export function createModelService(config, overrides = {}) {
fullReply += data.message.content;
onChunk?.(data.message.content, fullReply);
}
if (data.done === true) completed = true;
};
while (true) {
const {value, done} = await reader.read();
if (done) break;
buffer += decoder.decode(value, {stream: true});
const lines = buffer.split("\n");
buffer = lines.pop() || "";
for (const line of lines) {
const trimmed = line.trim();
if (!trimmed) continue;
consume(trimmed);
const cancel = () => { void reader.cancel().catch(() => {}); };
signal?.addEventListener("abort", cancel, {once: true});
try {
while (!completed) {
signal?.throwIfAborted();
const {value, done} = await reader.read();
signal?.throwIfAborted();
if (done) break;
touch();
buffer += decoder.decode(value, {stream: true});
const lines = buffer.split("\n");
buffer = lines.pop() || "";
for (const line of lines) {
const trimmed = line.trim();
if (!trimmed) continue;
consume(trimmed);
if (completed) break;
}
}
}
if (buffer.trim()) {
consume(buffer.trim());
buffer += decoder.decode();
if (!completed && buffer.trim()) {
consume(buffer.trim());
}
if (!completed) throw new ModelServiceError("MODEL_INTERRUPTED", "Model response was interrupted.", 502);
return fullReply.trim();
} finally {
signal?.removeEventListener("abort", cancel);
await reader.cancel().catch(() => {});
reader.releaseLock();
}
return fullReply.trim();
};
const buildRequestBody = (model, messagesPayload, generation = {}) => {
const buildRequestBody = (model, messagesPayload, generation = {}, reasoning = false) => {
const options = {};
if (generation.temperature != null) options.temperature = generation.temperature;
if (generation.contextLength != null) options.num_ctx = generation.contextLength;
@@ -127,6 +179,8 @@ export function createModelService(config, overrides = {}) {
// retries, roleplay openers, and summaries) receives runtime date context.
messages: [buildRuntimeDateContext(), ...messagesPayload],
stream: true,
think: reasoning === true && supportsReasoning(model),
keep_alive: config.ollamaKeepAlive ?? "30m",
...(Object.keys(options).length ? {options} : {})
};
};
@@ -138,11 +192,11 @@ export function createModelService(config, overrides = {}) {
try {
const startedAt = Date.now();
const fullReply = await withTimeout(config.modelRequestTimeoutMs, async (signal) => {
const fullReply = await withGenerationTimeout(async (signal, touch) => {
const response = await fetchImpl(`${config.modelApiBaseUrl}/chat`, {
method: "POST",
headers: {"Content-Type": "application/json"},
body: JSON.stringify(buildRequestBody(model, messagesPayload, options.generation)),
body: JSON.stringify(buildRequestBody(model, messagesPayload, options.generation, options.reasoning)),
signal
});
@@ -150,7 +204,8 @@ export function createModelService(config, overrides = {}) {
throw new ModelServiceError("MODEL_UPSTREAM_ERROR", `Model backend returned ${response.status}.`, 502);
}
return readStreamingReply(response, null, options.onThinking);
touch();
return readStreamingReply(response, null, options.reasoning === true && supportsReasoning(model) ? options.onThinking : null, touch, signal);
}, options.signal);
lastHealth = {ok: true, checkedAt: new Date().toISOString(), error: null, latencyMs: Date.now() - startedAt};
return fullReply;
@@ -176,11 +231,11 @@ export function createModelService(config, overrides = {}) {
try {
const startedAt = Date.now();
const fullReply = await withTimeout(config.modelRequestTimeoutMs, async (signal) => {
const fullReply = await withGenerationTimeout(async (signal, touch) => {
const response = await fetchImpl(`${config.modelApiBaseUrl}/chat`, {
method: "POST",
headers: {"Content-Type": "application/json"},
body: JSON.stringify(buildRequestBody(model, messagesPayload, options.generation)),
body: JSON.stringify(buildRequestBody(model, messagesPayload, options.generation, options.reasoning)),
signal
});
@@ -188,7 +243,8 @@ export function createModelService(config, overrides = {}) {
throw new ModelServiceError("MODEL_UPSTREAM_ERROR", `Model backend returned ${response.status}.`, 502);
}
return readStreamingReply(response, onChunk, options.onThinking);
touch();
return readStreamingReply(response, onChunk, options.reasoning === true && supportsReasoning(model) ? options.onThinking : null, touch, signal);
}, options.signal);
lastHealth = {ok: true, checkedAt: new Date().toISOString(), error: null, latencyMs: Date.now() - startedAt};
return fullReply;