Fretux/KrisHD_AI
3
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

show reasoning in frontend and some other stuff

Browse Source
This commit is contained in:
fretuks
2026-09-21 20:24:40 +02:00
parent ceb3c752ff
commit fd76a79aaf
12 changed files with 220 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/config.js
+4
View File
@@ -54,6 +54,10 @@ export function createConfig(overrides = {}) {
chatMaxRetries: int(process.env.CHAT_MAX_RETRIES, 5),
modelApiBaseUrl: process.env.MODEL_API_BASE_URL || "https://ai.krishd.ch/api",
modelRequestTimeoutMs: int(process.env.MODEL_REQUEST_TIMEOUT_MS, 180000),
modelConnectTimeoutMs: Math.max(1, int(process.env.MODEL_CONNECT_TIMEOUT_MS, 30000)),
modelInactivityTimeoutMs: Math.max(1, int(process.env.MODEL_INACTIVITY_TIMEOUT_MS, int(process.env.MODEL_REQUEST_TIMEOUT_MS, 180000))),
modelMaxGenerationMs: Math.max(1, int(process.env.MODEL_MAX_GENERATION_MS, 600000)),
ollamaKeepAlive: process.env.OLLAMA_KEEP_ALIVE || "30m",
searxngUrl: process.env.SEARXNG_URL || "",
webSearchTimeoutMs: Math.max(1, int(process.env.WEB_SEARCH_TIMEOUT_MS, 8000)),
modelListTimeoutMs: int(process.env.MODEL_LIST_TIMEOUT_MS, 4000),
src/routes/chats.js
+7 -3
View File
@@ -22,7 +22,8 @@ const sendMessageValidator = (body) => {
if (!chatId) return {error: "chatId is required"};
if (!message) return {error: "message is required"};
if (body.webSearch !== undefined && typeof body.webSearch !== "boolean") return {error: "webSearch must be a boolean"};
return {value: {message, chatId, model: String(body?.model || "qwen3:14b"), webSearch: body.webSearch === true}};
if (body.reasoning !== undefined && typeof body.reasoning !== "boolean") return {error: "reasoning must be a boolean"};
return {value: {message, chatId, model: String(body?.model || "qwen3:14b"), webSearch: body.webSearch === true, reasoning: body.reasoning === true}};
};
const organizationValidator = (body) => ({
@@ -236,7 +237,8 @@ export function createChatsRouter({repositories, chatService, modelService, conf
targetMessage,
content: req.validatedBody.message,
model: req.validatedBody.model,
webSearch: req.validatedBody.webSearch
webSearch: req.validatedBody.webSearch,
reasoning: req.validatedBody.reasoning
});
if (result.error) return res.status(result.status).json({error: result.error});
return res.json(result);
@@ -431,7 +433,8 @@ export function createChatsRouter({repositories, chatService, modelService, conf
res.writeHead(200, {
"Content-Type": "text/event-stream",
"Cache-Control": "no-cache, no-transform",
Connection: "keep-alive"
Connection: "keep-alive",
"X-Accel-Buffering": "no"
});
const sendEvent = (event, data) => {
@@ -449,6 +452,7 @@ export function createChatsRouter({repositories, chatService, modelService, conf
onChunk: (chunk, fullReply) => sendEvent("chunk", {chunk, fullReply}),
onThinking: (chunk, fullThinking) => sendEvent("thinking", {chunk, fullThinking}),
webSearch: req.validatedBody.webSearch,
reasoning: req.validatedBody.reasoning,
onStatus: (status) => sendEvent("status", status),
signal: generationController.signal
});
src/services/chatService.js
+25 -14
View File
@@ -19,6 +19,7 @@ import {
} from "./chatMessageHelpers.js";
import {SearXNGSearchProvider, WebSearchError, buildWebSearchContext, buildWebSearchQuery, decideWebSearch, selectSearchResults, MAX_CANDIDATES} from "./webSearchService.js";
import {supportsReasoning} from "../../public/app/modelCapabilities.js";
export function createChatService(repositories, modelService, config, webSearchProvider = new SearXNGSearchProvider(config)) {
const chatGenerationQueues = new Map();
@@ -192,21 +193,25 @@ export function createChatService(repositories, modelService, config, webSearchP
], {onThinking});
};
const generateValidatedRoleplayReply = async ({selectedModel, messagesPayload, userPersona, signal = null, generation = {}, onThinking = null}) => {
let reply = await modelService.generateReply(selectedModel, messagesPayload, {signal, generation, onThinking});
const generateValidatedRoleplayReply = async ({selectedModel, messagesPayload, userPersona, signal = null, generation = {}, onThinking = null, onChunk = null, reasoning = false}) => {
const generate = (messages) => onChunk
? modelService.streamReply(selectedModel, messages, onChunk, {signal, generation, onThinking, reasoning})
: modelService.generateReply(selectedModel, messages, {signal, generation, onThinking, reasoning});
let reply = await generate(messagesPayload);
if (reply && !containsUserVoiceInRoleplayOpener(reply, userPersona)) {
return reply;
}
onThinking?.("", "");
reply = await modelService.generateReply(selectedModel, [
onChunk?.("", "");
reply = await generate([
...messagesPayload,
{role: "assistant", content: reply || ""},
{
role: "user",
content: "Rewrite your last reply. Hard rule: only the assistant character may speak or act. Never write or imply user speech, thoughts, feelings, reactions, decisions, or actions. Stop before the user's reply."
}
], {signal, generation, onThinking});
]);
if (!reply || containsUserVoiceInRoleplayOpener(reply, userPersona)) {
return null;
@@ -320,7 +325,7 @@ export function createChatService(repositories, modelService, config, webSearchP
return {fullReply, thinking, promptMessageId: promptMessage.id};
};
const generateChatTurn = ({user, chatId, message, model, webSearch = false, onStatus = null, onChunk = null, onThinking = null, signal = null, parentMessageId = undefined}) =>
const generateChatTurn = ({user, chatId, message, model, webSearch = false, reasoning = false, onStatus = null, onChunk = null, onThinking = null, signal = null, parentMessageId = undefined}) =>
withChatGenerationLock(chatId, async () => {
await (summaryQueues.get(chatId) || Promise.resolve());
const session = repositories.getChat(chatId, user);
@@ -337,17 +342,22 @@ export function createChatService(repositories, modelService, config, webSearchP
return {error: "Branch point not found", status: 404};
}
const {model: effectiveModel, generation} = getGenerationSettings(session, model);
const useReasoning = reasoning === true && supportsReasoning(effectiveModel);
signal?.throwIfAborted();
const inserted = repositories.insertBranchChatMessage(chatId, "user", message, null, null, "pending", null, effectiveParentId);
const userMessageId = Number(inserted?.lastInsertRowid);
try {
let sources = null;
let thinking = "";
const captureThinking = (chunk, fullThinking) => {
if (!useReasoning) return;
thinking = fullThinking;
onThinking?.(chunk, fullThinking);
};
let payload;
const fullReply = await withActiveGeneration(chatId, signal, async (activeSignal) => {
activeSignal.throwIfAborted();
onStatus?.({status: "accepted"});
if (webSearch) {
const searchDecision = decideWebSearch(message);
let searchQuery = null;
@@ -364,7 +374,6 @@ export function createChatService(repositories, modelService, config, webSearchP
candidateCount = candidates.length;
sources = selectSearchResults(message, candidates, searchDecision);
selectedCount = sources.length;
onStatus?.({status: "generating", sourceCount: selectedCount});
}
if (config.structuredRequestLoggingEnabled) {
console.log(JSON.stringify({
@@ -379,6 +388,7 @@ export function createChatService(repositories, modelService, config, webSearchP
}));
}
}
onStatus?.({status: "generating", ...(sources !== null ? {sourceCount: sources.length} : {})});
payload = buildConversationContext({user, chatId, session, message: null, sources});
return payload.activePersona
? generateValidatedRoleplayReply({
@@ -387,18 +397,19 @@ export function createChatService(repositories, modelService, config, webSearchP
userPersona: payload.activeUserPersona,
signal: activeSignal,
generation,
onThinking: captureThinking
onThinking: captureThinking,
onChunk,
reasoning: useReasoning
})
: onChunk
? modelService.streamReply(effectiveModel, payload.messagesPayload, onChunk, {signal: activeSignal, generation, onThinking: captureThinking})
: modelService.generateReply(effectiveModel, payload.messagesPayload, {signal: activeSignal, generation, onThinking: captureThinking});
? modelService.streamReply(effectiveModel, payload.messagesPayload, onChunk, {signal: activeSignal, generation, onThinking: captureThinking, reasoning: useReasoning})
: modelService.generateReply(effectiveModel, payload.messagesPayload, {signal: activeSignal, generation, onThinking: captureThinking, reasoning: useReasoning});
});
if (!fullReply) {
repositories.failChatTurn(chatId, userMessageId, "Failed to generate a valid character reply", webSearch ? userMessageId : effectiveParentId);
return {error: "Failed to generate a valid character reply", status: 500};
}
if (payload.activePersona && onChunk) onChunk(fullReply, fullReply);
const retryState = {
retryVariants: [fullReply],
@@ -508,11 +519,11 @@ export function createChatService(repositories, modelService, config, webSearchP
containsUserVoiceInRoleplayOpener,
persistChatMessageRetryState,
buildMessagesPayload: buildConversationContext,
async sendChatMessage({user, chatId, message, model, webSearch = false}) {
return generateChatTurn({user, chatId, message, model, webSearch});
async sendChatMessage({user, chatId, message, model, webSearch = false, reasoning = false}) {
return generateChatTurn({user, chatId, message, model, webSearch, reasoning});
},
async streamChatMessage({user, chatId, message, model, webSearch = false, onStatus, onChunk, onThinking, signal = null}) {
return generateChatTurn({user, chatId, message, model, webSearch, onStatus, onChunk, onThinking, signal});
async streamChatMessage({user, chatId, message, model, webSearch = false, reasoning = false, onStatus, onChunk, onThinking, signal = null}) {
return generateChatTurn({user, chatId, message, model, webSearch, reasoning, onStatus, onChunk, onThinking, signal});
},
async retryMessage({user, chatId, selectedModel, targetMessage, retryStyle = null}) {
return withChatGenerationLock(chatId, async () => {
src/services/modelService.js
+84 -28
View File
@@ -1,4 +1,4 @@
import {execFile} from "child_process";
import {supportsReasoning} from "../../public/app/modelCapabilities.js";
class ModelServiceError extends Error {
constructor(code, message, status = 500) {
@@ -41,9 +41,8 @@ export function createModelService(config, overrides = {}) {
if (state.timer) clearTimeout(state.timer);
state.timer = setTimeout(() => {
if (state.activeRequests > 0) return;
execFile("ollama", ["stop", model], () => {
modelState.delete(model);
});
// Ollama owns unloading through keep_alive; only clear local bookkeeping.
modelState.delete(model);
}, config.modelUnloadAfterMs);
state.timer.unref?.();
};
@@ -74,7 +73,37 @@ export function createModelService(config, overrides = {}) {
}
};
const readStreamingReply = async (response, onChunk = null, onThinking = null) => {
const withGenerationTimeout = async (callback, externalSignal) => {
const controller = new AbortController();
let timedOut = false;
let idleTimer;
const expire = () => { timedOut = true; controller.abort(); };
const touch = (ms = config.modelInactivityTimeoutMs ?? config.modelRequestTimeoutMs ?? 180000) => {
clearTimeout(idleTimer);
idleTimer = setTimeout(expire, ms);
};
const abort = () => controller.abort();
if (externalSignal?.aborted) abort();
else externalSignal?.addEventListener("abort", abort, {once: true});
touch(config.modelConnectTimeoutMs ?? 30000);
const maxTimer = setTimeout(expire, config.modelMaxGenerationMs ?? 600000);
try {
controller.signal.throwIfAborted();
return await callback(controller.signal, touch);
} catch (error) {
if (controller.signal.aborted) {
throw new ModelServiceError(timedOut ? "MODEL_TIMEOUT" : "MODEL_CANCELLED",
timedOut ? "Model backend timed out." : "Model request was cancelled.", timedOut ? 504 : 499);
}
throw error;
} finally {
clearTimeout(idleTimer);
clearTimeout(maxTimer);
externalSignal?.removeEventListener("abort", abort);
}
};
const readStreamingReply = async (response, onChunk = null, onThinking = null, touch = () => {}, signal = null) => {
if (!response.body) {
throw new ModelServiceError("MODEL_BAD_RESPONSE", "Model backend returned no body.", 502);
}
@@ -84,9 +113,18 @@ export function createModelService(config, overrides = {}) {
let buffer = "";
let fullReply = "";
let fullThinking = "";
let completed = false;
const consume = (line) => {
const data = JSON.parse(line);
if (data.message?.thinking) {
if (!data || typeof data !== "object" || data.error) {
throw new ModelServiceError("MODEL_BAD_RESPONSE", "Model backend returned an invalid stream.", 502);
}
for (const field of ["content", "thinking"]) {
if (data.message?.[field] != null && typeof data.message[field] !== "string") {
throw new ModelServiceError("MODEL_BAD_RESPONSE", "Model backend returned an invalid stream.", 502);
}
}
if (data.message?.thinking && onThinking) {
fullThinking += data.message.thinking;
onThinking?.(data.message.thinking, fullThinking);
}
@@ -94,29 +132,43 @@ export function createModelService(config, overrides = {}) {
fullReply += data.message.content;
onChunk?.(data.message.content, fullReply);
}
if (data.done === true) completed = true;
};
while (true) {
const {value, done} = await reader.read();
if (done) break;
buffer += decoder.decode(value, {stream: true});
const lines = buffer.split("\n");
buffer = lines.pop() || "";
for (const line of lines) {
const trimmed = line.trim();
if (!trimmed) continue;
consume(trimmed);
const cancel = () => { void reader.cancel().catch(() => {}); };
signal?.addEventListener("abort", cancel, {once: true});
try {
while (!completed) {
signal?.throwIfAborted();
const {value, done} = await reader.read();
signal?.throwIfAborted();
if (done) break;
touch();
buffer += decoder.decode(value, {stream: true});
const lines = buffer.split("\n");
buffer = lines.pop() || "";
for (const line of lines) {
const trimmed = line.trim();
if (!trimmed) continue;
consume(trimmed);
if (completed) break;
}
}
}
if (buffer.trim()) {
consume(buffer.trim());
buffer += decoder.decode();
if (!completed && buffer.trim()) {
consume(buffer.trim());
}
if (!completed) throw new ModelServiceError("MODEL_INTERRUPTED", "Model response was interrupted.", 502);
return fullReply.trim();
} finally {
signal?.removeEventListener("abort", cancel);
await reader.cancel().catch(() => {});
reader.releaseLock();
}
return fullReply.trim();
};
const buildRequestBody = (model, messagesPayload, generation = {}) => {
const buildRequestBody = (model, messagesPayload, generation = {}, reasoning = false) => {
const options = {};
if (generation.temperature != null) options.temperature = generation.temperature;
if (generation.contextLength != null) options.num_ctx = generation.contextLength;
@@ -127,6 +179,8 @@ export function createModelService(config, overrides = {}) {
// retries, roleplay openers, and summaries) receives runtime date context.
messages: [buildRuntimeDateContext(), ...messagesPayload],
stream: true,
think: reasoning === true && supportsReasoning(model),
keep_alive: config.ollamaKeepAlive ?? "30m",
...(Object.keys(options).length ? {options} : {})
};
};
@@ -138,11 +192,11 @@ export function createModelService(config, overrides = {}) {
try {
const startedAt = Date.now();
const fullReply = await withTimeout(config.modelRequestTimeoutMs, async (signal) => {
const fullReply = await withGenerationTimeout(async (signal, touch) => {
const response = await fetchImpl(`${config.modelApiBaseUrl}/chat`, {
method: "POST",
headers: {"Content-Type": "application/json"},
body: JSON.stringify(buildRequestBody(model, messagesPayload, options.generation)),
body: JSON.stringify(buildRequestBody(model, messagesPayload, options.generation, options.reasoning)),
signal
});
@@ -150,7 +204,8 @@ export function createModelService(config, overrides = {}) {
throw new ModelServiceError("MODEL_UPSTREAM_ERROR", `Model backend returned ${response.status}.`, 502);
}
return readStreamingReply(response, null, options.onThinking);
touch();
return readStreamingReply(response, null, options.reasoning === true && supportsReasoning(model) ? options.onThinking : null, touch, signal);
}, options.signal);
lastHealth = {ok: true, checkedAt: new Date().toISOString(), error: null, latencyMs: Date.now() - startedAt};
return fullReply;
@@ -176,11 +231,11 @@ export function createModelService(config, overrides = {}) {
try {
const startedAt = Date.now();
const fullReply = await withTimeout(config.modelRequestTimeoutMs, async (signal) => {
const fullReply = await withGenerationTimeout(async (signal, touch) => {
const response = await fetchImpl(`${config.modelApiBaseUrl}/chat`, {
method: "POST",
headers: {"Content-Type": "application/json"},
body: JSON.stringify(buildRequestBody(model, messagesPayload, options.generation)),
body: JSON.stringify(buildRequestBody(model, messagesPayload, options.generation, options.reasoning)),
signal
});
@@ -188,7 +243,8 @@ export function createModelService(config, overrides = {}) {
throw new ModelServiceError("MODEL_UPSTREAM_ERROR", `Model backend returned ${response.status}.`, 502);
}
return readStreamingReply(response, onChunk, options.onThinking);
touch();
return readStreamingReply(response, onChunk, options.reasoning === true && supportsReasoning(model) ? options.onThinking : null, touch, signal);
}, options.signal);
lastHealth = {ok: true, checkedAt: new Date().toISOString(), error: null, latencyMs: Date.now() - startedAt};
return fullReply;