Fretux/KrisHD_AI
3
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

add vision analysis service with prompt versioning, response schema, and structured JSON output

Browse Source
This commit is contained in:
fretuks
2026-09-23 20:34:33 +02:00
parent 85e6d0c957
commit 80a30e2ce0
7 changed files with 142 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/services/chatService.js
+89 -14
View File
@@ -17,11 +17,13 @@ import {
parseSources,
retryStylePrompts
} from "./chatMessageHelpers.js";
import {createHash} from "node:crypto";
import { SearXNGSearchProvider, WebSearchError, buildWebSearchContext, retrieveWebSearch } from "./webSearchService.js";
import { SearXNGSearchProvider, WebSearchError, buildWebSearchContext, decideWebSearch, retrieveWebSearch } from "./webSearchService.js";
import { supportsReasoning } from "../../public/js/modelCapabilities.js";
import {VISION_EVIDENCE_GUIDANCE} from "./visionAnalysis.js";
export function createChatService(repositories, modelService, config, webSearchProvider = new SearXNGSearchProvider(config), attachmentService = null) {
export function createChatService(repositories, modelService, config, webSearchProvider = new SearXNGSearchProvider(config), attachmentService = null, visionDebugService = null) {
const formatChatMessage = row => {
const message = formatBaseChatMessage(row);
if (message?.role === "bot") {
@@ -42,6 +44,36 @@ export function createChatService(repositories, modelService, config, webSearchP
const regeneratingMessages = new Map();
const maxRetries = Number(config.chatMaxRetries ?? 5);
const now = typeof config.now === "function" ? config.now : () => new Date();
const visionGuidanceSha256 = createHash("sha256").update(VISION_EVIDENCE_GUIDANCE).digest("hex");
const beginVisionDiagnostics = ({requestId, user, chatId, triggeringMessageId, userQuestion, selectedModel, reasoningRequested, webSearchRequested, visionSelection}) => {
if (!visionDebugService?.enabled || !visionSelection || !attachmentService.hasImages(visionSelection)) return;
visionDebugService.begin({requestId, user, chatId, triggeringMessageId, userQuestion, selectedModel, reasoningRequested, webSearchRequested});
const decision = decideWebSearch(userQuestion, webSearchRequested);
visionDebugService.recordSearch(requestId, {
requested: webSearchRequested,
needed: decision.needsSearch,
explicitSearchIntent: decision.explicitSearchIntent,
state: webSearchRequested ? (decision.shouldSearch ? "pending" : "skipped") : "disabled"
});
};
const recordQwenDiagnostics = ({requestId, messagesPayload, visionSelection, model, generation, reasoning}) => {
if (!visionDebugService?.enabled || !requestId) return;
const evidenceMessage = messagesPayload.find(message => message.role === "user" && message.content.startsWith("<image_evidence>"));
const metadata = typeof modelService.getRequestMetadata === "function"
? modelService.getRequestMetadata(model, generation, reasoning)
: {model, think: reasoning === true && supportsReasoning(model), stream: true,
temperature: generation.temperature ?? null, numCtx: generation.contextLength ?? null,
numPredict: generation.responseLength ?? null, keepAlive: config.ollamaKeepAlive ?? "30m"};
visionDebugService.recordQwen(requestId, {
imageEvidence: evidenceMessage?.content || null,
visionGuidanceSha256,
visionEvidenceIncluded: Boolean(evidenceMessage),
imageCount: (visionSelection?.currentMessageImages.length || 0) + (visionSelection?.historicalImages.length || 0),
currentImageLabels: visionSelection?.currentMessageImages.map(image => image.label) || [],
...metadata
});
};
const insertChatMessage = (chatId, role, content, retryState = null, modelName = null, deliveryStatus = "complete", errorMessage = null) => {
const normalizedContent = String(content || "");
@@ -342,19 +374,35 @@ export function createChatService(repositories, modelService, config, webSearchP
const imageHistory = repositories.getRecentChatMessagesUpToId(chatId, promptMessage.id).reverse();
const visionSelection = attachmentService?.selectImages(imageHistory, promptMessage.content);
beginVisionDiagnostics({requestId, user, chatId, triggeringMessageId: promptMessage.id,
userQuestion: promptMessage.content, selectedModel, reasoningRequested: reasoning,
webSearchRequested: webSearch, visionSelection});
if (visionSelection && attachmentService.hasImages(visionSelection)) onStatus?.({ status: "analyzing" });
if (visionSelection) await attachmentService.analyzeContext(visionSelection, promptMessage.content, signal,
{requestId, chatId, triggeringMessageId: promptMessage.id});
let sources = null;
let searchState = "skipped";
if (webSearch) {
const search = await retrieveWebSearch(promptMessage.content, webSearchProvider, {
signal,
referenceText: visionSelection ? attachmentService.searchContext(visionSelection, promptMessage.content) : "",
onSearching: () => onStatus?.({ status: "searching" })
});
sources = search.sources;
searchState = search.state;
const visualReference = visionSelection ? attachmentService.searchContext(visionSelection, promptMessage.content) : "";
try {
const search = await retrieveWebSearch(promptMessage.content, webSearchProvider, {
signal, referenceText: visualReference,
onSearching: () => {
visionDebugService?.recordSearch(requestId, {requested: true, state: "searching", visualReference});
onStatus?.({ status: "searching" });
}
});
sources = search.sources;
searchState = search.state;
visionDebugService?.recordSearch(requestId, {requested: true, needed: search.decision.needsSearch,
explicitSearchIntent: search.decision.explicitSearchIntent, state: search.state, visualReference,
queries: search.queries, candidateCount: search.candidates.length,
selectedCount: search.sources?.length || 0, selectedSources: search.sources || []});
} catch (error) {
visionDebugService?.recordSearch(requestId, {requested: true, state: "failed", visualReference});
visionDebugService?.fail(requestId, error);
throw error;
}
}
onStatus?.({ status: "generating", searchState, sources });
const { messagesPayload } = buildConversationContext({
@@ -369,6 +417,8 @@ export function createChatService(repositories, modelService, config, webSearchP
});
const styledPayload = appendRetryStyleInstruction(messagesPayload, retryStyle);
recordQwenDiagnostics({requestId, messagesPayload: styledPayload, visionSelection,
model: selectedModel, generation, reasoning});
const fullReply = activePersona
? await generateValidatedRoleplayReply({ selectedModel, messagesPayload: styledPayload, userPersona: activeUserPersona, signal, generation, onThinking: captureThinking, onChunk, reasoning })
: onChunk
@@ -376,6 +426,7 @@ export function createChatService(repositories, modelService, config, webSearchP
: await modelService.generateReply(selectedModel, styledPayload, { signal, generation, onThinking: captureThinking, reasoning });
if (!fullReply) return { error: "Failed to regenerate message" };
visionDebugService?.complete(requestId, {finalAnswer: fullReply, visibleReasoning: reasoning ? thinking : null});
return { fullReply, thinking, sources, searchState, promptMessageId: promptMessage.id };
};
@@ -416,22 +467,41 @@ export function createChatService(repositories, modelService, config, webSearchP
onStatus?.({ status: "accepted" });
const imageHistory = repositories.getRecentChatMessagesUpToId(chatId, userMessageId).reverse();
const visionSelection = attachmentService?.selectImages(imageHistory, message);
beginVisionDiagnostics({requestId, user, chatId, triggeringMessageId: userMessageId,
userQuestion: message, selectedModel: effectiveModel, reasoningRequested: useReasoning,
webSearchRequested: webSearch, visionSelection});
if (visionSelection && attachmentService.hasImages(visionSelection)) {
onStatus?.({ status: "analyzing" });
await attachmentService.analyzeContext(visionSelection, message, activeSignal,
{requestId, chatId, triggeringMessageId: userMessageId});
}
if (webSearch) {
const search = await retrieveWebSearch(message, webSearchProvider, {
referenceText: visionSelection ? attachmentService.searchContext(visionSelection, message) : "",
signal: activeSignal,
onSearching: () => onStatus?.({ status: "searching" })
});
const visualReference = visionSelection ? attachmentService.searchContext(visionSelection, message) : "";
let search;
try {
search = await retrieveWebSearch(message, webSearchProvider, {
referenceText: visualReference,
signal: activeSignal,
onSearching: () => {
visionDebugService?.recordSearch(requestId, {requested: true, state: "searching", visualReference});
onStatus?.({ status: "searching" });
}
});
} catch (error) {
visionDebugService?.recordSearch(requestId, {requested: true, state: "failed", visualReference});
visionDebugService?.fail(requestId, error);
throw error;
}
sources = search.sources;
searchState = search.state;
visionDebugService?.recordSearch(requestId, {requested: true, needed: search.decision.needsSearch,
explicitSearchIntent: search.decision.explicitSearchIntent, state: search.state, visualReference,
queries: search.queries, candidateCount: search.candidates.length,
selectedCount: search.sources?.length || 0, selectedSources: search.sources || []});
if (config.structuredRequestLoggingEnabled) {
console.log(JSON.stringify({
event: "web_search",
requestId,
webSearchRequested: true,
explicitSearchRequested: search.decision.explicitSearchIntent,
automaticSearchDecision: search.decision.automaticSearchDecision,
@@ -448,6 +518,8 @@ export function createChatService(repositories, modelService, config, webSearchP
onStatus?.({ status: "generating", searchState, ...(sources !== null ? { sourceCount: sources.length } : {}) });
payload = buildConversationContext({ user, chatId, session, message: null, upToMessageId: userMessageId,
historyOverride: imageHistory, visionSelection, contextSummaryOverride: session.context_summary, sources });
recordQwenDiagnostics({requestId, messagesPayload: payload.messagesPayload, visionSelection,
model: effectiveModel, generation, reasoning: useReasoning});
return payload.activePersona
? generateValidatedRoleplayReply({
selectedModel: effectiveModel,
@@ -469,6 +541,8 @@ export function createChatService(repositories, modelService, config, webSearchP
return { error: "Failed to generate a valid character reply", status: 500 };
}
visionDebugService?.complete(requestId, {finalAnswer: fullReply, visibleReasoning: useReasoning ? thinking : null});
const retryState = {
retryVariants: [fullReply],
retryActiveIndex: 0,
@@ -480,6 +554,7 @@ export function createChatService(repositories, modelService, config, webSearchP
scheduleContextSummaryUpdate({ user, chatId, selectedModel: effectiveModel });
return { reply: fullReply, userMessageId, searchState, ...(sources ? { sources } : {}) };
} catch (error) {
visionDebugService?.fail(requestId, error);
repositories.failChatTurn(chatId, userMessageId, String(error?.message || "Model request failed").slice(0, 500), webSearch ? userMessageId : effectiveParentId);
repositories.touchChat(chatId, user);
if (error instanceof WebSearchError) return { error: error.message, status: error.status, searchState: "failed" };