adjust attachments to hopefully fix them?
This commit is contained in:
Frederik Spirgi
@@ -1,5 +1,5 @@
|
||||
import sharp from "sharp";
|
||||
import {randomUUID} from "node:crypto";
|
||||
import {createHash, randomUUID} from "node:crypto";
|
||||
import {attachmentMetadata} from "../db/attachments.js";
|
||||
import {LocalFileStorage} from "./localFileStorage.js";
|
||||
import {VISION_PROMPT_VERSION, VISION_RESPONSE_SCHEMA, VISION_ANALYSIS_PROMPT, VISION_EVIDENCE_GUIDANCE, normalizeVisionQuestion, normalizeVisionAnalysis} from "./visionAnalysis.js";
|
||||
@@ -27,14 +27,30 @@ export function createAttachmentService(db, config, modelService) {
|
||||
const storage = new LocalFileStorage(config.uploadDir);
|
||||
const log = (event, fields) => { if (config.structuredRequestLoggingEnabled) console.log(JSON.stringify({event, ...fields})); };
|
||||
const get = id => db.prepare("SELECT * FROM attachments WHERE id = ?").get(id);
|
||||
const rows = messageId => db.prepare("SELECT a.* FROM attachments a JOIN message_attachments m ON m.attachment_id=a.id WHERE m.message_id=? ORDER BY m.position").all(messageId);
|
||||
const imageRows = messages => {
|
||||
const selected = [];
|
||||
for (const message of [...messages].reverse()) for (const row of rows(message.id)) {
|
||||
if (selected.length < config.uploadMaxFiles && !selected.some(item => item.id === row.id)) selected.push({...row, messageId: message.id});
|
||||
}
|
||||
return selected;
|
||||
const rows = messageId => db.prepare("SELECT a.*, m.position FROM attachments a JOIN message_attachments m ON m.attachment_id=a.id WHERE m.message_id=? ORDER BY m.position").all(messageId);
|
||||
const refersToEarlierImage = question => {
|
||||
const text = String(question || "");
|
||||
return /\b(?:previous|prior|earlier|first|last)\s+(?:(?:attached|uploaded)\s+)?(?:image|photo|picture|screenshot)\b|\b(?:image|photo|picture|screenshot)\s+(?:from\s+)?(?:before|earlier|previously)\b/i.test(text)
|
||||
|| (/\b(?:image|photo|picture|screenshot)\b/i.test(text) && /\b(?:previous|prior|earlier|first|last)\s+one\b/i.test(text));
|
||||
};
|
||||
const selectImages = (messages, question) => {
|
||||
const currentMessage = [...messages].reverse().find(message => message.role === "user");
|
||||
const currentMessageImages = currentMessage?.id != null ? rows(currentMessage.id).map((row, index) => ({
|
||||
...row, messageId: currentMessage.id, scope: "current", label: `Current image ${index + 1}`
|
||||
})) : [];
|
||||
const historicalImages = [];
|
||||
if (refersToEarlierImage(question) && (!currentMessageImages.length || /\b(?:previous|prior|earlier)\b/i.test(String(question || "")))) {
|
||||
const earlier = messages.filter(message => message.id !== currentMessage?.id);
|
||||
const candidates = earlier.flatMap(message => rows(message.id).map(row => ({...row, messageId: message.id})));
|
||||
const distinctCandidates = candidates.filter(row => !currentMessageImages.some(image => image.id === row.id));
|
||||
const row = /\bfirst\b/i.test(question) ? distinctCandidates[0] : distinctCandidates.at(-1);
|
||||
if (row) {
|
||||
historicalImages.push({...row, scope: "historical", label: `Earlier image from message ${row.messageId} (image ${row.position + 1})`});
|
||||
}
|
||||
}
|
||||
return {currentMessageImages, historicalImages};
|
||||
};
|
||||
const selectedRows = selection => [...selection.currentMessageImages, ...selection.historicalImages];
|
||||
const cached = (row, question) => {
|
||||
const entry = db.prepare("SELECT analysis FROM attachment_vision_analyses WHERE attachment_id=? AND vision_model=? AND prompt_version=? AND question=?")
|
||||
.get(row.id, config.visionModel, VISION_PROMPT_VERSION, normalizeVisionQuestion(question));
|
||||
@@ -42,18 +58,28 @@ export function createAttachmentService(db, config, modelService) {
|
||||
try { const analysis = JSON.parse(entry.analysis); return typeof analysis?.summary === "string" ? analysis : null; }
|
||||
catch { return null; }
|
||||
};
|
||||
const analyzeRows = async (images, question, signal) => {
|
||||
const analyzeRows = async (images, question, signal, diagnostics = {}) => {
|
||||
for (const row of images) {
|
||||
signal?.throwIfAborted();
|
||||
if (cached(row, question)) { log("visionAnalysisCacheHit", {attachmentId: row.id}); continue; }
|
||||
const fields = {requestId: diagnostics.requestId ?? null, chatId: diagnostics.chatId ?? null,
|
||||
triggeringMessageId: diagnostics.triggeringMessageId ?? row.messageId, attachmentId: row.id,
|
||||
sourceMessageId: row.messageId, attachmentPosition: row.position, scope: row.scope || "current",
|
||||
storageKey: row.storage_key, originalFilename: row.original_name, visionModel: config.visionModel,
|
||||
visionPromptVersion: VISION_PROMPT_VERSION, question: normalizeVisionQuestion(question).slice(0, 500)};
|
||||
const hit = cached(row, question);
|
||||
if (hit) { log("visionAnalysisCacheHit", {...fields, cacheHit: true, resultSummary: hit.summary.slice(0, 160)}); continue; }
|
||||
try {
|
||||
log("visionAnalysisStart", {attachmentId: row.id, model: config.visionModel, promptVersion: VISION_PROMPT_VERSION});
|
||||
log("visionAnalysisStart", {...fields, cacheHit: false});
|
||||
const buffer = await storage.read(row.storage_key);
|
||||
if (config.visionDebugSha256Enabled && config.structuredRequestLoggingEnabled) {
|
||||
log("visionImageRead", {...fields, sha256: createHash("sha256").update(buffer).digest("hex")});
|
||||
}
|
||||
const result = await modelService.generateReply(config.visionModel, [
|
||||
{role: "system", content: VISION_ANALYSIS_PROMPT},
|
||||
{role: "user", content: `User question: ${JSON.stringify(String(question ?? ""))}\nAnalyze this image specifically to help answer that question.`, images: [buffer.toString("base64")]}
|
||||
], {signal, reasoning: false, format: VISION_RESPONSE_SCHEMA, generation: {temperature: 0.1, responseLength: 1600}});
|
||||
const analysis = JSON.stringify(parseVisionAnalysis(result));
|
||||
const parsed = parseVisionAnalysis(result);
|
||||
const analysis = JSON.stringify(parsed);
|
||||
signal?.throwIfAborted();
|
||||
db.transaction(() => {
|
||||
db.prepare(`INSERT INTO attachment_vision_analyses(attachment_id,vision_model,prompt_version,question,analysis)
|
||||
@@ -65,7 +91,7 @@ export function createAttachmentService(db, config, modelService) {
|
||||
db.prepare("UPDATE attachments SET vision_model=?,vision_analysis=?,vision_analyzed_at=CURRENT_TIMESTAMP WHERE id=?")
|
||||
.run(config.visionModel, analysis, row.id);
|
||||
})();
|
||||
log("visionAnalysisComplete", {attachmentId: row.id});
|
||||
log("visionAnalysisComplete", {...fields, cacheHit: false, resultSummary: parsed.summary.slice(0, 160)});
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
throw new AttachmentError("Image analysis failed. Check that the vision model is available, then retry.", 502);
|
||||
@@ -147,24 +173,29 @@ export function createAttachmentService(db, config, modelService) {
|
||||
})();
|
||||
},
|
||||
copy(sourceId, targetId) { db.prepare("INSERT INTO message_attachments SELECT ?,attachment_id,position FROM message_attachments WHERE message_id=?").run(targetId, sourceId); },
|
||||
searchContext(messages, question) {
|
||||
return imageRows(messages).map(row => {
|
||||
selectImages,
|
||||
searchContext(selection, question) {
|
||||
const searchImages = selection.currentMessageImages.length ? selection.currentMessageImages : selection.historicalImages;
|
||||
return searchImages.map(row => {
|
||||
const analysis = cached(row, question);
|
||||
if (!analysis) return "";
|
||||
return [analysis.answer, ...analysis.visibleText, analysis.summary].filter(Boolean).join(" ").slice(0, 300);
|
||||
}).join(" ").slice(0, 400);
|
||||
},
|
||||
hasImages: messages => imageRows(messages).length > 0,
|
||||
analyze: (messageId, question, signal) => analyzeRows(rows(messageId), question, signal),
|
||||
analyzeContext: (messages, question, signal) => analyzeRows(imageRows(messages), question, signal),
|
||||
context(messages, question) {
|
||||
const evidence = imageRows(messages).flatMap(row => {
|
||||
hasImages: selection => selectedRows(selection).length > 0,
|
||||
analyze: (messageId, question, signal) => analyzeRows(rows(messageId).map((row, index) => ({...row, messageId, scope: "current", label: `Current image ${index + 1}`})), question, signal),
|
||||
analyzeContext: (selection, question, signal, diagnostics) => analyzeRows(selectedRows(selection), question, signal, diagnostics),
|
||||
context(selection, question) {
|
||||
const evidence = selectedRows(selection).flatMap(row => {
|
||||
const analysis = cached(row, question);
|
||||
return analysis ? [{id: row.id, messageId: row.messageId, analysis}] : [];
|
||||
return analysis ? [{id: row.id, messageId: row.messageId, position: row.position, scope: row.scope, label: row.label, analysis}] : [];
|
||||
});
|
||||
if (!evidence.length) return null;
|
||||
const currentLabels = evidence.filter(image => image.scope === "current").map(image => image.label).join(", ");
|
||||
return [
|
||||
{role: "system", content: VISION_EVIDENCE_GUIDANCE},
|
||||
{role: "system", content: VISION_EVIDENCE_GUIDANCE + (currentLabels
|
||||
? `\nThe following images are attached to the user's current message: ${currentLabels}.`
|
||||
: "\nThe following evidence refers to earlier messages; the current user message has no image attachment.")},
|
||||
{role: "user", content: "<image_evidence>\n" + JSON.stringify({question: normalizeVisionQuestion(question), images: evidence}).replaceAll("<", "\\u003c").replaceAll(">", "\\u003e") + "\n</image_evidence>"}
|
||||
];
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user