import { CACHE_URL_PREFIX, UPLOAD_URL, REQUEST_TIMEOUT_MS, } from "@/app/constant"; import { MultimodalContent, RequestMessage } from "@/app/client/api"; import Locale from "@/app/locales"; import { EventStreamContentType, fetchEventSource, } from "@fortaine/fetch-event-source"; import { prettyObject } from "./format"; import { fetch as tauriFetch } from "./stream"; export function compressImage(file: Blob, maxSize: number): Promise<string> { return new Promise((resolve, reject) => { const reader = new FileReader(); reader.onload = (readerEvent: any) => { const image = new Image(); image.onload = () => { let canvas = document.createElement("canvas"); let ctx = canvas.getContext("2d"); let width = image.width; let height = image.height; let quality = 0.9; let dataUrl; do { canvas.width = width; canvas.height = height; ctx?.clearRect(0, 0, canvas.width, canvas.height); ctx?.drawImage(image, 0, 0, width, height); dataUrl = canvas.toDataURL("image/jpeg", quality); if (dataUrl.length < maxSize) break; if (quality > 0.5) { // Prioritize quality reduction quality -= 0.1; } else { // Then reduce the size width *= 0.9; height *= 0.9; } } while (dataUrl.length > maxSize); resolve(dataUrl); }; image.onerror = reject; image.src = readerEvent.target.result; }; reader.onerror = reject; if (file.type.includes("heic")) { try { const heic2any = require("heic2any"); heic2any({ blob: file, toType: "image/jpeg" }) .then((blob: Blob) => { reader.readAsDataURL(blob); }) .catch((e: any) => { reject(e); }); } catch (e) { reject(e); } } reader.readAsDataURL(file); }); } export async function preProcessImageContentBase( content: RequestMessage["content"], transformImageUrl: (url: string) => Promise<{ [key: string]: any }>, ) { if (typeof content === "string") { return content; } const result = []; for (const part of content) { if (part?.type == "image_url" && part?.image_url?.url) { try { const url = await cacheImageToBase64Image(part?.image_url?.url); result.push(await transformImageUrl(url)); } catch (error) { console.error("Error processing image URL:", error); } } else { result.push({ ...part }); } } return result; } export async function preProcessImageContent( content: RequestMessage["content"], ) { return preProcessImageContentBase(content, async (url) => ({ type: "image_url", image_url: { url }, })) as Promise<MultimodalContent[] | string>; } export async function preProcessImageContentForAlibabaDashScope( content: RequestMessage["content"], ) { return preProcessImageContentBase(content, async (url) => ({ image: url, })); } const imageCaches: Record<string, string> = {}; export function cacheImageToBase64Image(imageUrl: string) { if (imageUrl.includes(CACHE_URL_PREFIX)) { if (!imageCaches[imageUrl]) { const reader = new FileReader(); return fetch(imageUrl, { method: "GET", mode: "cors", credentials: "include", }) .then((res) => res.blob()) .then( async (blob) => (imageCaches[imageUrl] = await compressImage(blob, 256 * 1024)), ); // compressImage } return Promise.resolve(imageCaches[imageUrl]); } return Promise.resolve(imageUrl); } export function base64Image2Blob(base64Data: string, contentType: string) { const byteCharacters = atob(base64Data); const byteNumbers = new Array(byteCharacters.length); for (let i = 0; i < byteCharacters.length; i++) { byteNumbers[i] = byteCharacters.charCodeAt(i); } const byteArray = new Uint8Array(byteNumbers); return new Blob([byteArray], { type: contentType }); } export function uploadImage(file: Blob): Promise<string> { if (!window._SW_ENABLED) { // if serviceWorker register error, using compressImage return compressImage(file, 256 * 1024); } const body = new FormData(); body.append("file", file); return fetch(UPLOAD_URL, { method: "post", body, mode: "cors", credentials: "include", }) .then((res) => res.json()) .then((res) => { // console.log("res", res); if (res?.code == 0 && res?.data) { return res?.data; } throw Error(`upload Error: ${res?.msg}`); }); } export function removeImage(imageUrl: string) { return fetch(imageUrl, { method: "DELETE", mode: "cors", credentials: "include", }); } export function stream( chatPath: string, requestPayload: any, headers: any, tools: any[], funcs: Record<string, Function>, controller: AbortController, parseSSE: (text: string, runTools: any[]) => string | undefined, processToolMessage: ( requestPayload: any, toolCallMessage: any, toolCallResult: any[], ) => void, options: any, ) { let responseText = ""; let remainText = ""; let finished = false; let running = false; let runTools: any[] = []; let responseRes: Response; // animate response to make it looks smooth function animateResponseText() { if (finished || controller.signal.aborted) { responseText += remainText; console.log("[Response Animation] finished"); if (responseText?.length === 0) { options.onError?.(new Error("empty response from server")); } return; } if (remainText.length > 0) { const fetchCount = Math.max(1, Math.round(remainText.length / 60)); const fetchText = remainText.slice(0, fetchCount); responseText += fetchText; remainText = remainText.slice(fetchCount); options.onUpdate?.(responseText, fetchText); } requestAnimationFrame(animateResponseText); } // start animaion animateResponseText(); const finish = () => { if (!finished) { if (!running && runTools.length > 0) { const toolCallMessage = { role: "assistant", tool_calls: [...runTools], }; running = true; runTools.splice(0, runTools.length); // empty runTools return Promise.all( toolCallMessage.tool_calls.map((tool) => { options?.onBeforeTool?.(tool); return Promise.resolve( // @ts-ignore funcs[tool.function.name]( // @ts-ignore tool?.function?.arguments ? JSON.parse(tool?.function?.arguments) : {}, ), ) .then((res) => { let content = res.data || res?.statusText; // hotfix #5614 content = typeof content === "string" ? content : JSON.stringify(content); if (res.status >= 300) { return Promise.reject(content); } return content; }) .then((content) => { options?.onAfterTool?.({ ...tool, content, isError: false, }); return content; }) .catch((e) => { options?.onAfterTool?.({ ...tool, isError: true, errorMsg: e.toString(), }); return e.toString(); }) .then((content) => ({ name: tool.function.name, role: "tool", content, tool_call_id: tool.id, })); }), ).then((toolCallResult) => { processToolMessage(requestPayload, toolCallMessage, toolCallResult); setTimeout(() => { // call again console.debug("[ChatAPI] restart"); running = false; chatApi(chatPath, headers, requestPayload, tools); // call fetchEventSource }, 60); }); return; } if (running) { return; } console.debug("[ChatAPI] end"); finished = true; options.onFinish(responseText + remainText, responseRes); // 将res传递给onFinish } }; controller.signal.onabort = finish; function chatApi( chatPath: string, headers: any, requestPayload: any, tools: any, ) { const chatPayload = { method: "POST", body: JSON.stringify({ ...requestPayload, tools: tools && tools.length ? tools : undefined, }), signal: controller.signal, headers, }; const requestTimeoutId = setTimeout( () => controller.abort(), REQUEST_TIMEOUT_MS, ); fetchEventSource(chatPath, { fetch: tauriFetch as any, ...chatPayload, async onopen(res) { clearTimeout(requestTimeoutId); const contentType = res.headers.get("content-type"); console.log("[Request] response content type: ", contentType); responseRes = res; if (contentType?.startsWith("text/plain")) { responseText = await res.clone().text(); return finish(); } if ( !res.ok || !res.headers .get("content-type") ?.startsWith(EventStreamContentType) || res.status !== 200 ) { const responseTexts = [responseText]; let extraInfo = await res.clone().text(); try { const resJson = await res.clone().json(); extraInfo = prettyObject(resJson); } catch {} if (res.status === 401) { responseTexts.push(Locale.Error.Unauthorized); } if (extraInfo) { responseTexts.push(extraInfo); } responseText = responseTexts.join("\n\n"); return finish(); } }, onmessage(msg) { if (msg.data === "[DONE]" || finished) { return finish(); } const text = msg.data; // Skip empty messages if (!text || text.trim().length === 0) { return; } try { const chunk = parseSSE(text, runTools); if (chunk) { remainText += chunk; } } catch (e) { console.error("[Request] parse error", text, msg, e); } }, onclose() { finish(); }, onerror(e) { options?.onError?.(e); throw e; }, openWhenHidden: true, }); } console.debug("[ChatAPI] start"); chatApi(chatPath, headers, requestPayload, tools); // call fetchEventSource } export function streamWithThink( chatPath: string, requestPayload: any, headers: any, tools: any[], funcs: Record<string, Function>, controller: AbortController, parseSSE: ( text: string, runTools: any[], ) => { isThinking: boolean; content: string | undefined; }, processToolMessage: ( requestPayload: any, toolCallMessage: any, toolCallResult: any[], ) => void, options: any, ) { let responseText = ""; let remainText = ""; let finished = false; let running = false; let runTools: any[] = []; let responseRes: Response; let isInThinkingMode = false; let lastIsThinking = false; let lastIsThinkingTagged = false; //between <think> and </think> tags // animate response to make it looks smooth function animateResponseText() { if (finished || controller.signal.aborted) { responseText += remainText; console.log("[Response Animation] finished"); if (responseText?.length === 0) { options.onError?.(new Error("empty response from server")); } return; } if (remainText.length > 0) { const fetchCount = Math.max(1, Math.round(remainText.length / 60)); const fetchText = remainText.slice(0, fetchCount); responseText += fetchText; remainText = remainText.slice(fetchCount); options.onUpdate?.(responseText, fetchText); } requestAnimationFrame(animateResponseText); } // start animaion animateResponseText(); const finish = () => { if (!finished) { if (!running && runTools.length > 0) { const toolCallMessage = { role: "assistant", tool_calls: [...runTools], }; running = true; runTools.splice(0, runTools.length); // empty runTools return Promise.all( toolCallMessage.tool_calls.map((tool) => { options?.onBeforeTool?.(tool); return Promise.resolve( // @ts-ignore funcs[tool.function.name]( // @ts-ignore tool?.function?.arguments ? JSON.parse(tool?.function?.arguments) : {}, ), ) .then((res) => { let content = res.data || res?.statusText; // hotfix #5614 content = typeof content === "string" ? content : JSON.stringify(content); if (res.status >= 300) { return Promise.reject(content); } return content; }) .then((content) => { options?.onAfterTool?.({ ...tool, content, isError: false, }); return content; }) .catch((e) => { options?.onAfterTool?.({ ...tool, isError: true, errorMsg: e.toString(), }); return e.toString(); }) .then((content) => ({ name: tool.function.name, role: "tool", content, tool_call_id: tool.id, })); }), ).then((toolCallResult) => { processToolMessage(requestPayload, toolCallMessage, toolCallResult); setTimeout(() => { // call again console.debug("[ChatAPI] restart"); running = false; chatApi(chatPath, headers, requestPayload, tools); // call fetchEventSource }, 60); }); return; } if (running) { return; } console.debug("[ChatAPI] end"); finished = true; options.onFinish(responseText + remainText, responseRes); } }; controller.signal.onabort = finish; function chatApi( chatPath: string, headers: any, requestPayload: any, tools: any, ) { const chatPayload = { method: "POST", body: JSON.stringify({ ...requestPayload, tools: tools && tools.length ? tools : undefined, }), signal: controller.signal, headers, }; const requestTimeoutId = setTimeout( () => controller.abort(), REQUEST_TIMEOUT_MS, ); fetchEventSource(chatPath, { fetch: tauriFetch as any, ...chatPayload, async onopen(res) { clearTimeout(requestTimeoutId); const contentType = res.headers.get("content-type"); console.log("[Request] response content type: ", contentType); responseRes = res; if (contentType?.startsWith("text/plain")) { responseText = await res.clone().text(); return finish(); } if ( !res.ok || !res.headers .get("content-type") ?.startsWith(EventStreamContentType) || res.status !== 200 ) { const responseTexts = [responseText]; let extraInfo = await res.clone().text(); try { const resJson = await res.clone().json(); extraInfo = prettyObject(resJson); } catch {} if (res.status === 401) { responseTexts.push(Locale.Error.Unauthorized); } if (extraInfo) { responseTexts.push(extraInfo); } responseText = responseTexts.join("\n\n"); return finish(); } }, onmessage(msg) { if (msg.data === "[DONE]" || finished) { return finish(); } const text = msg.data; // Skip empty messages if (!text || text.trim().length === 0) { return; } try { const chunk = parseSSE(text, runTools); // Skip if content is empty if (!chunk?.content || chunk.content.length === 0) { return; } // deal with <think> and </think> tags start if (!chunk.isThinking) { if (chunk.content.startsWith("<think>")) { chunk.isThinking = true; chunk.content = chunk.content.slice(7).trim(); lastIsThinkingTagged = true; } else if (chunk.content.endsWith("</think>")) { chunk.isThinking = false; chunk.content = chunk.content.slice(0, -8).trim(); lastIsThinkingTagged = false; } else if (lastIsThinkingTagged) { chunk.isThinking = true; } } // deal with <think> and </think> tags start // Check if thinking mode changed const isThinkingChanged = lastIsThinking !== chunk.isThinking; lastIsThinking = chunk.isThinking; if (chunk.isThinking) { // If in thinking mode if (!isInThinkingMode || isThinkingChanged) { // If this is a new thinking block or mode changed, add prefix isInThinkingMode = true; if (remainText.length > 0) { remainText += "\n"; } remainText += "> " + chunk.content; } else { // Handle newlines in thinking content if (chunk.content.includes("\n\n")) { const lines = chunk.content.split("\n\n"); remainText += lines.join("\n\n> "); } else { remainText += chunk.content; } } } else { // If in normal mode if (isInThinkingMode || isThinkingChanged) { // If switching from thinking mode to normal mode isInThinkingMode = false; remainText += "\n\n" + chunk.content; } else { remainText += chunk.content; } } } catch (e) { console.error("[Request] parse error", text, msg, e); // Don't throw error for parse failures, just log them } }, onclose() { finish(); }, onerror(e) { options?.onError?.(e); throw e; }, openWhenHidden: true, }); } console.debug("[ChatAPI] start"); chatApi(chatPath, headers, requestPayload, tools); // call fetchEventSource }