🐛 fix: fix image prompts with some user cases (#6406)

* refactor the lobeThinking to remark mode

* improve error style

* try to fix prompts

* fix tests

* improve prompts

* fix tests

src/features/Conversation/Error/index.tsx

@@ -14,6 +14,7 @@ import ClerkLogin from './ClerkLogin';
 import ErrorJsonViewer from './ErrorJsonViewer';
 import InvalidAPIKey from './InvalidAPIKey';
 import InvalidAccessCode from './InvalidAccessCode';
+import { ErrorActionContainer } from './style';
 
 const loading = () => <Skeleton active />;
 
@@ -120,7 +121,13 @@ const ErrorMessageExtra = memo<{ data: ChatMessage }>(({ data }) => {
 });
 
 export default memo<{ data: ChatMessage }>(({ data }) => (
-  <Suspense fallback={<Skeleton active style={{ width: '100%' }} />}>
+  <Suspense
+    fallback={
+      <ErrorActionContainer>
+        <Skeleton active style={{ width: '100%' }} />
+      </ErrorActionContainer>
+    }
+  >
     <ErrorMessageExtra data={data} />
   </Suspense>
 ));

src/features/Conversation/components/MarkdownElements/LobeThinking/Render.tsx

@@ -6,21 +6,12 @@ import { useChatStore } from '@/store/chat';
 import { chatSelectors } from '@/store/chat/selectors';
 
 import { MarkdownElementProps } from '../type';
-
-/**
- * Replace all line breaks in the matched `lobeArtifact` tag with an empty string
- */
-export const isLobeThinkingClosed = (input: string = '') => {
-  const openTag = `<${ARTIFACT_THINKING_TAG}>`;
-  const closeTag = `</${ARTIFACT_THINKING_TAG}>`;
-
-  return input.includes(openTag) && input.includes(closeTag);
-};
+import { isTagClosed } from '../utils';
 
 const Render = memo<MarkdownElementProps>(({ children, id }) => {
   const [isGenerating] = useChatStore((s) => {
     const message = chatSelectors.getMessageById(id)(s);
-    return [!isLobeThinkingClosed(message?.content)];
+    return [!isTagClosed(ARTIFACT_THINKING_TAG, message?.content)];
   });
 
   return (

src/features/Conversation/components/MarkdownElements/LobeThinking/index.ts

@@ -1,12 +1,12 @@
 import { ARTIFACT_THINKING_TAG } from '@/const/plugin';
 
+import { createRemarkCustomTagPlugin } from '../remarkPlugins/createRemarkCustomTagPlugin';
 import { MarkdownElement } from '../type';
 import Component from './Render';
-import rehypePlugin from './rehypePlugin';
 
 const LobeThinkingElement: MarkdownElement = {
   Component,
-  rehypePlugin,
+  remarkPlugin: createRemarkCustomTagPlugin(ARTIFACT_THINKING_TAG),
   tag: ARTIFACT_THINKING_TAG,
 };
 

src/features/Conversation/components/MarkdownElements/utils.ts

@@ -0,0 +1,6 @@
+export const isTagClosed = (tag: string, input: string = '') => {
+  const openTag = `<${tag}>`;
+  const closeTag = `</${tag}>`;
+
+  return input.includes(openTag) && input.includes(closeTag);
+};

src/libs/agent-runtime/utils/openaiCompatibleFactory/index.ts

@@ -211,21 +211,23 @@ export const LobeOpenAICompatibleFactory = <T extends Record<string, any> = any>
           callbacks: options?.callback,
           provider,
         };
+
         if (customClient?.createChatCompletionStream) {
           response = customClient.createChatCompletionStream(this.client, payload, this) as any;
         } else {
-          response = await this.client.chat.completions.create(
-            {
-              ...postPayload,
-              messages,
-              ...(chatCompletion?.noUserId ? {} : { user: options?.user }),
-            },
-            {
-              // https://github.com/lobehub/lobe-chat/pull/318
-              headers: { Accept: '*/*', ...options?.requestHeaders },
-              signal: options?.signal,
-            },
-          );
+          const finalPayload = {
+            ...postPayload,
+            messages,
+            ...(chatCompletion?.noUserId ? {} : { user: options?.user }),
+          };
+          if (debug?.chatCompletion?.()) {
+            console.log('[requestPayload]:', JSON.stringify(finalPayload, null, 2));
+          }
+          response = await this.client.chat.completions.create(finalPayload, {
+            // https://github.com/lobehub/lobe-chat/pull/318
+            headers: { Accept: '*/*', ...options?.requestHeaders },
+            signal: options?.signal,
+          });
         }
 
         if (postPayload.stream) {

src/prompts/files/index.test.ts

@@ -27,13 +27,20 @@ describe('filesPrompts', () => {
     });
 
     expect(result).toEqual(
-      `<files_info>
+      `<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->
+<context.instruction>following part contains context information injected by the system. Please follow these instructions:
+
+1. Always prioritize handling user-visible content.
+2. the context is only required when user's queries rely on it.
+</context.instruction>
+<files_info>
 <images>
 <images_docstring>here are user upload images you can refer to</images_docstring>
 <image name="test image" url="https://example.com/image.jpg"></image>
 </images>
 
-</files_info>`,
+</files_info>
+<!-- END SYSTEM CONTEXT -->`,
     );
   });
 
@@ -44,13 +51,20 @@ describe('filesPrompts', () => {
     });
 
     expect(result).toEqual(
-      `<files_info>
+      `<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->
+<context.instruction>following part contains context information injected by the system. Please follow these instructions:
+
+1. Always prioritize handling user-visible content.
+2. the context is only required when user's queries rely on it.
+</context.instruction>
+<files_info>
 
 <files>
 <files_docstring>here are user upload files you can refer to</files_docstring>
 <file id="file-1" name="test.pdf" type="application/pdf" size="1024" url="https://example.com/test.pdf"></file>
 </files>
-</files_info>`,
+</files_info>
+<!-- END SYSTEM CONTEXT -->`,
     );
   });
 
@@ -61,7 +75,13 @@ describe('filesPrompts', () => {
     });
 
     expect(result).toEqual(
-      `<files_info>
+      `<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->
+<context.instruction>following part contains context information injected by the system. Please follow these instructions:
+
+1. Always prioritize handling user-visible content.
+2. the context is only required when user's queries rely on it.
+</context.instruction>
+<files_info>
 <images>
 <images_docstring>here are user upload images you can refer to</images_docstring>
 <image name="test image" url="https://example.com/image.jpg"></image>
@@ -70,7 +90,8 @@ describe('filesPrompts', () => {
 <files_docstring>here are user upload files you can refer to</files_docstring>
 <file id="file-1" name="test.pdf" type="application/pdf" size="1024" url="https://example.com/test.pdf"></file>
 </files>
-</files_info>`,
+</files_info>
+<!-- END SYSTEM CONTEXT -->`,
     );
   });
 

src/prompts/files/index.ts

@@ -12,10 +12,17 @@ export const filesPrompts = ({
 }) => {
   if (imageList.length === 0 && (fileList || []).length === 0) return '';
 
-  const prompt = `<files_info>
+  const prompt = `<!-- SYSTEM CONTEXT (NOT PART OF USER QUERY) -->
+<context.instruction>following part contains context information injected by the system. Please follow these instructions:
+
+1. Always prioritize handling user-visible content.
+2. the context is only required when user's queries rely on it.
+</context.instruction>
+<files_info>
 ${imagesPrompts(imageList)}
 ${fileList ? filePrompts(fileList) : ''}
-</files_info>`;
+</files_info>
+<!-- END SYSTEM CONTEXT -->`;
 
   return prompt.trim();
 };