feat: upload audio file in chat

src/components/audio-player/audio-element.tsx

@@ -0,0 +1,20 @@
+import React from 'react';
+import styled from 'styled-components';
+
+const AudioWrapper = styled.div`
+  width: 100%;
+  height: 100%;
+  display: flex;
+  align-items: center;
+  justify-content: flex-start;
+`;
+
+const AudioElement: React.FC<any> = (props) => {
+  return (
+    <AudioWrapper>
+      <audio {...props}></audio>
+    </AudioWrapper>
+  );
+};
+
+export default AudioElement;

src/locales/en-US/playground.ts

@@ -90,7 +90,7 @@ export default {
   'playground.audio.stoprecord': 'Stop Recording',
   'playground.audio.generating.tips': 'Generated text will appear here.',
   'playground.audio.uploadfile.tips':
-    'Please upload an audio file, supported formats: {formats}',
+    'Upload an audio file, supported formats: {formats}',
   'playground.input.multiplePaste': 'Batch Input Mode',
   'playground.input.multiplePaste.tips':
     'When enabled, pasted multi-line text will be automatically split by newline into separate entries in the form.',

src/locales/zh-CN/playground.ts

@@ -87,7 +87,7 @@ export default {
   'playground.audio.startrecord': '开始录音',
   'playground.audio.stoprecord': '停止录音',
   'playground.audio.generating.tips': '生成的文本将出现在这里',
-  'playground.audio.uploadfile.tips': '请上传音频文件，支持格式：{formats}',
+  'playground.audio.uploadfile.tips': '上传音频文件，支持格式：{formats}',
   'playground.audio.button.generate': '生成文本',
   'playground.input.multiplePaste': '批量输入',
   'playground.input.multiplePaste.tips':

src/pages/playground/components/message-input.tsx

@@ -1,7 +1,14 @@
+import AudioElement from '@/components/audio-player/audio-element';
 import IconFont from '@/components/icon-font';
+import UploadAudio from '@/components/upload-audio';
 import HotKeys, { KeyMap } from '@/config/hotkeys';
-import { convertFileToBase64 } from '@/utils/load-audio-file';
-import { ClearOutlined, SendOutlined, SwapOutlined } from '@ant-design/icons';
+import { convertFileToBase64, readAudioFile } from '@/utils/load-audio-file';
+import {
+  ClearOutlined,
+  CustomerServiceOutlined,
+  SendOutlined,
+  SwapOutlined
+} from '@ant-design/icons';
 import { useIntl } from '@umijs/max';
 import { Button, Checkbox, Divider, Input, Tooltip } from 'antd';
 import _ from 'lodash';
@@ -14,8 +21,9 @@ import React, {
   useState
 } from 'react';
 import { useHotkeys } from 'react-hotkeys-hook';
+import styled from 'styled-components';
 import { Roles } from '../config';
-import { MessageItem } from '../config/types';
+import { AudioFormat, MessageItem } from '../config/types';
 import '../style/message-input.less';
 import ThumbImg from './thumb-img';
 import UploadImg from './upload-img';
@@ -26,6 +34,12 @@ const audioTypeMap: Record<string, string> = {
   'audio/mpeg': 'mp3'
 };
 
+const AudioWrapper = styled.div`
+  audio {
+    padding-top: 10px;
+  }
+`;
+
 type CurrentMessage = Omit<MessageItem, 'uid'>;
 
 type ActionType =
@@ -139,9 +153,14 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
       content: '',
       imgs: []
     });
-    const imgCountRef = useRef(0);
+    const uidCountRef = useRef(0);
     const inputRef = useRef<any>(null);
 
+    const updateUidCount = () => {
+      uidCountRef.current += 1;
+      return uidCountRef.current;
+    };
+
     const isDisabled = useMemo(() => {
       return disabled
         ? true
@@ -239,9 +258,9 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
 
           if (imgs.length) {
             const list = _.map(imgs, (img: string) => {
-              imgCountRef.current += 1;
+              updateUidCount();
               return {
-                uid: imgCountRef.current,
+                uid: uidCountRef.current,
                 dataUrl: img
               };
             });
@@ -273,14 +292,19 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
     }) => {
       // convert audio file to base64
       try {
-        console.log('audio file====', data.file);
         const base64Audio = await convertFileToBase64(data.file);
+        const audioData = await readAudioFile(data.file);
+        console.log('audioData====', audioData);
         setMessage({
           ...message,
-          audio: {
-            format: audioTypeMap[data.file.type],
-            dataUrl: base64Audio
-          }
+          audio: [
+            {
+              uid: updateUidCount(),
+              format: audioTypeMap[data.file.type] as AudioFormat,
+              base64: base64Audio.split(',')[1],
+              data: _.pick(audioData, ['url', 'name', 'duration'])
+            }
+          ]
         });
       } catch (error) {
         console.error('Error converting audio to Base64:', error);
@@ -298,6 +322,17 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
       });
     };
 
+    const handleDeleteAudio = (uid: number | string) => {
+      const list = _.filter(
+        message.audio,
+        (item: MessageItem) => item.uid !== uid
+      );
+      setMessage({
+        ...message,
+        audio: list
+      });
+    };
+
     const handleOnPaste = (e: any) => {
       const text = e.clipboardData.getData('text');
       if (!text) {
@@ -318,16 +353,6 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
       }
     };
 
-    const handleDeleteLastImage = useCallback(() => {
-      if (message.imgs && message.imgs?.length > 0) {
-        const newImgList = [...(message.imgs || [])];
-        const lastImage = newImgList.pop();
-        if (lastImage) {
-          handleDeleteImg(lastImage.uid);
-        }
-      }
-    }, [message.imgs, handleDeleteImg]);
-
     useImperativeHandle(ref, () => ({
       handleInputChange: handleInputChange
     }));
@@ -387,7 +412,7 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
                 size="middle"
               ></UploadImg>
             )}
-            {/* {actions.includes('upload') && message.role === Roles.User && (
+            {actions.includes('upload') && message.role === Roles.User && (
               <UploadAudio
                 type="text"
                 accept={'.mp3,.wav'}
@@ -396,12 +421,6 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
                 icon={<CustomerServiceOutlined />}
                 onChange={handleUploadAudioChange}
               ></UploadAudio>
-            )} */}
-            {actions.includes('upload') && message.role === Roles.User && (
-              <UploadImg
-                handleUpdateImgList={handleUpdateImgList}
-                size="middle"
-              ></UploadImg>
             )}
             {tools}
             {actions.includes('clear') && (
@@ -490,11 +509,22 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
             )}
           </div>
         </div>
-        <ThumbImg
-          dataList={message.imgs || []}
-          onDelete={handleDeleteImg}
-          editable={true}
-        ></ThumbImg>
+        <div className="flex">
+          <ThumbImg
+            dataList={message.imgs || []}
+            onDelete={handleDeleteImg}
+            editable={true}
+          ></ThumbImg>
+          {message.audio && message.audio.length > 0 && (
+            <AudioWrapper>
+              <AudioElement
+                src={message.audio?.[0].data?.url}
+                onDelete={handleDeleteAudio}
+                controls
+              ></AudioElement>
+            </AudioWrapper>
+          )}
+        </div>
         <div className="input-box">
           {actions.includes('paste') ? (
             <TextArea

src/pages/playground/components/multiple-chat/message-body.tsx

@@ -1,3 +1,4 @@
+import AudioElement from '@/components/audio-player/audio-element';
 import FullMarkdown from '@/components/markdown-viewer/full-markdown';
 import { Input } from 'antd';
 import classNames from 'classnames';
@@ -8,11 +9,19 @@ import React, {
   useImperativeHandle,
   useRef
 } from 'react';
+import styled from 'styled-components';
 import { Roles } from '../../config';
 import { MessageItem, MessageItemAction } from '../../config/types';
 import ThumbImg from '../thumb-img';
 import ThinkContent from './think-content';
 
+const AudioWrapper = styled.div`
+  padding-left: 10px;
+  audio {
+    padding-top: 10px;
+  }
+`;
+
 interface MessageBodyProps {
   ref?: any;
   data: MessageItem;
@@ -93,6 +102,7 @@ const MessageBody: React.FC<MessageBodyProps> = forwardRef(
               role: data.role,
               content: data.content,
               uid: data.uid,
+              audio: data.audio || [],
               imgs: [...(data.imgs || []), ...list]
             });
           }
@@ -117,6 +127,7 @@ const MessageBody: React.FC<MessageBodyProps> = forwardRef(
         role: data.role,
         content: data.content,
         uid: data.uid,
+        audio: data.audio || [],
         imgs: list
       });
     };
@@ -124,6 +135,7 @@ const MessageBody: React.FC<MessageBodyProps> = forwardRef(
     const handleMessageChange = (e: any) => {
       updateMessage?.({
         imgs: data.imgs || [],
+        audio: data.audio || [],
         role: data.role,
         content: e.target.value,
         uid: data.uid
@@ -143,6 +155,7 @@ const MessageBody: React.FC<MessageBodyProps> = forwardRef(
         role: data.role,
         content: editContent,
         uid: data.uid,
+        audio: data.audio || [],
         imgs: data.imgs
       });
     };
@@ -182,14 +195,25 @@ const MessageBody: React.FC<MessageBodyProps> = forwardRef(
       return (
         <div
           className={classNames('content-item-content', {
-            'has-img': data.imgs?.length
+            'has-img':
+              data.imgs?.length || (data.audio && data.audio?.length > 0)
           })}
         >
-          <ThumbImg
-            editable={editable}
-            dataList={data.imgs || []}
-            onDelete={handleDeleteImg}
-          />
+          <div className="justify-start">
+            <ThumbImg
+              editable={editable}
+              dataList={data.imgs || []}
+              onDelete={handleDeleteImg}
+            />
+            {data.audio && data.audio.length > 0 && (
+              <AudioWrapper>
+                <AudioElement
+                  src={data.audio?.[0]?.data.url}
+                  controls
+                ></AudioElement>
+              </AudioWrapper>
+            )}
+          </div>
           {data.content && <div className="text">{data.content}</div>}
         </div>
       );
@@ -199,15 +223,26 @@ const MessageBody: React.FC<MessageBodyProps> = forwardRef(
       return (
         <div
           className={classNames('message-content-input', {
-            'has-img': data.imgs?.length
+            'has-img':
+              data.imgs?.length || (data.audio && data.audio?.length > 0)
           })}
           onClick={handleClickWrapper}
         >
-          <ThumbImg
-            editable={editable}
-            dataList={data.imgs || []}
-            onDelete={handleDeleteImg}
-          />
+          <div className="justify-start">
+            <ThumbImg
+              editable={editable}
+              dataList={data.imgs || []}
+              onDelete={handleDeleteImg}
+            />
+            {data.audio && data.audio.length > 0 && (
+              <AudioWrapper>
+                <AudioElement
+                  src={data.audio?.[0]?.data.url}
+                  controls
+                ></AudioElement>
+              </AudioWrapper>
+            )}
+          </div>
           <>
             {data.role === Roles.User ? (
               <Input.TextArea

src/pages/playground/config/index.ts

@@ -1,6 +1,6 @@
 import _, { map } from 'lodash';
 import { CREAT_IMAGE_API } from '../apis';
-import { MessageItem } from './types';
+import { AudioData, MessageItem } from './types';
 
 export const Roles = {
   User: 'user',
@@ -76,7 +76,33 @@ export const generateMessagesByListContent = (messageList: any[]) => {
         content: content
       };
     }
-    return _.omit(item, ['uid', 'imgs']);
+
+    // audio
+    if (item.audio?.length) {
+      const content = map(item.audio, (item: AudioData) => {
+        return {
+          type: 'input_audio',
+          input_audio: {
+            data: item.base64,
+            format: item.format
+          }
+        };
+      });
+
+      if (item.content) {
+        content.push({
+          type: 'text',
+          text: item.content
+        });
+      }
+
+      return {
+        role: item.role,
+        content: content
+      };
+    }
+
+    return _.omit(item, ['uid', 'imgs', 'audio']);
   });
 };
 

src/pages/playground/config/types.ts

@@ -13,10 +13,23 @@ export type MessageItemAction =
   | 'markdown'
   | 'edit';
 
+export type AudioFormat = 'wav' | 'mp3';
+
+export interface AudioData {
+  uid: string | number;
+  base64: string;
+  format: AudioFormat;
+  data: {
+    url: string;
+    name: string;
+    duration: number;
+  };
+}
+
 export interface MessageItem {
   content: string;
   imgs?: { uid: string | number; dataUrl: string }[];
-  audio?: { uid: string | number; dataUrl: string; format: 'wav' | 'mp3' };
+  audio?: AudioData[];
   role: string;
   title?: React.ReactNode;
   uid: number;

src/utils/load-audio-file.ts

@@ -9,7 +9,16 @@ export const convertFileToBase64 = (file: File): Promise<string> => {
   });
 };
 
-export const loadAudioData = async (data: any, type: string) => {
+export const loadAudioData = async (
+  data: any,
+  type: string
+): Promise<{
+  data: Blob;
+  size: number | string;
+  type: string;
+  duration: number;
+  url: string;
+}> => {
   return new Promise((resolve, reject) => {
     try {
       const audioBlob = new Blob([data], { type: type });
@@ -40,7 +49,9 @@ export const loadAudioData = async (data: any, type: string) => {
   });
 };
 
-export const readAudioFile = async (file: File) => {
+export const readAudioFile = async (
+  file: File
+): Promise<{ url: string; name: string; duration: number }> => {
   console.log('file====', file);
   return new Promise((resolve, reject) => {
     const reader = new FileReader();