From dba24c5a5779aee3d6af292708fa32e815259399 Mon Sep 17 00:00:00 2001 From: an-lee Date: Wed, 14 Aug 2024 15:44:30 +0800 Subject: [PATCH] Improve chat (#966) * add text input for chat * fix up * fix message CRUD * update chat message actions * may cancel recording * audio input as default * fix quick start * improve action icons * update * fix prompt preview * add tool tips * update style --- enjoy/package.json | 2 + enjoy/src/constants/index.ts | 4 +- enjoy/src/i18n/en.json | 4 +- enjoy/src/i18n/zh-CN.json | 4 +- .../main/db/handlers/chat-agents-handler.ts | 4 +- .../main/db/handlers/chat-messages-handler.ts | 19 ++- enjoy/src/main/db/models/chat-member.ts | 2 + enjoy/src/main/db/models/chat-message.ts | 3 + .../components/chats/chat-agent-message.tsx | 64 +++---- .../renderer/components/chats/chat-form.tsx | 56 ++++--- .../renderer/components/chats/chat-input.tsx | 158 +++++++++++++++--- .../components/chats/chat-sidebar.tsx | 26 ++- .../components/chats/chat-user-message.tsx | 60 +++++-- enjoy/src/renderer/components/chats/chat.tsx | 4 +- .../context/chat-session-provider.tsx | 79 ++++++--- enjoy/src/renderer/hooks/use-chat-message.tsx | 63 +++++-- enjoy/src/types/enjoy-app.d.ts | 2 +- yarn.lock | 9 + 18 files changed, 416 insertions(+), 147 deletions(-) diff --git a/enjoy/package.json b/enjoy/package.json index 688df4b7..e352d503 100644 --- a/enjoy/package.json +++ b/enjoy/package.json @@ -52,6 +52,7 @@ "@types/intl-tel-input": "^18.1.4", "@types/lodash": "^4.17.7", "@types/mark.js": "^8.11.12", + "@types/mustache": "^4", "@types/node": "^22.2.0", "@types/rails__actioncable": "^6.1.11", "@types/react": "^18.3.3", @@ -153,6 +154,7 @@ "lucide-react": "^0.427.0", "mark.js": "^8.11.1", "microsoft-cognitiveservices-speech-sdk": "^1.40.0", + "mustache": "^4.2.0", "next-themes": "^0.3.0", "openai": "^4.55.5", "pitchfinder": "^2.3.2", diff --git a/enjoy/src/constants/index.ts b/enjoy/src/constants/index.ts index 54c1ee0d..803a1893 100644 --- a/enjoy/src/constants/index.ts +++ b/enjoy/src/constants/index.ts @@ -58,7 +58,9 @@ export const NOT_SUPPORT_JSON_FORMAT_MODELS = [ "gpt-4-32k", ]; -export const CHAT_SYSTEM_PROMPT_TEMPLATE = `You are {name}. {agent_prompt} +export const CHAT_SYSTEM_PROMPT_TEMPLATE = `You are {name}. +{agent_prompt} + You are chatting in an online chat room. {agent_chat_prompt} diff --git a/enjoy/src/i18n/en.json b/enjoy/src/i18n/en.json index 7a506157..0371f8dd 100644 --- a/enjoy/src/i18n/en.json +++ b/enjoy/src/i18n/en.json @@ -714,5 +714,7 @@ "quickStart": "Quick start", "itsYourTurn": "It's your turn. Please start speaking.", "displayContent": "Display content", - "hideContent": "Hide content" + "hideContent": "Hide content", + "audioInput": "Audio input", + "textInput": "Text input" } diff --git a/enjoy/src/i18n/zh-CN.json b/enjoy/src/i18n/zh-CN.json index 7ebcd5df..63fbf952 100644 --- a/enjoy/src/i18n/zh-CN.json +++ b/enjoy/src/i18n/zh-CN.json @@ -714,5 +714,7 @@ "quickStart": "快速开始", "itsYourTurn": "轮到您了，请开始说点什么", "displayContent": "显示内容", - "hideContent": "隐藏内容" + "hideContent": "隐藏内容", + "audioInput": "语音输入", + "textInput": "文字输入" } diff --git a/enjoy/src/main/db/handlers/chat-agents-handler.ts b/enjoy/src/main/db/handlers/chat-agents-handler.ts index a5b8213a..a4bf3be8 100644 --- a/enjoy/src/main/db/handlers/chat-agents-handler.ts +++ b/enjoy/src/main/db/handlers/chat-agents-handler.ts @@ -34,9 +34,7 @@ class ChatAgentsHandler { private async findOne( _event: IpcMainEvent, - options: FindOptions> & { - where: WhereOptions; - } + options: FindOptions> ) { const agent = await ChatAgent.findOne(options); return agent?.toJSON(); diff --git a/enjoy/src/main/db/handlers/chat-messages-handler.ts b/enjoy/src/main/db/handlers/chat-messages-handler.ts index 2e503335..07ae1e4c 100644 --- a/enjoy/src/main/db/handlers/chat-messages-handler.ts +++ b/enjoy/src/main/db/handlers/chat-messages-handler.ts @@ -70,15 +70,10 @@ class ChatMessagesHandler { transaction ); message.recording = recording; - } else { - message.recording?.update( - { referenceText: message.content }, - { transaction } - ); } await transaction.commit(); - return message.toJSON(); + return (await message.reload()).toJSON(); } private async update( @@ -117,11 +112,17 @@ class ChatMessagesHandler { transaction ); message.recording = recording; + } else if (message.recording) { + message.recording.update( + { + referenceText: message.content, + }, + { transaction } + ); } - await transaction.commit(); - return message.toJSON(); + return (await message.reload()).toJSON(); } private async destroy(_event: IpcMainEvent, id: string) { @@ -129,6 +130,8 @@ class ChatMessagesHandler { if (!message) return; await message.destroy(); + + return message.toJSON(); } register() { diff --git a/enjoy/src/main/db/models/chat-member.ts b/enjoy/src/main/db/models/chat-member.ts index 9d559db6..b91c53fe 100644 --- a/enjoy/src/main/db/models/chat-member.ts +++ b/enjoy/src/main/db/models/chat-member.ts @@ -82,6 +82,8 @@ export class ChatMember extends Model { } return user; + } else { + return null; } } diff --git a/enjoy/src/main/db/models/chat-message.ts b/enjoy/src/main/db/models/chat-message.ts index 3d4e6136..e0b28060 100644 --- a/enjoy/src/main/db/models/chat-message.ts +++ b/enjoy/src/main/db/models/chat-message.ts @@ -29,6 +29,7 @@ const logger = log.scope("db/models/chat-message"); include: [ { association: ChatMessage.associations.member, + model: ChatMember, include: [ { association: ChatMember.associations.agent, @@ -37,6 +38,7 @@ const logger = log.scope("db/models/chat-message"); }, { association: ChatMessage.associations.recording, + model: Recording, include: [ { association: Recording.associations.pronunciationAssessment, @@ -45,6 +47,7 @@ const logger = log.scope("db/models/chat-message"); }, { association: ChatMessage.associations.speech, + model: Speech, }, ], order: [["createdAt", "ASC"]], diff --git a/enjoy/src/renderer/components/chats/chat-agent-message.tsx b/enjoy/src/renderer/components/chats/chat-agent-message.tsx index 0ecbb408..9b3098d2 100644 --- a/enjoy/src/renderer/components/chats/chat-agent-message.tsx +++ b/enjoy/src/renderer/components/chats/chat-agent-message.tsx @@ -200,7 +200,7 @@ export const ChatAgentMessage = (props: { chatMessage: ChatMessageType }) => {
{agent.name}
-
+
{Boolean(chatMessage.speech) ? ( <> { src={chatMessage.speech.src} /> {displayContent && ( - - {chatMessage.content} - - )} - {translation && ( - - {translation} - + <> + + {chatMessage.content} + + {translation && ( + + {translation} + + )} + )} ) : speeching ? ( @@ -230,21 +232,6 @@ export const ChatAgentMessage = (props: { chatMessage: ChatMessageType }) => { )}
- {displayContent ? ( - setDisplayContent(false)} - /> - ) : ( - setDisplayContent(true)} - /> - )} {Boolean(chatMessage.speech) && (resourcing ? ( { className="w-4 h-4 cursor-pointer" /> ))} + {displayContent ? ( + setDisplayContent(false)} + /> + ) : ( + setDisplayContent(true)} + /> + )} {translating ? ( { className="w-4 h-4 animate-spin" /> ) : ( - + displayContent && ( + + ) )} {copied ? ( diff --git a/enjoy/src/renderer/components/chats/chat-form.tsx b/enjoy/src/renderer/components/chats/chat-form.tsx index c0d38578..1d962544 100644 --- a/enjoy/src/renderer/components/chats/chat-form.tsx +++ b/enjoy/src/renderer/components/chats/chat-form.tsx @@ -38,7 +38,8 @@ import { AISettingsProviderContext, AppSettingsProviderContext, } from "@renderer/context"; -import { LANGUAGES } from "@/constants"; +import { CHAT_SYSTEM_PROMPT_TEMPLATE, LANGUAGES } from "@/constants"; +import Mustache from "mustache"; export const ChatForm = (props: { chat?: ChatType; @@ -297,6 +298,7 @@ export const ChatForm = (props: { {editingMember && ( >; member: Partial; @@ -385,6 +388,7 @@ const MemberForm = (props: { onConfigChange?: (config: ChatMemberType["config"]) => void; }) => { const { + language, topic, members, member, @@ -396,30 +400,32 @@ const MemberForm = (props: { const { user } = useContext(AppSettingsProviderContext); const chatAgent = chatAgents.find((a) => a.id === member.userId); - const fullPrompt = - chatAgent && - `${chatAgent.config.prompt} -You are chatting in a chat room. You always reply in ${ - LANGUAGES.find((lang) => lang.code === chatAgent.language).name - }. -${member.config.prompt || ""} - - -${topic} - - -${members - .map((m) => { - if (m.userType === "User") { - return `- ${user.name} (${m.config.introduction})`; - } else { - const agent = chatAgents.find((a) => a.id === m.userId); - if (!agent) return ""; - return `- ${agent.name} (${agent.introduction})`; - } - }) - .join("\n")} -`; + const fullPrompt = chatAgent + ? Mustache.render( + CHAT_SYSTEM_PROMPT_TEMPLATE, + { + name: chatAgent.name, + agent_prompt: chatAgent.config.prompt, + agent_chat_prompt: member.config.chatPrompt, + language, + topic, + members: members + .map((m) => { + if (m.userType === "User") { + return `- ${user.name} (${m.config.introduction})`; + } else { + const agent = chatAgents.find((a) => a.id === m.userId); + if (!agent) return ""; + return `- ${agent.name} (${agent.introduction})`; + } + }) + .join("\n"), + history: "...", + }, + {}, + ["{", "}"] + ) + : ""; if (member.userType === "User") { return ( diff --git a/enjoy/src/renderer/components/chats/chat-input.tsx b/enjoy/src/renderer/components/chats/chat-input.tsx index 58603763..3d161d57 100644 --- a/enjoy/src/renderer/components/chats/chat-input.tsx +++ b/enjoy/src/renderer/components/chats/chat-input.tsx @@ -1,37 +1,59 @@ import { + CheckIcon, LoaderIcon, MicIcon, PauseIcon, PlayIcon, - SquareIcon, + SendIcon, StepForwardIcon, + TextIcon, + XIcon, } from "lucide-react"; -import { Button } from "@renderer/components/ui"; -import { useContext } from "react"; +import { Button, Textarea } from "@renderer/components/ui"; +import { useContext, useEffect, useRef, useState } from "react"; import { LiveAudioVisualizer } from "react-audio-visualize"; import { ChatSessionProviderContext } from "@renderer/context"; import { t } from "i18next"; +import autosize from "autosize"; export const ChatInput = () => { const { submitting, startRecording, stopRecording, + cancelRecording, togglePauseResume, isRecording, mediaRecorder, recordingTime, isPaused, askAgent, + onCreateMessage, } = useContext(ChatSessionProviderContext); + const inputRef = useRef(null); + const submitRef = useRef(null); + const [inputMode, setInputMode] = useState<"text" | "audio">("audio"); + const [content, setContent] = useState(""); - if (submitting) { - return ( -
- -
- ); - } + useEffect(() => { + if (!inputRef.current) return; + + autosize(inputRef.current); + + inputRef.current.addEventListener("keypress", (event) => { + if (event.key === "Enter" && !event.shiftKey) { + event.preventDefault(); + submitRef.current?.click(); + } + }); + + inputRef.current.focus(); + + return () => { + inputRef.current?.removeEventListener("keypress", () => {}); + autosize.destroy(inputRef.current); + }; + }, [inputRef.current]); if (isRecording) { return ( @@ -52,46 +74,142 @@ export const ChatInput = () => { {Math.floor(recordingTime / 60)}: {String(recordingTime % 60).padStart(2, "0")} +
); } + if (inputMode === "text") { + return ( +
+ +