From 54977405cb3da2184749c3a0370e3094d09f0a5d Mon Sep 17 00:00:00 2001 From: an-lee Date: Thu, 22 Aug 2024 18:39:22 +0800 Subject: [PATCH] Feat add chat suggetion helper (#1006) * use recorder config * add message timestamp in prompt * refactor * add chat suggest * update prompt * update style * update style * update ux * refactor suggestion button * fix chat input * add hotkey for record * refactor agent reply * fix `replaceAll` * hotkey for continue * refactor chat list --- enjoy/src/commands/chat-suggestion.command.ts | 59 +++ enjoy/src/commands/index.ts | 1 + enjoy/src/commands/json.command.ts | 1 - enjoy/src/i18n/en.json | 1 + enjoy/src/i18n/zh-CN.json | 3 +- .../components/chats/chat-agent-message.tsx | 31 +- .../renderer/components/chats/chat-input.tsx | 224 ++++++++- .../components/chats/chat-message.tsx | 21 +- .../components/chats/chat-messages.tsx | 3 +- .../components/chats/chat-user-message.tsx | 463 ++++++++++-------- .../components/misc/wavesurfer-player.tsx | 3 + .../context/chat-session-provider.tsx | 24 +- enjoy/src/renderer/hooks/use-ai-command.tsx | 30 ++ 13 files changed, 631 insertions(+), 233 deletions(-) create mode 100644 enjoy/src/commands/chat-suggestion.command.ts diff --git a/enjoy/src/commands/chat-suggestion.command.ts b/enjoy/src/commands/chat-suggestion.command.ts new file mode 100644 index 00000000..2f0f6953 --- /dev/null +++ b/enjoy/src/commands/chat-suggestion.command.ts @@ -0,0 +1,59 @@ +import { ChatPromptTemplate } from "@langchain/core/prompts"; +import { z } from "zod"; +import { jsonCommand } from "./json.command"; + +export const chatSuggestionCommand = async ( + params: { + learningLanguage: string; + nativeLanguage: string; + context: string; + }, + options: { + key: string; + modelName?: string; + temperature?: number; + baseUrl?: string; + } +): Promise<{ + suggestions: { + text: string; + explaination: string; + }[]; +}> => { + const { learningLanguage, nativeLanguage, context } = params; + const schema = z.object({ + words: z.array( + z.object({ + content: z.string().min(1), + explaination: z.string().min(1), + }) + ), + }); + + const prompt = await ChatPromptTemplate.fromMessages([ + ["system", SYSTEM_PROMPT], + ["human", PROMPT], + ]).format({ + native_language: nativeLanguage, + learning_language: learningLanguage, + context, + }); + + return jsonCommand(prompt, { ...options, schema }); +}; + +const SYSTEM_PROMPT = `I speak {native_language}. You're my {learning_language} coach. I'am chatting with foreign friends. And I don't know what to say next. + +{context}`; + +const PROMPT = `Please provide me with at least 5 suggestions for what counld I say in {learning_language} and explain them in {native_language}. + +Reply in JSON format only. The output should be structured like this: +{{ + suggestions: [ + {{ + text: "suggestion in {learning_language}", + explaination: "explaination" + }} + ] +}}`; diff --git a/enjoy/src/commands/index.ts b/enjoy/src/commands/index.ts index 84edc67c..10ae9d7e 100644 --- a/enjoy/src/commands/index.ts +++ b/enjoy/src/commands/index.ts @@ -8,3 +8,4 @@ export * from "./punctuate.command"; export * from "./summarize-topic.command"; export * from "./text.command"; export * from "./refine.command"; +export * from "./chat-suggestion.command"; diff --git a/enjoy/src/commands/json.command.ts b/enjoy/src/commands/json.command.ts index d4efa3b6..505dc338 100644 --- a/enjoy/src/commands/json.command.ts +++ b/enjoy/src/commands/json.command.ts @@ -31,7 +31,6 @@ export const jsonCommand = async ( configuration: { baseURL: baseUrl, }, - cache: true, verbose: true, maxRetries: 1, }); diff --git a/enjoy/src/i18n/en.json b/enjoy/src/i18n/en.json index d059c087..3c0e3d37 100644 --- a/enjoy/src/i18n/en.json +++ b/enjoy/src/i18n/en.json @@ -697,6 +697,7 @@ "deleteChatAgentConfirmation": "Are you sure to delete this chat agent?", "deleteMessage": "Delete message", "deleteMessageConfirmation": "Are you sure to delete this message?", + "refine": "Refine", "suggestion": "Suggestion", "editChat": "Edit chat", "newChat": "New chat", diff --git a/enjoy/src/i18n/zh-CN.json b/enjoy/src/i18n/zh-CN.json index c50d02e4..d2581463 100644 --- a/enjoy/src/i18n/zh-CN.json +++ b/enjoy/src/i18n/zh-CN.json @@ -697,7 +697,8 @@ "deleteChatAgentConfirmation": "您确定要删除此角色吗？", "deleteMessage": "删除消息", "deleteMessageConfirmation": "您确定要删除此消息吗？", - "suggestion": "修改建议", + "refine": "修改润色", + "suggestion": "建议", "editChat": "编辑聊天", "newChat": "新聊天", "addChat": "添加聊天", diff --git a/enjoy/src/renderer/components/chats/chat-agent-message.tsx b/enjoy/src/renderer/components/chats/chat-agent-message.tsx index 9b3098d2..52fd7d1a 100644 --- a/enjoy/src/renderer/components/chats/chat-agent-message.tsx +++ b/enjoy/src/renderer/components/chats/chat-agent-message.tsx @@ -29,6 +29,7 @@ import { MicIcon, MoreVerticalIcon, RotateCcwIcon, + Volume2Icon, } from "lucide-react"; import { useContext, useEffect, useRef, useState } from "react"; import { @@ -39,8 +40,11 @@ import { useAiCommand, useConversation } from "@renderer/hooks"; import { useCopyToClipboard } from "@uidotdev/usehooks"; import { md5 } from "js-md5"; -export const ChatAgentMessage = (props: { chatMessage: ChatMessageType }) => { - const { chatMessage } = props; +export const ChatAgentMessage = (props: { + chatMessage: ChatMessageType; + isLastMessage: boolean; +}) => { + const { chatMessage, isLastMessage } = props; const { dispatchChatMessages, setShadowing, onDeleteMessage } = useContext( ChatSessionProviderContext ); @@ -51,11 +55,12 @@ export const ChatAgentMessage = (props: { chatMessage: ChatMessageType }) => { const [copied, setCopied] = useState(false); const [speeching, setSpeeching] = useState(false); const [resourcing, setResourcing] = useState(false); - const [displayContent, setDisplayContent] = useState(false); const { tts } = useConversation(); const [translation, setTranslation] = useState(); const [translating, setTranslating] = useState(false); const { translate, summarizeTopic } = useAiCommand(); + const [displayContent, setDisplayContent] = useState(!isLastMessage); + const [displayPlayer, setDisplayPlayer] = useState(false); const handleTranslate = async () => { if (translating) return; @@ -203,10 +208,22 @@ export const ChatAgentMessage = (props: { chatMessage: ChatMessageType }) => {
{Boolean(chatMessage.speech) ? ( <> - + {displayPlayer ? ( + + ) : ( + + )} {displayContent && ( <> diff --git a/enjoy/src/renderer/components/chats/chat-input.tsx b/enjoy/src/renderer/components/chats/chat-input.tsx index 10160000..9f9e5da7 100644 --- a/enjoy/src/renderer/components/chats/chat-input.tsx +++ b/enjoy/src/renderer/components/chats/chat-input.tsx @@ -7,16 +7,37 @@ import { SendIcon, StepForwardIcon, TextIcon, + WandIcon, XIcon, } from "lucide-react"; -import { Button, Textarea } from "@renderer/components/ui"; -import { useContext, useEffect, useRef, useState } from "react"; +import { + Button, + Popover, + PopoverArrow, + PopoverContent, + PopoverTrigger, + ScrollArea, + Separator, + Textarea, +} from "@renderer/components/ui"; +import { ReactElement, useContext, useEffect, useRef, useState } from "react"; import { LiveAudioVisualizer } from "react-audio-visualize"; -import { ChatSessionProviderContext } from "@renderer/context"; +import { + AppSettingsProviderContext, + ChatProviderContext, + ChatSessionProviderContext, + HotKeysSettingsProviderContext, +} from "@renderer/context"; import { t } from "i18next"; import autosize from "autosize"; +import { LoaderSpin } from "@renderer/components"; +import { useAiCommand } from "@renderer/hooks"; +import { formatDateTime } from "@renderer/lib/utils"; +import { md5 } from "js-md5"; +import { useHotkeys } from "react-hotkeys-hook"; export const ChatInput = () => { + const { currentChat } = useContext(ChatProviderContext); const { submitting, startRecording, @@ -30,10 +51,12 @@ export const ChatInput = () => { askAgent, onCreateMessage, } = useContext(ChatSessionProviderContext); + const { EnjoyApp } = useContext(AppSettingsProviderContext); const inputRef = useRef(null); const submitRef = useRef(null); const [inputMode, setInputMode] = useState<"text" | "audio">("audio"); const [content, setContent] = useState(""); + const { currentHotkeys } = useContext(HotKeysSettingsProviderContext); useEffect(() => { if (!inputRef.current) return; @@ -55,6 +78,38 @@ export const ChatInput = () => { }; }, [inputRef.current]); + useEffect(() => { + EnjoyApp.cacheObjects + .get(`chat-input-mode-${currentChat.id}`) + .then((cachedInputMode) => { + if (cachedInputMode) { + setInputMode(cachedInputMode as typeof inputMode); + } + }); + }, []); + + useEffect(() => { + EnjoyApp.cacheObjects.set(`chat-input-mode-${currentChat.id}`, inputMode); + }, [inputMode]); + + useHotkeys( + currentHotkeys.StartOrStopRecording, + () => { + if (isRecording) { + stopRecording(); + } else { + startRecording(); + } + }, + { + preventDefault: true, + } + ); + + useHotkeys(currentHotkeys.PlayNextSegment, () => askAgent(), { + preventDefault: true, + }); + if (isRecording) { return (
@@ -158,6 +213,16 @@ export const ChatInput = () => { )} + + + +
); }; + +const ChatSuggestionButton = (props: { + asChild?: boolean; + children?: ReactElement; +}) => { + const { currentChat } = useContext(ChatProviderContext); + const { chatMessages, onCreateMessage } = useContext( + ChatSessionProviderContext + ); + const [suggestions, setSuggestions] = useState< + { text: string; explaination: string }[] + >([]); + const [loading, setLoading] = useState(false); + const [open, setOpen] = useState(false); + const { EnjoyApp } = useContext(AppSettingsProviderContext); + + const { chatSuggestion } = useAiCommand(); + + const context = `I'm ${ + currentChat.members.find((member) => member.user).user.name + }. + + [Chat Topic] + ${currentChat.topic} + + [Chat Members] + ${currentChat.members.map((m) => { + if (m.user) { + return `- ${m.user.name} (${m.config.introduction})[It's me]`; + } else if (m.agent) { + return `- ${m.agent.name} (${m.agent.introduction})`; + } + })} + + [Chat History] + ${chatMessages + .filter((m) => m.state === "completed") + .map( + (message) => + `- ${(message.member.user || message.member.agent).name}: ${ + message.content + }(${formatDateTime(message.createdAt)})` + ) + .join("\n")} + `; + + const contextCacheKey = `chat-suggestion-${md5( + chatMessages + .filter((m) => m.state === "completed") + .map((m) => m.content) + .join("\n") + )}`; + + const suggest = async () => { + setLoading(true); + chatSuggestion(context, { + cacheKey: contextCacheKey, + }) + .then((res) => setSuggestions(res.suggestions)) + .finally(() => { + setLoading(false); + }); + }; + + useEffect(() => { + if (open && !suggestions?.length) { + suggest(); + } + }, [open]); + + useEffect(() => { + EnjoyApp.cacheObjects.get(contextCacheKey).then((result) => { + if (result && result?.suggestions) { + setSuggestions(result.suggestions as typeof suggestions); + } else { + setSuggestions([]); + } + }); + }, [contextCacheKey]); + + return ( + + + {props.asChild ? ( + { ...props.children } + ) : ( + + )} + + + {loading || suggestions.length === 0 ? ( + + ) : ( + +
+ {suggestions.map((suggestion, index) => ( +
+
{suggestion.explaination}
+
+
{suggestion.text}
+
+ +
+
+ +
+ ))} +
+ +
+
+ + )} + + + + ); +}; diff --git a/enjoy/src/renderer/components/chats/chat-message.tsx b/enjoy/src/renderer/components/chats/chat-message.tsx index fb76edf4..b3432e26 100644 --- a/enjoy/src/renderer/components/chats/chat-message.tsx +++ b/enjoy/src/renderer/components/chats/chat-message.tsx @@ -5,8 +5,11 @@ import { ChatSessionProviderContext, } from "@renderer/context"; -export const ChatMessage = (props: { chatMessage: ChatMessageType }) => { - const { chatMessage } = props; +export const ChatMessage = (props: { + chatMessage: ChatMessageType; + isLastMessage: boolean; +}) => { + const { chatMessage, isLastMessage } = props; const { EnjoyApp } = useContext(AppSettingsProviderContext); const { dispatchChatMessages } = useContext(ChatSessionProviderContext); @@ -22,8 +25,18 @@ export const ChatMessage = (props: { chatMessage: ChatMessageType }) => { }, [chatMessage]); if (chatMessage.member?.userType === "User") { - return ; + return ( + + ); } else if (props.chatMessage.member?.userType === "Agent") { - return ; + return ( + + ); } }; diff --git a/enjoy/src/renderer/components/chats/chat-messages.tsx b/enjoy/src/renderer/components/chats/chat-messages.tsx index 329bd052..a9c194bf 100644 --- a/enjoy/src/renderer/components/chats/chat-messages.tsx +++ b/enjoy/src/renderer/components/chats/chat-messages.tsx @@ -6,11 +6,12 @@ import { useContext } from "react"; export const ChatMessages = () => { const { chatMessages } = useContext(ChatSessionProviderContext); + const lastMessage = chatMessages[chatMessages.length - 1]; return (
{chatMessages.map((message) => ( - + ))}
); diff --git a/enjoy/src/renderer/components/chats/chat-user-message.tsx b/enjoy/src/renderer/components/chats/chat-user-message.tsx index ebadf4ed..08ca9934 100644 --- a/enjoy/src/renderer/components/chats/chat-user-message.tsx +++ b/enjoy/src/renderer/components/chats/chat-user-message.tsx @@ -24,7 +24,7 @@ import { t } from "i18next"; import { CheckIcon, ChevronDownIcon, - ChevronUpIcon, + ChevronRightIcon, CopyIcon, DownloadIcon, EditIcon, @@ -33,7 +33,8 @@ import { LoaderIcon, MicIcon, MoreVerticalIcon, - SparkleIcon, + SparklesIcon, + Volume2Icon, } from "lucide-react"; import { useContext, useEffect, useRef, useState } from "react"; import { @@ -45,44 +46,151 @@ import { useAiCommand } from "@renderer/hooks"; import { md5 } from "js-md5"; import { useCopyToClipboard } from "@uidotdev/usehooks"; -export const ChatUserMessage = (props: { chatMessage: ChatMessageType }) => { - const { chatMessage } = props; +export const ChatUserMessage = (props: { + chatMessage: ChatMessageType; + isLastMessage: boolean; +}) => { + const { chatMessage, isLastMessage } = props; + const { recording } = chatMessage; + const ref = useRef(null); + const [editing, setEditing] = useState(false); + const [content, setContent] = useState(chatMessage.content); + const { onUpdateMessage } = useContext(ChatSessionProviderContext); + const [displayPlayer, setDisplayPlayer] = useState(isLastMessage); + + useEffect(() => { + if (ref.current) { + ref.current.scrollIntoView({ behavior: "smooth" }); + } + }, [ref]); + + return ( +
+
+
+ {chatMessage.member.user.name} +
+ + + + {chatMessage.member.user.name} + + +
+
+
+ {recording && + (displayPlayer ? ( + <> + + {recording?.pronunciationAssessment && ( +
+ +
+ )} + + ) : ( + + ))} + {editing ? ( +
+