From 29932eeb44b3571ede9ea069b935ce35dd684882 Mon Sep 17 00:00:00 2001
From: an-lee <an.lee.work@gmail.com>
Date: Thu, 17 Oct 2024 16:05:33 +0800
Subject: [PATCH] Fix voice chat (#1132)

* fix use-transcribe

* fix transcribe response

* fix voice chat
---
 .../main/db/handlers/chat-messages-handler.ts | 16 ++--
 enjoy/src/main/db/models/chat-member.ts       |  4 +-
 enjoy/src/main/db/models/chat-message.ts      |  4 +-
 enjoy/src/main/db/models/chat.ts              | 15 +---
 enjoy/src/main/db/models/recording.ts         |  4 +-
 .../components/chats/chat-user-message.tsx    | 82 +++++++++++--------
 enjoy/src/renderer/hooks/use-transcribe.tsx   |  3 +
 7 files changed, 70 insertions(+), 58 deletions(-)

diff --git a/enjoy/src/main/db/handlers/chat-messages-handler.ts b/enjoy/src/main/db/handlers/chat-messages-handler.ts
index 16f8527d..c3886ddc 100644
--- a/enjoy/src/main/db/handlers/chat-messages-handler.ts
+++ b/enjoy/src/main/db/handlers/chat-messages-handler.ts
@@ -103,9 +103,6 @@ class ChatMessagesHandler {
     const transaction = await ChatMessage.sequelize.transaction();
 
     try {
-      // update content
-      await message.update({ ...data }, { transaction });
-
       if (recordingUrl) {
         // destroy existing recording
         await message.recording?.destroy({ transaction });
@@ -113,27 +110,30 @@ class ChatMessagesHandler {
         // create new recording
         const filePath = enjoyUrlToPath(recordingUrl);
         const blob = fs.readFileSync(filePath);
-        const recording = await Recording.createFromBlob(
+        await Recording.createFromBlob(
           {
             type: "audio/wav",
             arrayBuffer: blob,
           },
           {
             targetType: "ChatMessage",
-            targetId: message.id,
-            referenceText: message.content,
+            targetId: id,
+            referenceText: data.content,
           },
           transaction
         );
-        message.recording = recording;
       } else if (message.recording) {
         await message.recording.update(
           {
-            referenceText: message.content,
+            referenceText: data.content,
           },
           { transaction }
         );
       }
+
+      // update content
+      await message.update({ ...data }, { transaction });
+
       await transaction.commit();
 
       return (await message.reload()).toJSON();
diff --git a/enjoy/src/main/db/models/chat-member.ts b/enjoy/src/main/db/models/chat-member.ts
index 2eb45a16..a184cae8 100644
--- a/enjoy/src/main/db/models/chat-member.ts
+++ b/enjoy/src/main/db/models/chat-member.ts
@@ -134,8 +134,8 @@ export class ChatMember extends Model<ChatMember> {
   ) {
     if (!mainWindow.win) return;
 
-    if (action !== "destroy" && !member.agent) {
-      await member.reload();
+    if (action !== "destroy") {
+      member = await ChatMember.findByPk(member.id);
     }
     mainWindow.win.webContents.send("db-on-transaction", {
       model: "ChatMember",
diff --git a/enjoy/src/main/db/models/chat-message.ts b/enjoy/src/main/db/models/chat-message.ts
index 30ad3837..cfb08cad 100644
--- a/enjoy/src/main/db/models/chat-message.ts
+++ b/enjoy/src/main/db/models/chat-message.ts
@@ -200,8 +200,8 @@ export class ChatMessage extends Model<ChatMessage> {
   ) {
     if (!mainWindow.win) return;
 
-    if (action !== "destroy" && !chatMessage.agent) {
-      await chatMessage.reload();
+    if (action !== "destroy") {
+      chatMessage = await ChatMessage.findByPk(chatMessage.id);
     }
 
     mainWindow.win.webContents.send("db-on-transaction", {
diff --git a/enjoy/src/main/db/models/chat.ts b/enjoy/src/main/db/models/chat.ts
index 12c65798..26431c75 100644
--- a/enjoy/src/main/db/models/chat.ts
+++ b/enjoy/src/main/db/models/chat.ts
@@ -108,19 +108,10 @@ export class Chat extends Model<Chat> {
   static async notify(chat: Chat, action: "create" | "update" | "destroy") {
     if (!mainWindow.win) return;
 
-    if (
-      action !== "destroy" &&
-      (!chat.members || !chat.members.some((m) => m.agent))
-    ) {
-      chat.members = await ChatMember.findAll({
-        where: { chatId: chat.id },
-        include: [
-          {
-            association: "agent",
-          },
-        ],
-      });
+    if (action !== "destroy") {
+      chat = await Chat.findByPk(chat.id);
     }
+
     mainWindow.win.webContents.send("db-on-transaction", {
       model: "Chat",
       id: chat.id,
diff --git a/enjoy/src/main/db/models/recording.ts b/enjoy/src/main/db/models/recording.ts
index bda8167e..6887a3b4 100644
--- a/enjoy/src/main/db/models/recording.ts
+++ b/enjoy/src/main/db/models/recording.ts
@@ -173,7 +173,7 @@ export class Recording extends Model<Recording> {
       .then((result) => {
         logger.debug("upload result:", result.data);
         if (result.data.success) {
-          this.update({ uploadedAt: new Date() });
+          this.update({ uploadedAt: new Date() }, { hooks: false });
         } else {
           throw new Error(result.data);
         }
@@ -194,7 +194,7 @@ export class Recording extends Model<Recording> {
     });
 
     return webApi.syncRecording(this.toJSON()).then(() => {
-      this.update({ syncedAt: new Date() });
+      this.update({ syncedAt: new Date() }, { hooks: false });
     });
   }
 
diff --git a/enjoy/src/renderer/components/chats/chat-user-message.tsx b/enjoy/src/renderer/components/chats/chat-user-message.tsx
index 5c4dbe0b..a0deec8f 100644
--- a/enjoy/src/renderer/components/chats/chat-user-message.tsx
+++ b/enjoy/src/renderer/components/chats/chat-user-message.tsx
@@ -67,13 +67,11 @@ export const ChatUserMessage = (props: {
   useEffect(() => {
     if (!isLastMessage) return;
     // If the message is from recording, wait for user to confirm before asking agent
-    if (
-      chatMessage.recording &&
-      chatMessage.state !== ChatMessageStateEnum.COMPLETED
-    )
+    if (chatMessage.state !== ChatMessageStateEnum.COMPLETED) {
       return;
-
-    askAgent();
+    } else {
+      askAgent();
+    }
   }, [chatMessage]);
 
   return (
@@ -87,32 +85,11 @@ export const ChatUserMessage = (props: {
                 : "bg-muted"
             }`}
           >
-            {recording &&
-              (displayPlayer ? (
-                <>
-                  <WavesurferPlayer
-                    id={recording.id}
-                    src={recording.src}
-                    autoplay={true}
-                  />
-                  {recording?.pronunciationAssessment && (
-                    <div className="flex justify-end">
-                      <PronunciationAssessmentScoreDetail
-                        assessment={recording.pronunciationAssessment}
-                      />
-                    </div>
-                  )}
-                </>
-              ) : (
-                <Button
-                  onClick={() => setDisplayPlayer(true)}
-                  className="w-8 h-8"
-                  variant="ghost"
-                  size="icon"
-                >
-                  <Volume2Icon className="w-5 h-5" />
-                </Button>
-              ))}
+            <ChatUserMessageRecording
+              chatMessage={chatMessage}
+              displayPlayer={displayPlayer}
+              setDisplayPlayer={setDisplayPlayer}
+            />
             {editing ? (
               <div className="">
                 <Textarea
@@ -181,6 +158,47 @@ export const ChatUserMessage = (props: {
   );
 };
 
+const ChatUserMessageRecording = (props: {
+  chatMessage: ChatMessageType;
+  displayPlayer: boolean;
+  setDisplayPlayer: (value: boolean) => void;
+}) => {
+  const { chatMessage, displayPlayer, setDisplayPlayer } = props;
+  const { recording } = chatMessage;
+
+  if (!recording?.src) return null;
+
+  if (displayPlayer) {
+    return (
+      <>
+        <WavesurferPlayer
+          id={recording.id}
+          src={recording.src}
+          autoplay={true}
+        />
+        {recording?.pronunciationAssessment && (
+          <div className="flex justify-end">
+            <PronunciationAssessmentScoreDetail
+              assessment={recording.pronunciationAssessment}
+            />
+          </div>
+        )}
+      </>
+    );
+  }
+
+  return (
+    <Button
+      onClick={() => setDisplayPlayer(true)}
+      className="w-8 h-8"
+      variant="ghost"
+      size="icon"
+    >
+      <Volume2Icon className="w-5 h-5" />
+    </Button>
+  );
+};
+
 const ChatUserMessageActions = (props: {
   chatMessage: ChatMessageType;
   setContent: (content: string) => void;
diff --git a/enjoy/src/renderer/hooks/use-transcribe.tsx b/enjoy/src/renderer/hooks/use-transcribe.tsx
index 1213be6f..4a541745 100644
--- a/enjoy/src/renderer/hooks/use-transcribe.tsx
+++ b/enjoy/src/renderer/hooks/use-transcribe.tsx
@@ -106,6 +106,7 @@ export const useTranscribe = () => {
       return {
         ...result,
         timeline: [],
+        url,
       };
     }
 
@@ -128,6 +129,7 @@ export const useTranscribe = () => {
       return {
         ...result,
         timeline,
+        url,
       };
     } else if (transcript) {
       setOutput("Aligning the transcript...");
@@ -154,6 +156,7 @@ export const useTranscribe = () => {
       return {
         ...result,
         timeline,
+        url,
       };
     } else {
       throw new Error(t("transcribeFailed"));