Feat assessment for vocabulary (#1228)

* add placeholder * record & assess for word * auto stop * fix warning * improve style * show in shadow player * improve ui
2024-12-07 09:40:08 +08:00
parent cf71d48ae5
commit 10e0f3bd06
7 changed files with 336 additions and 72 deletions
--- a/enjoy/src/main/db/handlers/recordings-handler.ts
+++ b/enjoy/src/main/db/handlers/recordings-handler.ts
@@ -48,9 +48,11 @@ class RecordingsHandler {
      });
  }

-  private async findOne(event: IpcMainEvent, where: WhereOptions<Recording>) {
+  private async findOne(_event: IpcMainEvent, where: WhereOptions<Recording>) {
    return Recording.scope("withoutDeleted")
      .findOne({
+        include: PronunciationAssessment,
+        order: [["createdAt", "DESC"]],
        where: {
          ...where,
        },
@@ -65,12 +67,6 @@ class RecordingsHandler {
        }

        return recording.toJSON();
-      })
-      .catch((err) => {
-        event.sender.send("on-notification", {
-          type: "error",
-          message: err.message,
-        });
      });
  }

--- a/enjoy/src/renderer/components/medias/media-bottom-panel/media-current-recording.tsx
+++ b/enjoy/src/renderer/components/medias/media-bottom-panel/media-current-recording.tsx
@@ -491,6 +491,7 @@ export const MediaCurrentRecording = () => {
      icon: MediaRecordButton,
      active: isRecording,
      onClick: () => {},
+      asChild: true,
    },
    {
      id: "recording-play-or-pause-button",
@@ -509,6 +510,7 @@ export const MediaCurrentRecording = () => {
          player?.playPause();
        }
      },
+      asChild: false,
    },
    {
      id: "media-pronunciation-assessment-button",
@@ -524,6 +526,7 @@ export const MediaCurrentRecording = () => {
          : "text-red-500"
        : "",
      onClick: () => setDetailIsOpen(!detailIsOpen),
+      asChild: false,
    },
    {
      id: "media-compare-button",
@@ -532,6 +535,7 @@ export const MediaCurrentRecording = () => {
      icon: GitCompareIcon,
      active: isComparing,
      onClick: toggleCompare,
+      asChild: false,
    },
    {
      id: "media-select-region-button",
@@ -540,6 +544,7 @@ export const MediaCurrentRecording = () => {
      icon: TextCursorInputIcon,
      active: isSelectingRegion,
      onClick: () => setIsSelectingRegion(!isSelectingRegion),
+      asChild: false,
    },
    {
      id: "media-share-button",
@@ -548,6 +553,7 @@ export const MediaCurrentRecording = () => {
      icon: Share2Icon,
      active: isSharing,
      onClick: () => setIsSharing(true),
+      asChild: false,
    },
    {
      id: "media-download-button",
@@ -556,6 +562,7 @@ export const MediaCurrentRecording = () => {
      icon: DownloadIcon,
      active: false,
      onClick: handleDownload,
+      asChild: false,
    },
  ];

@@ -625,6 +632,7 @@ export const MediaCurrentRecording = () => {
            data-tooltip-content={action.label}
            className="relative p-0 w-full h-full rounded-none"
            onClick={action.onClick}
+            asChild={action.asChild}
          >
            <action.icon className={`w-4 h-4 ${cn(action.iconClassName)}`} />
          </Button>
--- a/enjoy/src/renderer/components/medias/media-right-panel/media-caption-translation.tsx
+++ b/enjoy/src/renderer/components/medias/media-right-panel/media-caption-translation.tsx
@@ -14,6 +14,7 @@ import {
  AiLookupResult,
  TranslateResult,
  DictSelect,
+  VocabularyPronunciationAssessment,
 } from "@renderer/components";

 /*
@@ -109,6 +110,8 @@ const SelectedWords = (props: {
        </div>
      </div>

+      <VocabularyPronunciationAssessment word={word} />
+
      <Separator className="my-4" />

      <div className="rounded-lg overflow-hidden mr-10">
--- a/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-card.tsx
+++ b/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-card.tsx
@@ -7,9 +7,9 @@ import {
  RadialProgress,
  Badge,
 } from "@renderer/components/ui";
-import { scoreColor } from "./pronunciation-assessment-score-result";
+import { scoreColor } from "@renderer/components";
 import { t } from "i18next";
-import { formatDateTime } from "@/renderer/lib/utils";
+import { formatDateTime } from "@renderer/lib/utils";
 import { MoreHorizontalIcon, Trash2Icon } from "lucide-react";
 import { Link } from "react-router-dom";

@@ -108,8 +108,26 @@ export const PronunciationAssessmentCard = (props: {

 export const PronunciationAssessmentScoreDetail = (props: {
  assessment: PronunciationAssessmentType;
+  pronunciationScore?: boolean;
+  accuracyScore?: boolean;
+  fluencyScore?: boolean;
+  completenessScore?: boolean;
+  prosodyScore?: boolean;
+  grammarScore?: boolean;
+  vocabularyScore?: boolean;
+  topicScore?: boolean;
 }) => {
-  const { assessment } = props;
+  const {
+    assessment,
+    pronunciationScore = true,
+    accuracyScore = true,
+    fluencyScore = true,
+    completenessScore = true,
+    prosodyScore = true,
+    grammarScore = true,
+    vocabularyScore = true,
+    topicScore = true,
+  } = props;

  return (
    <div className="flex items-center gap-2 flex-wrap">
@@ -117,39 +135,47 @@ export const PronunciationAssessmentScoreDetail = (props: {
        {
          label: t("models.pronunciationAssessment.pronunciationScore"),
          value: assessment.pronunciationScore,
+          show: pronunciationScore,
        },
        {
          label: t("models.pronunciationAssessment.accuracyScore"),
          value: assessment.accuracyScore,
+          show: accuracyScore,
        },
        {
          label: t("models.pronunciationAssessment.fluencyScore"),
          value: assessment.fluencyScore,
+          show: fluencyScore,
        },
        {
          label: t("models.pronunciationAssessment.completenessScore"),
          value: assessment.completenessScore,
+          show: completenessScore,
        },
        {
          label: t("models.pronunciationAssessment.prosodyScore"),
          value: assessment.prosodyScore,
+          show: prosodyScore,
        },
        {
          label: t("models.pronunciationAssessment.grammarScore"),
          value: assessment.grammarScore,
+          show: grammarScore,
        },
        {
          label: t("models.pronunciationAssessment.vocabularyScore"),
          value: assessment.vocabularyScore,
+          show: vocabularyScore,
        },
        {
          label: t("models.pronunciationAssessment.topicScore"),
          value: assessment.topicScore,
+          show: topicScore,
        },
-      ].map(({ label, value }) => {
-        if (typeof value === "number") {
+      ].map(({ label, value, show }) => {
+        if (show && typeof value === "number") {
          return (
-            <div key={label} className="flex items-center space-x-2 mb-2">
+            <div key={label} className="flex items-center gap-2">
              <span className="text-muted-foreground text-sm">{label}:</span>
              <span className={`text-sm font-bold ${scoreColor(value || 0)}`}>
                {value}
--- a/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-form.tsx
+++ b/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-form.tsx
@@ -20,7 +20,7 @@ import {
 } from "@renderer/components/ui";
 import { t } from "i18next";
 import { useNavigate } from "react-router-dom";
-import { useContext, useEffect, useRef, useState } from "react";
+import { useContext, useEffect, useState } from "react";
 import { AppSettingsProviderContext } from "@/renderer/context";
 import { LANGUAGES } from "@/constants";
 import { z } from "zod";
--- a/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-word-result.tsx
+++ b/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-word-result.tsx
@@ -4,6 +4,8 @@ import {
  PopoverTrigger,
  PopoverContent,
  Button,
+  ScrollArea,
+  ScrollBar,
 } from "@renderer/components/ui";
 import { Volume2Icon } from "lucide-react";
 import { useEffect, useRef } from "react";
@@ -139,36 +141,13 @@ export const PronunciationAssessmentWordResult = (props: {
      </PopoverTrigger>

      <PopoverContent className="bg-muted">
-        {result.phonemes.length > 0 ? (
-          <>
-            <div className="text-sm flex items-center space-x-2 mb-2">
-              <span className="font-serif">{t("score")}:</span>
-              <span className="font-serif">
-                {result.pronunciationAssessment.accuracyScore}
-              </span>
-            </div>
-            <div className="flex items-center space-x-2 flex-wrap">
-              {result.phonemes.map((phoneme, index) => (
-                <div key={index} className="text-sm text-center">
-                  <div className="font-bold">{phoneme.phoneme}</div>
-                  <div
-                    className={`text-sm font-serif ${scoreColor(
-                      phoneme.pronunciationAssessment.accuracyScore
-                    )}`}
-                  >
-                    {phoneme.pronunciationAssessment.accuracyScore}
-                  </div>
-                </div>
-              ))}
-            </div>
-          </>
-        ) : (
-          <div>
-            {t(
-              `models.pronunciationAssessment.errors.${result.pronunciationAssessment.errorType.toLowerCase()}`
-            )}
-          </div>
-        )}
+        <div className="text-sm flex items-center space-x-2 mb-2">
+          <span className="font-serif">{t("score")}:</span>
+          <span className="font-serif">
+            {result.pronunciationAssessment.accuracyScore}
+          </span>
+        </div>
+        <PronunciationAssessmentPhonemeResult result={result} />

        <div className="flex items-center space-x-2">
          <span className="text-sm">{t("myPronunciation")}:</span>
@@ -239,3 +218,30 @@ const scoreColor = (score: number) => {

  return "font-bold text-red-600";
 };
+
+export const PronunciationAssessmentPhonemeResult = (props: {
+  result: PronunciationAssessmentWordResultType;
+}) => {
+  const { result } = props;
+  console.log(result);
+
+  return (
+    <ScrollArea className="w-full">
+      <div className="w-full flex items-center gap-2">
+        {result.phonemes.map((phoneme, index) => (
+          <div key={index} className="text-sm text-center">
+            <div className="font-bold font-code">{phoneme.phoneme}</div>
+            <div
+              className={`text-xs font-serif ${scoreColor(
+                phoneme.pronunciationAssessment.accuracyScore
+              )}`}
+            >
+              {phoneme.pronunciationAssessment.accuracyScore}
+            </div>
+          </div>
+        ))}
+      </div>
+      <ScrollBar orientation="horizontal" />
+    </ScrollArea>
+  );
+};
--- a/enjoy/src/renderer/components/widgets/lookup/lookup-widget.tsx
+++ b/enjoy/src/renderer/components/widgets/lookup/lookup-widget.tsx
@@ -1,22 +1,43 @@
-import { useEffect, useContext, useState } from "react";
+import { useEffect, useContext, useState, useRef } from "react";
 import {
  AppSettingsProviderContext,
  DictProviderContext,
 } from "@renderer/context";
 import {
  Button,
+  Collapsible,
+  CollapsibleContent,
+  CollapsibleTrigger,
  Popover,
  PopoverAnchor,
  PopoverContent,
  ScrollArea,
+  toast,
 } from "@renderer/components/ui";
 import {
  DictLookupResult,
  DictSelect,
  AiLookupResult,
  CamdictLookupResult,
+  scoreColor,
+  PronunciationAssessmentScoreDetail,
+  PronunciationAssessmentFulltextResult,
+  PronunciationAssessmentScoreResult,
+  PronunciationAssessmentPhonemeResult,
 } from "@renderer/components";
-import { ChevronLeft, ChevronFirst } from "lucide-react";
+import {
+  ChevronLeft,
+  ChevronFirst,
+  SpeakerIcon,
+  MicIcon,
+  SquareIcon,
+  LoaderIcon,
+  Volume2Icon,
+  CheckIcon,
+} from "lucide-react";
+import { useAudioRecorder } from "react-audio-voice-recorder";
+import { t } from "i18next";
+import { usePronunciationAssessments } from "@/renderer/hooks";

 export const LookupWidget = () => {
  const { EnjoyApp } = useContext(AppSettingsProviderContext);
@@ -107,7 +128,7 @@ export const LookupWidget = () => {
          top: selected?.position?.y,
          left: selected?.position?.x,
        }}
-      ></PopoverAnchor>
+      />
      <PopoverContent
        className="w-full p-0 z-50"
        updatePositionStrategy="always"
@@ -115,33 +136,36 @@ export const LookupWidget = () => {
        {selected?.word && (
          <ScrollArea>
            <div className="w-96 h-96 flex flex-col">
-              <div className="p-2 border-b flex justify-between items-center">
-                <div className="flex items-center">
-                  {history.length > 1 && (
-                    <div className="mr-1 flex items-center">
-                      <Button
-                        variant="ghost"
-                        className="w-6 h-6 p-0"
-                        onClick={handleViewFirst}
-                      >
-                        <ChevronFirst />
-                      </Button>
-
-                      <Button
-                        variant="ghost"
-                        className="w-6 h-6 p-0"
-                        onClick={handleViewLast}
-                      >
-                        <ChevronLeft />
-                      </Button>
-                    </div>
-                  )}
-
-                  <div className="font-bold">{current}</div>
+              <div className="p-2 border-b space-y-2">
+                <div className="flex justify-between items-center">
+                  <div className="flex items-center">
+                    {history.length > 1 && (
+                      <div className="mr-1 flex items-center">
+                        <Button
+                          variant="ghost"
+                          className="w-6 h-6 p-0"
+                          onClick={handleViewFirst}
+                        >
+                          <ChevronFirst />
+                        </Button>
+                        <Button
+                          variant="ghost"
+                          className="w-6 h-6 p-0"
+                          onClick={handleViewLast}
+                        >
+                          <ChevronLeft />
+                        </Button>
+                      </div>
+                    )}
+                    <div className="font-bold">{current}</div>
+                  </div>
+                  <div className="w-40">
+                    <DictSelect />
+                  </div>
                </div>

-                <div className="w-40">
-                  <DictSelect />
+                <div className="">
+                  <VocabularyPronunciationAssessment word={current} />
                </div>
              </div>
              <div className="p-2 pr-1 flex-1">
@@ -165,3 +189,204 @@ export const LookupWidget = () => {
    </Popover>
  );
 };
+
+LookupWidget.displayName = "LookupWidget";
+
+export const VocabularyPronunciationAssessment = (props: { word: string }) => {
+  const { word } = props;
+  const { EnjoyApp, recorderConfig, learningLanguage } = useContext(
+    AppSettingsProviderContext
+  );
+  const {
+    startRecording,
+    stopRecording,
+    recordingBlob,
+    isRecording,
+    recordingTime,
+  } = useAudioRecorder(recorderConfig, (exception) => {
+    toast.error(exception.message);
+  });
+  const { createAssessment } = usePronunciationAssessments();
+  const [access, setAccess] = useState(true);
+  const [submitting, setSubmitting] = useState(false);
+  const [recording, setRecording] = useState<RecordingType>();
+  const [assessment, setAssessment] = useState<PronunciationAssessmentType>();
+  const [open, setOpen] = useState(false);
+  const audio = useRef<HTMLAudioElement>(null);
+
+  const askForMediaAccess = () => {
+    EnjoyApp.system.preferences.mediaAccess("microphone").then((access) => {
+      if (access) {
+        setAccess(true);
+      } else {
+        setAccess(false);
+        toast.warning(t("noMicrophoneAccess"));
+      }
+    });
+  };
+
+  const findRecording = () => {
+    EnjoyApp.recordings
+      .findOne({
+        referenceText: word,
+      })
+      .then((recording) => {
+        if (recording?.pronunciationAssessment) {
+          setRecording(recording);
+          setAssessment(recording.pronunciationAssessment);
+        }
+      });
+  };
+
+  const onRecorded = async (blob: Blob) => {
+    console.log(blob);
+
+    if (!blob) return;
+
+    let recording: RecordingType;
+    try {
+      recording = await EnjoyApp.recordings.create({
+        language: learningLanguage,
+        blob: {
+          type: recordingBlob.type.split(";")[0],
+          arrayBuffer: await blob.arrayBuffer(),
+        },
+        referenceText: word,
+      });
+    } catch (err) {
+      toast.error(err.message);
+    }
+    if (!recording) return;
+
+    setSubmitting(true);
+    createAssessment({
+      language: learningLanguage,
+      reference: word,
+      recording,
+    })
+      .then((assessment) => {
+        setAssessment(assessment);
+        setRecording(recording);
+        setOpen(true);
+      })
+      .catch((err) => {
+        toast.error(err.message);
+        EnjoyApp.recordings.destroy(recording.id);
+      })
+      .finally(() => setSubmitting(false));
+  };
+
+  useEffect(() => {
+    askForMediaAccess();
+    findRecording();
+  }, [word]);
+
+  useEffect(() => {
+    if (recording) {
+      console.log(recording);
+      audio.current = new Audio(recording.src);
+    }
+
+    return () => {
+      if (audio.current) {
+        audio.current.pause();
+        audio.current = null;
+      }
+    };
+  }, [recording]);
+
+  useEffect(() => {
+    if (recordingBlob) {
+      onRecorded(recordingBlob);
+    }
+  }, [recordingBlob]);
+
+  /**
+   * Auto stop recording after 5 seconds
+   */
+  useEffect(() => {
+    if (!isRecording) return;
+
+    if (recordingTime >= 5) {
+      stopRecording();
+    }
+  }, [recordingTime]);
+
+  if (!word) return null;
+  if (!access) return null;
+
+  if (submitting) {
+    return (
+      <Button variant="ghost" className="size-6 p-0" disabled>
+        <LoaderIcon className="size-4 animate-spin" />
+      </Button>
+    );
+  }
+
+  if (isRecording) {
+    return (
+      <Button
+        variant="ghost"
+        className="rounded-full size-6 p-0 bg-red-500 hover:bg-red-500/90"
+        onClick={stopRecording}
+      >
+        <SquareIcon fill="white" className="size-3 text-white" />
+      </Button>
+    );
+  }
+
+  return (
+    <Collapsible open={open} onOpenChange={setOpen}>
+      <div className="flex items-center gap-1">
+        <Button
+          variant="ghost"
+          className="rounded-full size-6 p-0 bg-red-500 hover:bg-red-500/90"
+          onClick={startRecording}
+        >
+          <MicIcon className="size-4 text-white" />
+        </Button>
+        {recording && (
+          <Button
+            variant="ghost"
+            size="icon"
+            className="rounded-full p-0 size-6 border border-secondary"
+            onClick={() => audio.current?.play()}
+          >
+            <Volume2Icon className="size-4" />
+          </Button>
+        )}
+        {assessment && (
+          <CollapsibleTrigger asChild>
+            <Button
+              variant={open ? "secondary" : "ghost"}
+              size="icon"
+              className="rounded-full p-0 size-6 border border-secondary"
+            >
+              <span
+                className={`text-xs ${scoreColor(
+                  assessment.pronunciationScore || 0
+                )}`}
+              >
+                {assessment.pronunciationScore.toFixed(0)}
+              </span>
+            </Button>
+          </CollapsibleTrigger>
+        )}
+      </div>
+      {assessment && (
+        <CollapsibleContent className="mt-2">
+          <div className="space-y-2">
+            <PronunciationAssessmentPhonemeResult
+              result={assessment.result.words[0]}
+            />
+            <PronunciationAssessmentScoreDetail
+              assessment={assessment}
+              fluencyScore={false}
+              completenessScore={false}
+            />
+          </div>
+        </CollapsibleContent>
+      )}
+    </Collapsible>
+  );
+};