From 10e0f3bd06b8facb926f540047f7663dfaace0f2 Mon Sep 17 00:00:00 2001 From: an-lee Date: Sat, 7 Dec 2024 09:40:08 +0800 Subject: [PATCH] Feat assessment for vocabulary (#1228) * add placeholder * record & assess for word * auto stop * fix warning * improve style * show in shadow player * improve ui --- .../main/db/handlers/recordings-handler.ts | 10 +- .../media-current-recording.tsx | 8 + .../media-caption-translation.tsx | 3 + .../pronunciation-assessment-card.tsx | 38 ++- .../pronunciation-assessment-form.tsx | 2 +- .../pronunciation-assessment-word-result.tsx | 66 ++-- .../widgets/lookup/lookup-widget.tsx | 281 ++++++++++++++++-- 7 files changed, 336 insertions(+), 72 deletions(-) diff --git a/enjoy/src/main/db/handlers/recordings-handler.ts b/enjoy/src/main/db/handlers/recordings-handler.ts index 19348cbc..60351977 100644 --- a/enjoy/src/main/db/handlers/recordings-handler.ts +++ b/enjoy/src/main/db/handlers/recordings-handler.ts @@ -48,9 +48,11 @@ class RecordingsHandler { }); } - private async findOne(event: IpcMainEvent, where: WhereOptions) { + private async findOne(_event: IpcMainEvent, where: WhereOptions) { return Recording.scope("withoutDeleted") .findOne({ + include: PronunciationAssessment, + order: [["createdAt", "DESC"]], where: { ...where, }, @@ -65,12 +67,6 @@ class RecordingsHandler { } return recording.toJSON(); - }) - .catch((err) => { - event.sender.send("on-notification", { - type: "error", - message: err.message, - }); }); } diff --git a/enjoy/src/renderer/components/medias/media-bottom-panel/media-current-recording.tsx b/enjoy/src/renderer/components/medias/media-bottom-panel/media-current-recording.tsx index 41193745..6c4928de 100644 --- a/enjoy/src/renderer/components/medias/media-bottom-panel/media-current-recording.tsx +++ b/enjoy/src/renderer/components/medias/media-bottom-panel/media-current-recording.tsx @@ -491,6 +491,7 @@ export const MediaCurrentRecording = () => { icon: MediaRecordButton, active: isRecording, onClick: () => {}, + asChild: true, }, { id: "recording-play-or-pause-button", @@ -509,6 +510,7 @@ export const MediaCurrentRecording = () => { player?.playPause(); } }, + asChild: false, }, { id: "media-pronunciation-assessment-button", @@ -524,6 +526,7 @@ export const MediaCurrentRecording = () => { : "text-red-500" : "", onClick: () => setDetailIsOpen(!detailIsOpen), + asChild: false, }, { id: "media-compare-button", @@ -532,6 +535,7 @@ export const MediaCurrentRecording = () => { icon: GitCompareIcon, active: isComparing, onClick: toggleCompare, + asChild: false, }, { id: "media-select-region-button", @@ -540,6 +544,7 @@ export const MediaCurrentRecording = () => { icon: TextCursorInputIcon, active: isSelectingRegion, onClick: () => setIsSelectingRegion(!isSelectingRegion), + asChild: false, }, { id: "media-share-button", @@ -548,6 +553,7 @@ export const MediaCurrentRecording = () => { icon: Share2Icon, active: isSharing, onClick: () => setIsSharing(true), + asChild: false, }, { id: "media-download-button", @@ -556,6 +562,7 @@ export const MediaCurrentRecording = () => { icon: DownloadIcon, active: false, onClick: handleDownload, + asChild: false, }, ]; @@ -625,6 +632,7 @@ export const MediaCurrentRecording = () => { data-tooltip-content={action.label} className="relative p-0 w-full h-full rounded-none" onClick={action.onClick} + asChild={action.asChild} > diff --git a/enjoy/src/renderer/components/medias/media-right-panel/media-caption-translation.tsx b/enjoy/src/renderer/components/medias/media-right-panel/media-caption-translation.tsx index 9f1ab26f..b9ccac4f 100644 --- a/enjoy/src/renderer/components/medias/media-right-panel/media-caption-translation.tsx +++ b/enjoy/src/renderer/components/medias/media-right-panel/media-caption-translation.tsx @@ -14,6 +14,7 @@ import { AiLookupResult, TranslateResult, DictSelect, + VocabularyPronunciationAssessment, } from "@renderer/components"; /* @@ -109,6 +110,8 @@ const SelectedWords = (props: { + +
diff --git a/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-card.tsx b/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-card.tsx index bfbbc855..663d6b9c 100644 --- a/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-card.tsx +++ b/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-card.tsx @@ -7,9 +7,9 @@ import { RadialProgress, Badge, } from "@renderer/components/ui"; -import { scoreColor } from "./pronunciation-assessment-score-result"; +import { scoreColor } from "@renderer/components"; import { t } from "i18next"; -import { formatDateTime } from "@/renderer/lib/utils"; +import { formatDateTime } from "@renderer/lib/utils"; import { MoreHorizontalIcon, Trash2Icon } from "lucide-react"; import { Link } from "react-router-dom"; @@ -108,8 +108,26 @@ export const PronunciationAssessmentCard = (props: { export const PronunciationAssessmentScoreDetail = (props: { assessment: PronunciationAssessmentType; + pronunciationScore?: boolean; + accuracyScore?: boolean; + fluencyScore?: boolean; + completenessScore?: boolean; + prosodyScore?: boolean; + grammarScore?: boolean; + vocabularyScore?: boolean; + topicScore?: boolean; }) => { - const { assessment } = props; + const { + assessment, + pronunciationScore = true, + accuracyScore = true, + fluencyScore = true, + completenessScore = true, + prosodyScore = true, + grammarScore = true, + vocabularyScore = true, + topicScore = true, + } = props; return (
@@ -117,39 +135,47 @@ export const PronunciationAssessmentScoreDetail = (props: { { label: t("models.pronunciationAssessment.pronunciationScore"), value: assessment.pronunciationScore, + show: pronunciationScore, }, { label: t("models.pronunciationAssessment.accuracyScore"), value: assessment.accuracyScore, + show: accuracyScore, }, { label: t("models.pronunciationAssessment.fluencyScore"), value: assessment.fluencyScore, + show: fluencyScore, }, { label: t("models.pronunciationAssessment.completenessScore"), value: assessment.completenessScore, + show: completenessScore, }, { label: t("models.pronunciationAssessment.prosodyScore"), value: assessment.prosodyScore, + show: prosodyScore, }, { label: t("models.pronunciationAssessment.grammarScore"), value: assessment.grammarScore, + show: grammarScore, }, { label: t("models.pronunciationAssessment.vocabularyScore"), value: assessment.vocabularyScore, + show: vocabularyScore, }, { label: t("models.pronunciationAssessment.topicScore"), value: assessment.topicScore, + show: topicScore, }, - ].map(({ label, value }) => { - if (typeof value === "number") { + ].map(({ label, value, show }) => { + if (show && typeof value === "number") { return ( -
+
{label}: {value} diff --git a/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-form.tsx b/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-form.tsx index 993b0774..83153a5e 100644 --- a/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-form.tsx +++ b/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-form.tsx @@ -20,7 +20,7 @@ import { } from "@renderer/components/ui"; import { t } from "i18next"; import { useNavigate } from "react-router-dom"; -import { useContext, useEffect, useRef, useState } from "react"; +import { useContext, useEffect, useState } from "react"; import { AppSettingsProviderContext } from "@/renderer/context"; import { LANGUAGES } from "@/constants"; import { z } from "zod"; diff --git a/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-word-result.tsx b/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-word-result.tsx index fa58c92e..905726c1 100644 --- a/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-word-result.tsx +++ b/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-word-result.tsx @@ -4,6 +4,8 @@ import { PopoverTrigger, PopoverContent, Button, + ScrollArea, + ScrollBar, } from "@renderer/components/ui"; import { Volume2Icon } from "lucide-react"; import { useEffect, useRef } from "react"; @@ -139,36 +141,13 @@ export const PronunciationAssessmentWordResult = (props: { - {result.phonemes.length > 0 ? ( - <> -
- {t("score")}: - - {result.pronunciationAssessment.accuracyScore} - -
-
- {result.phonemes.map((phoneme, index) => ( -
-
{phoneme.phoneme}
-
- {phoneme.pronunciationAssessment.accuracyScore} -
-
- ))} -
- - ) : ( -
- {t( - `models.pronunciationAssessment.errors.${result.pronunciationAssessment.errorType.toLowerCase()}` - )} -
- )} +
+ {t("score")}: + + {result.pronunciationAssessment.accuracyScore} + +
+
{t("myPronunciation")}: @@ -239,3 +218,30 @@ const scoreColor = (score: number) => { return "font-bold text-red-600"; }; + +export const PronunciationAssessmentPhonemeResult = (props: { + result: PronunciationAssessmentWordResultType; +}) => { + const { result } = props; + console.log(result); + + return ( + +
+ {result.phonemes.map((phoneme, index) => ( +
+
{phoneme.phoneme}
+
+ {phoneme.pronunciationAssessment.accuracyScore} +
+
+ ))} +
+ + + ); +}; diff --git a/enjoy/src/renderer/components/widgets/lookup/lookup-widget.tsx b/enjoy/src/renderer/components/widgets/lookup/lookup-widget.tsx index b8b733ef..0fd0e9b4 100644 --- a/enjoy/src/renderer/components/widgets/lookup/lookup-widget.tsx +++ b/enjoy/src/renderer/components/widgets/lookup/lookup-widget.tsx @@ -1,22 +1,43 @@ -import { useEffect, useContext, useState } from "react"; +import { useEffect, useContext, useState, useRef } from "react"; import { AppSettingsProviderContext, DictProviderContext, } from "@renderer/context"; import { Button, + Collapsible, + CollapsibleContent, + CollapsibleTrigger, Popover, PopoverAnchor, PopoverContent, ScrollArea, + toast, } from "@renderer/components/ui"; import { DictLookupResult, DictSelect, AiLookupResult, CamdictLookupResult, + scoreColor, + PronunciationAssessmentScoreDetail, + PronunciationAssessmentFulltextResult, + PronunciationAssessmentScoreResult, + PronunciationAssessmentPhonemeResult, } from "@renderer/components"; -import { ChevronLeft, ChevronFirst } from "lucide-react"; +import { + ChevronLeft, + ChevronFirst, + SpeakerIcon, + MicIcon, + SquareIcon, + LoaderIcon, + Volume2Icon, + CheckIcon, +} from "lucide-react"; +import { useAudioRecorder } from "react-audio-voice-recorder"; +import { t } from "i18next"; +import { usePronunciationAssessments } from "@/renderer/hooks"; export const LookupWidget = () => { const { EnjoyApp } = useContext(AppSettingsProviderContext); @@ -107,7 +128,7 @@ export const LookupWidget = () => { top: selected?.position?.y, left: selected?.position?.x, }} - > + /> { {selected?.word && (
-
-
- {history.length > 1 && ( -
- - - -
- )} - -
{current}
+
+
+
+ {history.length > 1 && ( +
+ + +
+ )} +
{current}
+
+
+ +
-
- +
+
@@ -165,3 +189,204 @@ export const LookupWidget = () => { ); }; + +LookupWidget.displayName = "LookupWidget"; + +export const VocabularyPronunciationAssessment = (props: { word: string }) => { + const { word } = props; + const { EnjoyApp, recorderConfig, learningLanguage } = useContext( + AppSettingsProviderContext + ); + const { + startRecording, + stopRecording, + recordingBlob, + isRecording, + recordingTime, + } = useAudioRecorder(recorderConfig, (exception) => { + toast.error(exception.message); + }); + const { createAssessment } = usePronunciationAssessments(); + const [access, setAccess] = useState(true); + const [submitting, setSubmitting] = useState(false); + const [recording, setRecording] = useState(); + const [assessment, setAssessment] = useState(); + const [open, setOpen] = useState(false); + const audio = useRef(null); + + const askForMediaAccess = () => { + EnjoyApp.system.preferences.mediaAccess("microphone").then((access) => { + if (access) { + setAccess(true); + } else { + setAccess(false); + toast.warning(t("noMicrophoneAccess")); + } + }); + }; + + const findRecording = () => { + EnjoyApp.recordings + .findOne({ + referenceText: word, + }) + .then((recording) => { + if (recording?.pronunciationAssessment) { + setRecording(recording); + setAssessment(recording.pronunciationAssessment); + } + }); + }; + + const onRecorded = async (blob: Blob) => { + console.log(blob); + + if (!blob) return; + + let recording: RecordingType; + try { + recording = await EnjoyApp.recordings.create({ + language: learningLanguage, + blob: { + type: recordingBlob.type.split(";")[0], + arrayBuffer: await blob.arrayBuffer(), + }, + referenceText: word, + }); + } catch (err) { + toast.error(err.message); + } + if (!recording) return; + + setSubmitting(true); + createAssessment({ + language: learningLanguage, + reference: word, + recording, + }) + .then((assessment) => { + setAssessment(assessment); + setRecording(recording); + setOpen(true); + }) + .catch((err) => { + toast.error(err.message); + EnjoyApp.recordings.destroy(recording.id); + }) + .finally(() => setSubmitting(false)); + }; + + useEffect(() => { + askForMediaAccess(); + findRecording(); + }, [word]); + + useEffect(() => { + if (recording) { + console.log(recording); + audio.current = new Audio(recording.src); + } + + return () => { + if (audio.current) { + audio.current.pause(); + audio.current = null; + } + }; + }, [recording]); + + useEffect(() => { + if (recordingBlob) { + onRecorded(recordingBlob); + } + }, [recordingBlob]); + + /** + * Auto stop recording after 5 seconds + */ + useEffect(() => { + if (!isRecording) return; + + if (recordingTime >= 5) { + stopRecording(); + } + }, [recordingTime]); + + if (!word) return null; + if (!access) return null; + + if (submitting) { + return ( + + ); + } + + if (isRecording) { + return ( + + ); + } + + return ( + +
+ + {recording && ( + + )} + {assessment && ( + + + + )} +
+ {assessment && ( + +
+ + +
+ + )} + + ); +};