Improve assessment result display (#998)

* update media caption tabs * refactor recording detail * word play in assessment result * update style
2024-08-20 17:43:24 +08:00
parent 47fe325100
commit 3a561f7fda
7 changed files with 102 additions and 241 deletions
--- a/enjoy/src/renderer/components/medias/media-captions/media-caption-tabs.tsx
+++ b/enjoy/src/renderer/components/medias/media-captions/media-caption-tabs.tsx
@@ -26,7 +26,7 @@ export const MediaCaptionTabs = (props: {
    children,
  } = props;

-  const [tab, setTab] = useState<string>("note");
+  const [tab, setTab] = useState<string>("translation");

  if (!caption) return null;

@@ -51,12 +51,12 @@ export const MediaCaptionTabs = (props: {
        </div>

        <TabsList className="grid grid-cols-3 gap-4 rounded-none absolute w-full bottom-0 px-4">
-          <TabsTrigger value="note" className="block truncate px-1">
-            {t("captionTabs.note")}
-          </TabsTrigger>
          <TabsTrigger value="translation" className="block truncate px-1">
            {t("captionTabs.translation")}
          </TabsTrigger>
+          <TabsTrigger value="note" className="block truncate px-1">
+            {t("captionTabs.note")}
+          </TabsTrigger>
          <TabsTrigger value="analysis" className="block truncate px-1">
            {t("captionTabs.analysis")}
          </TabsTrigger>
--- a/enjoy/src/renderer/components/misc/wavesurfer-player.tsx
+++ b/enjoy/src/renderer/components/misc/wavesurfer-player.tsx
@@ -158,7 +158,7 @@ export const WavesurferPlayer = (props: {
  }

  return (
-    <>
+    <div className="w-full max-w-screen-lg">
      <div className="flex justify-end">
        <span className="text-xs text-muted-foreground">
          {secondsToTimestamp(duration)}
@@ -198,6 +198,6 @@ export const WavesurferPlayer = (props: {
          ref={containerRef}
        ></div>
      </div>
-    </>
+    </div>
  );
 };
--- a/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-fulltext-result.tsx
+++ b/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-fulltext-result.tsx
@@ -7,9 +7,9 @@ import { InfoIcon } from "lucide-react";
 export const PronunciationAssessmentFulltextResult = (props: {
  words: PronunciationAssessmentWordResultType[];
  currentTime?: number;
-  onSeek?: (time: number) => void;
+  src?: string;
 }) => {
-  const { words, currentTime, onSeek } = props;
+  const { words, currentTime, src } = props;
  const [errorStats, setErrorStats] = useState({
    mispronunciation: 0,
    omission: 0,
@@ -57,14 +57,14 @@ export const PronunciationAssessmentFulltextResult = (props: {
  return (
    <ScrollArea className="min-h-72 py-4 px-8">
      <div className="flex items-start justify-between space-x-6">
-        <div className="flex-1 py-4">
+        <div className="flex-1 py-4 flex items-center flex-wrap">
          {words.map((result, index: number) => (
            <PronunciationAssessmentWordResult
              key={index}
              result={result}
              errorDisplay={errorDisplay}
              currentTime={currentTime}
-              onSeek={onSeek}
+              src={src}
            />
          ))}
        </div>
--- a/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-word-result.tsx
+++ b/enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-word-result.tsx
@@ -1,12 +1,15 @@
 import { t } from "i18next";
 import {
-  Tooltip,
-  TooltipContent,
-  TooltipTrigger,
-  TooltipProvider,
+  Popover,
+  PopoverTrigger,
+  PopoverContent,
+  Button,
 } from "@renderer/components/ui";
+import { Volume2Icon } from "lucide-react";
+import { useEffect, useRef } from "react";

 export const PronunciationAssessmentWordResult = (props: {
+  src?: string;
  result: PronunciationAssessmentWordResultType;
  errorDisplay?: {
    mispronunciation: boolean;
@@ -17,7 +20,6 @@ export const PronunciationAssessmentWordResult = (props: {
    monotone: boolean;
  };
  currentTime?: number;
-  onSeek?: (time: number) => void;
 }) => {
  const {
    result,
@@ -30,9 +32,10 @@ export const PronunciationAssessmentWordResult = (props: {
      monotone: true,
    },
    currentTime = 0,
-    onSeek,
  } = props;

+  const audio = useRef<HTMLAudioElement>(null);
+
  const WordDisplay = {
    None: <CorrectWordDisplay word={result.word} />,
    Mispronunciation: errorDisplay.mispronunciation ? (
@@ -67,71 +70,90 @@ export const PronunciationAssessmentWordResult = (props: {
    ),
  }[result.pronunciationAssessment.errorType];

+  const play = () => {
+    const { offset, duration } = result;
+
+    // create a new audio element and play the segment
+    audio.current.src = `${props.src}#t=${(offset * 1.0) / 1e7},${
+      ((offset + duration) * 1.0) / 1e7
+    }`;
+    audio.current.play();
+  };
+
+  useEffect(() => {
+    if (!audio.current) {
+      audio.current = new Audio();
+    }
+  }, []);
+
  return (
-    <TooltipProvider>
-      <Tooltip delayDuration={500}>
-        <TooltipTrigger>
+    <Popover>
+      <PopoverTrigger asChild>
+        <div className="text-center mb-3 cursor-pointer">
          <div
-            onClick={() => {
-              onSeek && onSeek(result.offset / 1e7);
-            }}
-            className="text-center mb-3"
+            className={`${
+              currentTime * 1e7 >= result.offset &&
+              currentTime * 1e7 < result.offset + result.duration
+                ? "underline"
+                : ""
+            } underline-offset-4`}
          >
-            <div className="mb-1">
+            {WordDisplay}
+          </div>
+          <div className="mb-1">
+            {result.phonemes.map((phoneme, index) => (
+              <span
+                key={index}
+                className={`italic font-code ${scoreColor(
+                  phoneme.pronunciationAssessment.accuracyScore
+                )}`}
+              >
+                {phoneme.phoneme}
+              </span>
+            ))}
+          </div>
+        </div>
+      </PopoverTrigger>
+
+      <PopoverContent className="bg-muted">
+        {result.phonemes.length > 0 ? (
+          <>
+            <div className="text-sm flex items-center space-x-2 mb-2">
+              <span className="font-serif">{t("score")}:</span>
+              <span className="font-serif">
+                {result.pronunciationAssessment.accuracyScore}
+              </span>
+            </div>
+            <div className="flex items-center space-x-2 flex-wrap">
              {result.phonemes.map((phoneme, index) => (
-                <span
-                  key={index}
-                  className={`italic font-code ${scoreColor(
-                    phoneme.pronunciationAssessment.accuracyScore
-                  )}`}
-                >
-                  {phoneme.phoneme}
-                </span>
+                <div key={index} className="text-sm text-center">
+                  <div className="font-bold">{phoneme.phoneme}</div>
+                  <div
+                    className={`text-sm font-serif ${scoreColor(
+                      phoneme.pronunciationAssessment.accuracyScore
+                    )}`}
+                  >
+                    {phoneme.pronunciationAssessment.accuracyScore}
+                  </div>
+                </div>
              ))}
            </div>
-            <div
-              className={`${
-                currentTime * 1e7 >= result.offset &&
-                currentTime * 1e7 < result.offset + result.duration
-                  ? "underline"
-                  : ""
-              } underline-offset-4`}
-            >
-              {WordDisplay}
-            </div>
+          </>
+        ) : (
+          <div>
+            {t(
+              `models.pronunciationAssessment.errors.${result.pronunciationAssessment.errorType.toLowerCase()}`
+            )}
          </div>
-        </TooltipTrigger>
+        )}

-        <TooltipContent>
-          {result.phonemes.length > 0 ? (
-            <>
-              <div className="flex items-center space-x-2 mb-2">
-                <span className="font-serif">{t("score")}:</span>
-                <span className="font-serif">
-                  {result.pronunciationAssessment.accuracyScore}
-                </span>
-              </div>
-              <div className="flex items-center space-x-4">
-                {result.phonemes.map((phoneme, index) => (
-                  <div key={index} className="text-center">
-                    <div className="font-bold">{phoneme.phoneme}</div>
-                    <div className="text-sm font-serif">
-                      {phoneme.pronunciationAssessment.accuracyScore}
-                    </div>
-                  </div>
-                ))}
-              </div>
-            </>
-          ) : (
-            <div>
-              {t(
-                `models.pronunciationAssessment.errors.${result.pronunciationAssessment.errorType.toLowerCase()}`
-              )}
-            </div>
-          )}
-        </TooltipContent>
-      </Tooltip>
-    </TooltipProvider>
+        <div className="">
+          <Button onClick={play} variant="ghost" size="icon">
+            <Volume2Icon className="w-5 h-5" />
+          </Button>
+        </div>
+      </PopoverContent>
+    </Popover>
  );
 };

--- a/enjoy/src/renderer/components/recordings/index.ts
+++ b/enjoy/src/renderer/components/recordings/index.ts
@@ -1,4 +1,3 @@
-export * from "./recording-player";
 export * from "./recording-calendar";
 export * from "./recording-activities";
 export * from "./recording-stats";
--- a/enjoy/src/renderer/components/recordings/recording-detail.tsx
+++ b/enjoy/src/renderer/components/recordings/recording-detail.tsx
@@ -1,7 +1,7 @@
 import {
-  RecordingPlayer,
  PronunciationAssessmentFulltextResult,
  PronunciationAssessmentScoreResult,
+  WavesurferPlayer,
 } from "@renderer/components";
 import { Separator, ScrollArea, toast } from "@renderer/components/ui";
 import { useState, useContext, useEffect } from "react";
@@ -23,11 +23,6 @@ export const RecordingDetail = (props: {
    );
  const { result } = pronunciationAssessment || {};
  const [currentTime, setCurrentTime] = useState<number>(0);
-  const [seek, setSeek] = useState<{
-    seekTo: number;
-    timestamp: number;
-  }>();
-  const [isPlaying, setIsPlaying] = useState(false);

  const { learningLanguage } = useContext(AppSettingsProviderContext);
  const { createAssessment } = usePronunciationAssessments();
@@ -61,13 +56,11 @@ export const RecordingDetail = (props: {

  return (
    <div className="">
-      <div className="mb-6 px-4">
-        <RecordingPlayer
-          recording={recording}
-          isPlaying={isPlaying}
-          setIsPlaying={setIsPlaying}
-          onCurrentTimeChange={(time) => setCurrentTime(time)}
-          seek={seek}
+      <div className="flex justify-center mb-6 px-4">
+        <WavesurferPlayer
+          id={recording.id}
+          src={recording.src}
+          setCurrentTime={setCurrentTime}
        />
      </div>

@@ -77,13 +70,7 @@ export const RecordingDetail = (props: {
        <PronunciationAssessmentFulltextResult
          words={result.words}
          currentTime={currentTime}
-          onSeek={(time) => {
-            setSeek({
-              seekTo: time,
-              timestamp: Date.now(),
-            });
-            setIsPlaying(true);
-          }}
+          src={recording.src}
        />
      ) : (
        <ScrollArea className="min-h-72 py-4 px-8 select-text">
--- a/enjoy/src/renderer/components/recordings/recording-player.tsx
+++ b/enjoy/src/renderer/components/recordings/recording-player.tsx
@@ -1,147 +0,0 @@
-import { useEffect, useState, useRef, useCallback } from "react";
-import WaveSurfer from "wavesurfer.js";
-import { renderPitchContour } from "@renderer/lib/utils";
-import { extractFrequencies } from "@/utils";
-import { Button, Skeleton } from "@renderer/components/ui";
-import { PlayIcon, PauseIcon } from "lucide-react";
-import { useIntersectionObserver } from "@uidotdev/usehooks";
-
-export const RecordingPlayer = (props: {
-  recording: RecordingType;
-  isPlaying: boolean;
-  setIsPlaying: (isPlaying: boolean) => void;
-  id?: string;
-  height?: number;
-  onCurrentTimeChange?: (time: number) => void;
-  seek?: {
-    seekTo: number;
-    timestamp: number;
-  };
-}) => {
-  const {
-    recording,
-    height = 100,
-    onCurrentTimeChange,
-    seek,
-    isPlaying,
-    setIsPlaying,
-  } = props;
-  const [wavesurfer, setWavesurfer] = useState(null);
-  const containerRef = useRef();
-  const [ref, entry] = useIntersectionObserver({
-    threshold: 0,
-  });
-  const [initialized, setInitialized] = useState(false);
-
-  const onPlayClick = useCallback(() => {
-    wavesurfer.playPause();
-  }, [wavesurfer]);
-
-  useEffect(() => {
-    // use the intersection observer to only create the wavesurfer instance
-    // when the player is visible
-    if (!entry?.isIntersecting) return;
-    if (!recording?.src) return;
-    if (wavesurfer) return;
-
-    const ws = WaveSurfer.create({
-      container: containerRef.current,
-      url: recording.src,
-      height,
-      barWidth: 1,
-      cursorWidth: 0,
-      autoCenter: false,
-      autoScroll: true,
-      hideScrollbar: true,
-      minPxPerSec: 100,
-      waveColor: "#ddd",
-      normalize: true,
-      progressColor: "rgba(0, 0, 0, 0.25)",
-    });
-
-    setWavesurfer(ws);
-  }, [recording, entry]);
-
-  useEffect(() => {
-    if (!wavesurfer) return;
-
-    const subscriptions = [
-      wavesurfer.on("play", () => setIsPlaying(true)),
-      wavesurfer.on("pause", () => setIsPlaying(false)),
-      wavesurfer.on("timeupdate", (time: number) => {
-        onCurrentTimeChange?.(time);
-      }),
-      wavesurfer.on("ready", () => {
-        const peaks = wavesurfer.getDecodedData().getChannelData(0);
-        const sampleRate = wavesurfer.options.sampleRate;
-        const data = extractFrequencies({ peaks, sampleRate });
-        setTimeout(() => {
-          renderPitchContour({
-            wrapper: wavesurfer.getWrapper(),
-            canvasId: `pitch-contour-${recording.id}-canvas`,
-            labels: new Array(data.length).fill(""),
-            datasets: [
-              {
-                data,
-                cubicInterpolationMode: "monotone",
-              },
-            ],
-          });
-        }, 1000);
-        setInitialized(true);
-      }),
-    ];
-
-    return () => {
-      subscriptions.forEach((unsub) => unsub());
-      wavesurfer?.destroy();
-    };
-  }, [wavesurfer]);
-
-  useEffect(() => {
-    if (!wavesurfer) return;
-    if (!seek?.seekTo) return;
-
-    wavesurfer.setTime(parseFloat(seek.seekTo.toFixed(6)));
-  }, [seek, wavesurfer]);
-
-  useEffect(() => {
-    if (!wavesurfer) return;
-
-    if (isPlaying) {
-      wavesurfer.play();
-    } else {
-      wavesurfer.pause();
-    }
-  }, [isPlaying, wavesurfer]);
-
-  return (
-    <div ref={ref} className="grid grid-cols-11 xl:grid-cols-12 items-center">
-      {!initialized && (
-        <div className="col-span-9 flex flex-col justify-around h-[80px]">
-          <Skeleton className="h-3 w-full rounded-full" />
-          <Skeleton className="h-3 w-full rounded-full" />
-          <Skeleton className="h-3 w-full rounded-full" />
-        </div>
-      )}
-
-      <div className={`flex justify-center ${initialized ? "" : "hidden"}`}>
-        <Button
-          onClick={onPlayClick}
-          className="aspect-square rounded-full p-2 w-12 h-12 bg-blue-600 hover:bg-blue-500"
-        >
-          {isPlaying ? (
-            <PauseIcon className="w-6 h-6 text-white" />
-          ) : (
-            <PlayIcon className="w-6 h-6 text-white" />
-          )}
-        </Button>
-      </div>
-
-      <div
-        className={`col-span-10 xl:col-span-11 ${initialized ? "" : "hidden"}`}
-        ref={containerRef}
-      ></div>
-    </div>
-  );
-};