Files
everyone-can-use-english/enjoy/src/renderer/components/conversations/speech-player.tsx
an-lee 90f38e9226 Transcription force alignment & more (#416)
* add wavesurfer-provider

* brand new layout for player

* refactor pitch contour

* clean up

* update styl

* refactor

* update layout

* use new layout for video

* refactor

* may select word

* may edit word timestamp

* may toggle multiselect words

* clean code

* improve word region update

* improve layout

* update layout

* add echogarden

* fix test

* use aligned transcription

* fix ipa

* some refactor

* improve code

* implement ipa & translate & lookup

* recording play & share

* fix

* fix post audio

* improve layout

* may delete recording

* may record

* fix video player layout

* fix player in conversation

* render recording along with orignal audio

* may custom create region in recording

* fix float issue when seekTo

* fix recording player

* fix load more recordings

* fix seekTo

* clean up

* refactor pitch contour

* fix some warnings

* upgrade deps

* fix group transcription sentence

* zoom to fit when segment update

* add more hotkeys

* update player layout

* improve style

* play recording overlap audio when comparing

* update echogarden dep

* add recorded mark on transcription

* fix recording pitch contour rendering

* improve recording

* adjust pitch finder params
2024-03-16 19:42:37 +08:00

133 lines
4.0 KiB
TypeScript

import { useEffect, useState, useRef, useCallback } from "react";
import { renderPitchContour } from "@renderer/lib/utils";
import { extractFrequencies } from "@/utils";
import WaveSurfer from "wavesurfer.js";
import { Button, Skeleton } from "@renderer/components/ui";
import { PlayIcon, PauseIcon } from "lucide-react";
import { useIntersectionObserver } from "@uidotdev/usehooks";
import { secondsToTimestamp } from "@renderer/lib/utils";
export const SpeechPlayer = (props: {
speech: Partial<SpeechType>;
height?: number;
}) => {
const { speech, height = 100 } = props;
const [isPlaying, setIsPlaying] = useState(false);
const [wavesurfer, setWavesurfer] = useState(null);
const containerRef = useRef();
const [ref, entry] = useIntersectionObserver({
threshold: 1,
});
const [duration, setDuration] = useState<number>(0);
const [initialized, setInitialized] = useState(false);
const onPlayClick = useCallback(() => {
wavesurfer.isPlaying() ? wavesurfer.pause() : wavesurfer.play();
}, [wavesurfer]);
useEffect(() => {
// use the intersection observer to only create the wavesurfer instance
// when the player is visible
if (!entry?.isIntersecting) return;
if (!speech?.src) return;
if (wavesurfer) return;
const ws = WaveSurfer.create({
container: containerRef.current,
url: speech.src,
height,
barWidth: 1,
cursorWidth: 0,
autoCenter: false,
autoScroll: true,
hideScrollbar: true,
minPxPerSec: 100,
waveColor: "#ddd",
progressColor: "rgba(0, 0, 0, 0.25)",
normalize: true,
});
setWavesurfer(ws);
}, [speech, entry]);
useEffect(() => {
if (!wavesurfer) return;
const subscriptions = [
wavesurfer.on("play", () => {
setIsPlaying(true);
}),
wavesurfer.on("pause", () => {
setIsPlaying(false);
}),
wavesurfer.on("ready", () => {
setDuration(wavesurfer.getDuration());
const peaks = wavesurfer.getDecodedData().getChannelData(0);
const sampleRate = wavesurfer.options.sampleRate;
const data = extractFrequencies({ peaks, sampleRate });
setTimeout(() => {
renderPitchContour({
wrapper: wavesurfer.getWrapper(),
canvasId: `pitch-contour-${speech.id}-canvas`,
labels: new Array(data.length).fill(""),
datasets: [
{
data,
cubicInterpolationMode: "monotone",
pointRadius: 1,
},
],
});
}, 1000);
setInitialized(true);
}),
];
return () => {
subscriptions.forEach((unsub) => unsub());
wavesurfer?.destroy();
};
}, [wavesurfer]);
return (
<div className="w-full">
<div className="flex justify-end">
<span className="text-xs text-muted-foreground mb-1">
{secondsToTimestamp(duration)}
</span>
</div>
<div
ref={ref}
className="bg-background rounded-lg grid grid-cols-9 items-center relative pl-2 h-[100px]"
>
{!initialized && (
<div className="col-span-9 flex flex-col justify-around h-[80px]">
<Skeleton className="h-3 w-full rounded-full" />
<Skeleton className="h-3 w-full rounded-full" />
<Skeleton className="h-3 w-full rounded-full" />
</div>
)}
<div className={`flex justify-center ${initialized ? "" : "hidden"}`}>
<Button
onClick={onPlayClick}
className="aspect-square rounded-full p-2 w-12 h-12 bg-blue-600 hover:bg-blue-500"
>
{isPlaying ? (
<PauseIcon className="w-6 h-6 text-white" />
) : (
<PlayIcon className="w-6 h-6 text-white" />
)}
</Button>
</div>
<div
data-testid="wavesurfer-container"
className={`col-span-8 ${initialized ? "" : "hidden"}`}
ref={containerRef}
></div>
</div>
</div>
);
};