Feat: improve player caption (#433)

* update ipa mapping

* refactor player caption layout
This commit is contained in:
an-lee
2024-03-22 10:20:13 +08:00
committed by GitHub
parent d2a777a72b
commit 03d081592c
10 changed files with 149 additions and 134 deletions

View File

@@ -406,7 +406,7 @@ export const IPA_MAPPING: { [key: string]: string } = {
ʐ: "z",
ç: "",
ʝ: "j",
x: "h",
x: "k",
ɣ: "g",
χ: "h",
ʁ: "r",
@@ -420,7 +420,7 @@ export const IPA_MAPPING: { [key: string]: string } = {
ʈʃ: "tʃ",
: "dʒ",
ʋ: "v",
ɹ: "ɹ",
ɹ: "r",
ɻ: "r",
j: "j",
ɰ: "w",
@@ -437,40 +437,40 @@ export const IPA_MAPPING: { [key: string]: string } = {
ɪ: "ɪ",
ʏ: "ɪ",
ʊ: "ʊ",
ɨ: "ɪ",
: "ɪ",
ɨ: "i",
: "i:",
e: "e",
ø: "e",
ɘ: "ə",
ɵ: "ə",
ɤ: "ɒ",
ɤ: "ɑː",
o: "o",
ə: "ə",
oː: "oː",
ɛ: "ɛ",
ɛ: "e",
œ: "æ",
ɜ: "ɜ",
ɜ: "ɝ",
ɞ: "əː",
ʌ: "ʌ",
ɔ: "ɔ",
ɜː: "əː",
ɜː: "ɝː",
uː: "uː",
ɔː: "ɔː",
ɛː: "ɛ:",
ɛː: "e:",
æ: "æ",
a: "ɑ",
ɶ: "ɑ",
ɐ: "ə",
ɑ: "ɑ",
ɒ: "ɒ",
ɒ: "ɑː",
ɑː: "ɑː",
"◌˞": "",
ɚ: "ɚ",
ɝ: "ɝ",
ɹ̩: "r",
eɪ: "eɪ",
əʊ: "əʊ",
: "əʊ",
əʊ: "oʊ",
: "oʊ",
aɪ: "aɪ",
ɔɪ: "ɔɪ",
: "aʊ",
@@ -478,17 +478,17 @@ export const IPA_MAPPING: { [key: string]: string } = {
ɜr: "ɜr",
ɑr: "ɑr",
ɔr: "ɔr",
oʊr: "əʊr",
oːɹ: "ɔːɹ",
oʊr: "oʊr",
oːɹ: "ɔːr",
ir: "ir",
ɪɹ: "ɪɹ",
ɔːɹ: "ɔːɹ",
ɑːɹ: "ɑːɹ",
ʊɹ: ɹ",
ʊr: ɹ",
ɛr: "ɛr",
ɛɹ: "ɛɹ",
ɪɹ: "ɪr",
ɔːɹ: "ɔːr",
ɑːɹ: "ɑːr",
ʊɹ: r",
ʊr: r",
ɛr: "er",
ɛɹ: "er",
əl: "ə",
aɪɚ: "aɪ",
aɪə: "aɪ",
aɪə: "aɪə",
};

View File

@@ -510,5 +510,6 @@
"translateSetence": "translate setenece",
"reTranslate": "re-translate",
"analyzeSetence": "analyze setenece",
"useAIAssistantToAnalyze": "Use AI assistant to analyze",
"reAnalyze": "re-analyze"
}

View File

@@ -509,5 +509,6 @@
"translateSetence": "整句翻译",
"reTranslate": "重新翻译",
"analyzeSetence": "分析句子",
"useAIAssistantToAnalyze": "使用智能助手分析",
"reAnalyze": "重新分析"
}

View File

@@ -24,7 +24,7 @@ export const AudioPlayer = (props: { id?: string; md5?: string }) => {
return (
<div data-testid="audio-player">
<div className="h-[calc(100vh-37.5rem)] mb-4">
<div className="grid grid-cols-3 gap-4 px-6 h-full">
<div className="grid grid-cols-3 gap-6 px-6 h-full">
<div className="col-span-1 rounded-lg border shadow-lg h-[calc(100vh-37.5rem)]">
<MediaTabs />
</div>

View File

@@ -24,6 +24,7 @@ import { useNavigate } from "react-router-dom";
export const ConversationShortcuts = (props: {
trigger: React.ReactNode;
title?: string;
open?: boolean;
onOpenChange?: (open: boolean) => void;
prompt: string;
@@ -32,6 +33,7 @@ export const ConversationShortcuts = (props: {
}) => {
const { EnjoyApp } = useContext(AppSettingsProviderContext);
const {
title,
prompt,
onReply,
excludedIds = [],
@@ -196,7 +198,7 @@ export const ConversationShortcuts = (props: {
<DialogTrigger asChild>{trigger}</DialogTrigger>
<DialogContent>
<DialogHeader>
<DialogTitle>{t("sendToAIAssistant")}</DialogTitle>
<DialogTitle>{title || t("sendToAIAssistant")}</DialogTitle>
</DialogHeader>
{dialogContent()}
</DialogContent>

View File

@@ -282,8 +282,8 @@ export const MediaCaption = () => {
return (
<div className="h-full flex justify-between space-x-4">
<ScrollArea className="flex-1 px-6 py-4 font-serif h-full border shadow-lg rounded-lg">
<div className="flex flex-wrap mb-4">
<ScrollArea className="flex-1 font-serif h-full border shadow-lg rounded-lg">
<div className="flex flex-wrap px-4 py-2">
{/* use the words splitted by caption text if it is matched with the timeline length, otherwise use the timeline */}
{caption.text.split(" ").length !== caption.timeline.length
? (caption.timeline || []).map((w, index) => (
@@ -300,10 +300,12 @@ export const MediaCaption = () => {
onClick={() => toggleRegion(index)}
>
<div className="">
<div className="text-2xl">{w.text}</div>
<div className="text-lg xl:text-xl 2xl:text-2xl">
{w.text}
</div>
{displayIpa && (
<div
className={`text-muted-foreground font-code ${
className={`text-sm 2xl:text-base text-muted-foreground font-code ${
index === 0 ? "before:content-['/']" : ""
}
${
@@ -334,10 +336,12 @@ export const MediaCaption = () => {
onClick={() => toggleRegion(index)}
>
<div className="">
<div className="text-2xl">{word}</div>
<div className="text-lg xl:text-xl 2xl:text-2xl">
{word}
</div>
{displayIpa && (
<div
className={`text-muted-foreground font-code ${
className={`text-sm 2xl:text-base text-muted-foreground font-code ${
index === 0 ? "before:content-['/']" : ""
}
${
@@ -371,18 +375,23 @@ export const MediaCaption = () => {
variant={displayIpa ? "secondary" : "outline"}
size="icon"
className="rounded-full w-8 h-8 p-0"
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("displayIpa")}
onClick={() => setDisplayIpa(!displayIpa)}
>
<SpeechIcon className="w-4 h-4" />
</Button>
<AIButton
prompt={caption.text as string}
tooltip={t("sendToAIAssistant")}
/>
<Button
variant="outline"
size="icon"
className="rounded-full w-8 h-8 p-0"
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("copyText")}
onClick={() => {
copyToClipboard(caption.text);
@@ -396,7 +405,7 @@ export const MediaCaption = () => {
<CheckIcon className="w-4 h-4 text-green-500" />
) : (
<CopyIcon
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("copyText")}
className="w-4 h-4"
/>
@@ -549,12 +558,8 @@ const CaptionTabs = (props: {
}, [caption]);
return (
<Tabs
value={tab}
onValueChange={(value) => setTab(value)}
className="border rounded-lg"
>
<TabsList className="grid grid-cols-4 gap-4 rounded-b-none sticky top-0">
<Tabs value={tab} onValueChange={(value) => setTab(value)} className="">
<TabsList className="grid grid-cols-4 gap-4 rounded-none sticky top-0 px-4 mb-4">
<TabsTrigger value="selected">{t("captionTabs.selected")}</TabsTrigger>
<TabsTrigger value="translation">
{t("captionTabs.translation")}
@@ -563,7 +568,7 @@ const CaptionTabs = (props: {
<TabsTrigger value="note">{t("captionTabs.note")}</TabsTrigger>
</TabsList>
<div className="px-4 pb-2 min-h-32">
<div className="px-4 pb-4 min-h-32">
<TabsContent value="selected">
{selectedIndices.length > 0 ? (
<>
@@ -631,7 +636,7 @@ const CaptionTabs = (props: {
</div>
</>
) : (
<div className="text-muted-foreground py-4">
<div className="text-sm text-muted-foreground py-4">
{t("clickAnyWordToSelect")}
</div>
)}
@@ -640,7 +645,11 @@ const CaptionTabs = (props: {
<TabsContent value="translation">
{translation ? (
<>
<div className="mb-2 flex items-center justify-end">
<Markdown className="select-text prose prose-sm prose-h3:text-base max-w-full mb-4">
{translation}
</Markdown>
<div className="flex items-center justify-end">
<Button
variant="secondary"
size="sm"
@@ -653,9 +662,6 @@ const CaptionTabs = (props: {
{t("reTranslate")}
</Button>
</div>
<div className="select-text text-sm text-foreground">
{translation}
</div>
</>
) : (
<div className="flex items-center justify-center space-x-2 py-4">
@@ -676,7 +682,24 @@ const CaptionTabs = (props: {
<TabsContent value="analysis">
{analysisResult ? (
<>
<div className="mb-2 flex items-center space-x-2 justify-end">
<Markdown
className="select-text prose prose-sm prose-h3:text-base max-w-full mb-4"
components={{
a({ node, children, ...props }) {
try {
new URL(props.href ?? "");
props.target = "_blank";
props.rel = "noopener noreferrer";
} catch (e) {}
return <a {...props}>{children}</a>;
},
}}
>
{analysisResult}
</Markdown>
<div className="flex items-center space-x-2 justify-end">
<Button
variant="secondary"
size="sm"
@@ -695,24 +718,9 @@ const CaptionTabs = (props: {
setAnalysisResult(result);
EnjoyApp.cacheObjects.set(`analyze-${hash}`, result);
}}
tooltip={t("useAIAssistantToAnalyze")}
/>
</div>
<Markdown
className="select-text prose prose-sm prose-h3:text-base max-w-full"
components={{
a({ node, children, ...props }) {
try {
new URL(props.href ?? "");
props.target = "_blank";
props.rel = "noopener noreferrer";
} catch (e) {}
return <a {...props}>{children}</a>;
},
}}
>
{analysisResult}
</Markdown>
</>
) : (
<div className="flex items-center justify-center space-x-2 py-4">
@@ -729,6 +737,7 @@ const CaptionTabs = (props: {
setAnalysisResult(result);
EnjoyApp.cacheObjects.set(`analyze-${hash}`, result);
}}
tooltip={t("useAIAssistantToAnalyze")}
/>
</div>
)}
@@ -746,9 +755,10 @@ const CaptionTabs = (props: {
const AIButton = (props: {
prompt: string;
onReply: (replies: MessageType[]) => void;
onReply?: (replies: MessageType[]) => void;
tooltip: string;
}) => {
const { prompt, onReply } = props;
const { prompt, onReply, tooltip } = props;
const [asking, setAsking] = useState<boolean>(false);
return (
<ConversationShortcuts
@@ -756,15 +766,16 @@ const AIButton = (props: {
onOpenChange={setAsking}
prompt={prompt}
onReply={onReply}
title={tooltip}
trigger={
<Button
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-content={t("sendToAIAssistant")}
data-tooltip-id="media-player-tooltip"
data-tooltip-content={tooltip}
variant="outline"
size="sm"
className="p-0 w-8 h-8 rounded-full"
>
<BotIcon className="w-5 h-5 text-muted-foreground hover:text-primary" />
<BotIcon className="w-5 h-5" />
</Button>
}
/>

View File

@@ -448,7 +448,7 @@ export const MediaCurrentRecording = (props: { height?: number }) => {
variant="default"
size="icon"
id="recording-play-or-pause-button"
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("playRecording")}
className="rounded-full w-8 h-8 p-0"
onClick={() => {
@@ -478,7 +478,7 @@ export const MediaCurrentRecording = (props: { height?: number }) => {
<Button
variant={isComparing ? "secondary" : "outline"}
size="icon"
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("compare")}
className="rounded-full w-8 h-8 p-0"
onClick={toggleCompare}
@@ -489,7 +489,7 @@ export const MediaCurrentRecording = (props: { height?: number }) => {
<Button
variant={isSelectingRegion ? "secondary" : "outline"}
size="icon"
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("selectRegion")}
className="rounded-full w-8 h-8 p-0"
onClick={() => setIsSelectingRegion(!isSelectingRegion)}
@@ -502,7 +502,7 @@ export const MediaCurrentRecording = (props: { height?: number }) => {
<Button
variant="outline"
size="icon"
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("more")}
className="rounded-full w-8 h-8 p-0"
>
@@ -599,7 +599,7 @@ export const MediaRecordButton = (props: {
variant="ghost"
onClick={() => setIsRecording(!isRecording)}
id="media-record-button"
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={
isRecording ? t("stopRecording") : t("startRecording")
}

View File

@@ -26,11 +26,9 @@ import {
SkipBackIcon,
SaveIcon,
UndoIcon,
TextCursorInputIcon,
GroupIcon,
} from "lucide-react";
import { t } from "i18next";
import { Tooltip } from "react-tooltip";
import { useHotkeys } from "react-hotkeys-hook";
import cloneDeep from "lodash/cloneDeep";
import debounce from "lodash/debounce";
@@ -464,7 +462,7 @@ export const MediaPlayerControls = () => {
<PopoverTrigger asChild>
<Button
variant={`${playbackRate == 1.0 ? "ghost" : "secondary"}`}
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("playbackSpeed")}
className="relative aspect-square p-0 h-10"
>
@@ -502,7 +500,7 @@ export const MediaPlayerControls = () => {
<DropdownMenuTrigger asChild>
<Button
variant="ghost"
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("switchPlayMode")}
className="aspect-square p-0 h-10"
>
@@ -541,7 +539,7 @@ export const MediaPlayerControls = () => {
size="lg"
onClick={onPrev}
id="media-play-previous-button"
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("playPreviousSegment")}
className="aspect-square p-0 h-10"
>
@@ -553,7 +551,7 @@ export const MediaPlayerControls = () => {
variant="default"
onClick={debouncedPlayOrPause}
id="media-play-or-pause-button"
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("pause")}
className="aspect-square p-0 h-12 rounded-full"
>
@@ -564,7 +562,7 @@ export const MediaPlayerControls = () => {
variant="default"
onClick={debouncedPlayOrPause}
id="media-play-or-pause-button"
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("play")}
className="aspect-square p-0 h-12 rounded-full"
>
@@ -577,7 +575,7 @@ export const MediaPlayerControls = () => {
size="lg"
onClick={onNext}
id="media-play-next-button"
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("playNextSegment")}
className="aspect-square p-0 h-10"
>
@@ -587,7 +585,7 @@ export const MediaPlayerControls = () => {
<Button
variant={grouping ? "secondary" : "ghost"}
size="icon"
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("autoGroup")}
className="relative aspect-square p-0 h-10"
onClick={() => setGrouping(!grouping)}
@@ -598,7 +596,7 @@ export const MediaPlayerControls = () => {
<div className="relative">
<Button
variant={`${editingRegion ? "secondary" : "ghost"}`}
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={
editingRegion ? t("dragRegionBorderToEdit") : t("editRegion")
}
@@ -615,7 +613,7 @@ export const MediaPlayerControls = () => {
<Button
variant="secondary"
className="relative aspect-square p-0 h-10"
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("cancel")}
onClick={() => {
setEditingRegion(false);
@@ -627,7 +625,7 @@ export const MediaPlayerControls = () => {
<Button
variant="default"
className="relative aspect-square p-0 h-10"
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("save")}
onClick={() => {
if (!transcriptionDraft) return;
@@ -648,8 +646,6 @@ export const MediaPlayerControls = () => {
)}
</div>
</div>
<Tooltip className="z-10" id="media-player-controls-tooltip" />
</div>
);
};

View File

@@ -158,7 +158,7 @@ export const MediaPlayer = () => {
<div className="flex flex-col justify-around space-y-1.5">
<Button
variant={`${zoomRatio === fitZoomRatio ? "secondary" : "outline"}`}
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("zoomToFit")}
className="relative aspect-square rounded-full p-0 h-8"
onClick={() => {
@@ -174,7 +174,7 @@ export const MediaPlayer = () => {
<Button
variant={`${zoomRatio > 1.0 ? "secondary" : "outline"}`}
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("zoomIn")}
className="relative aspect-square rounded-full p-0 h-8"
onClick={() => {
@@ -191,7 +191,7 @@ export const MediaPlayer = () => {
<Button
variant={`${zoomRatio < 1.0 ? "secondary" : "outline"}`}
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("zoomOut")}
className="relative aspect-square rounded-full p-0 h-8"
onClick={() => {
@@ -208,7 +208,7 @@ export const MediaPlayer = () => {
<Button
variant={`${displayInlineCaption ? "secondary" : "outline"}`}
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("inlineCaption")}
className="relative aspect-square rounded-full p-0 h-8"
onClick={() => {
@@ -227,7 +227,7 @@ export const MediaPlayer = () => {
<Button
variant="outline"
size="icon"
data-tooltip-id="media-player-controls-tooltip"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("more")}
className="rounded-full w-8 h-8 p-0"
>

View File

@@ -10,6 +10,7 @@ import Chart from "chart.js/auto";
import { TimelineEntry } from "echogarden/dist/utilities/Timeline.d.js";
import { IPA_MAPPING } from "@/constants";
import { toast } from "@renderer/components/ui";
import { Tooltip } from "react-tooltip";
type MediaPlayerContextType = {
media: AudioType | VideoType;
@@ -437,48 +438,51 @@ export const MediaPlayerProvider = ({
}, [media, ref, mediaProvider]);
return (
<MediaPlayerProviderContext.Provider
value={{
media,
setMedia,
setMediaProvider,
wavesurfer,
setRef,
decoded,
decodeError,
setDecodeError,
currentTime,
currentSegmentIndex,
setCurrentSegmentIndex,
waveform,
zoomRatio,
setZoomRatio,
fitZoomRatio,
minPxPerSec,
transcription,
regions,
renderPitchContour,
pitchChart,
activeRegion,
setActiveRegion,
editingRegion,
setEditingRegion,
generateTranscription,
transcribing,
transcribingProgress,
transcriptionDraft,
setTranscriptionDraft,
isRecording,
setIsRecording,
currentRecording,
setCurrentRecording,
recordings,
fetchRecordings,
loadingRecordings,
hasMoreRecordings,
}}
>
{children}
</MediaPlayerProviderContext.Provider>
<>
<MediaPlayerProviderContext.Provider
value={{
media,
setMedia,
setMediaProvider,
wavesurfer,
setRef,
decoded,
decodeError,
setDecodeError,
currentTime,
currentSegmentIndex,
setCurrentSegmentIndex,
waveform,
zoomRatio,
setZoomRatio,
fitZoomRatio,
minPxPerSec,
transcription,
regions,
renderPitchContour,
pitchChart,
activeRegion,
setActiveRegion,
editingRegion,
setEditingRegion,
generateTranscription,
transcribing,
transcribingProgress,
transcriptionDraft,
setTranscriptionDraft,
isRecording,
setIsRecording,
currentRecording,
setCurrentRecording,
recordings,
fetchRecordings,
loadingRecordings,
hasMoreRecordings,
}}
>
{children}
</MediaPlayerProviderContext.Provider>
<Tooltip className="z-10" id="media-player-tooltip" />
</>
);
};