From 6399022e5e88fd66a4502af0632d8baaefce3984 Mon Sep 17 00:00:00 2001
From: an-lee <an.lee.work@gmail.com>
Date: Tue, 23 Jul 2024 16:28:24 +0800
Subject: [PATCH] display caption in video (#873)

---
 .../components/medias/media-provider.tsx      | 46 +++++++++++++++++--
 .../transcription-edit-button.tsx             |  4 +-
 enjoy/src/renderer/context/theme-provider.tsx |  7 +--
 enjoy/src/utils.ts                            |  2 +-
 4 files changed, 48 insertions(+), 11 deletions(-)

diff --git a/enjoy/src/renderer/components/medias/media-provider.tsx b/enjoy/src/renderer/components/medias/media-provider.tsx
index 98144b2d..f10ef6f8 100644
--- a/enjoy/src/renderer/components/medias/media-provider.tsx
+++ b/enjoy/src/renderer/components/medias/media-provider.tsx
@@ -1,29 +1,64 @@
-import { useContext } from "react";
-import { MediaPlayerProviderContext } from "@renderer/context";
+import { useContext, useEffect, useRef } from "react";
+import {
+  MediaPlayerProviderContext,
+  ThemeProviderContext,
+} from "@renderer/context";
 import {
   MediaPlayer as VidstackMediaPlayer,
   MediaProvider as VidstackMediaProvider,
   isAudioProvider,
   isVideoProvider,
   useMediaRemote,
+  TextTrack,
+  MediaPlayerInstance,
 } from "@vidstack/react";
 import {
   DefaultAudioLayout,
+  DefaultVideoLayout,
   defaultLayoutIcons,
 } from "@vidstack/react/player/layouts/default";
+import { TimelineEntry } from "echogarden/dist/utilities/Timeline.d.js";
+import { milisecondsToTimestamp } from "@/utils";
 
 export const MediaProvider = () => {
-  const { media, setMediaProvider, setDecodeError } = useContext(
+  const { theme } = useContext(ThemeProviderContext);
+  const { media, setMediaProvider, setDecodeError, transcription } = useContext(
     MediaPlayerProviderContext
   );
   const mediaRemote = useMediaRemote();
+  const player = useRef<MediaPlayerInstance>(null);
+
+  useEffect(() => {
+    if (!transcription?.result) return;
+    if (!player?.current) return;
+
+    const srt = transcription.result.timeline
+      .map(
+        (t: TimelineEntry) =>
+          `1\n${milisecondsToTimestamp(
+            t.startTime * 1000
+          )} --> ${milisecondsToTimestamp(t.endTime * 1000)}\n${t.text}`
+      )
+      .join("\n\n");
+
+    player.current.textTracks.clear();
+    player.current.textTracks.add(
+      new TextTrack({
+        content: srt,
+        kind: "subtitles",
+        type: "srt",
+        language: transcription.result.language,
+      })
+    );
+  }, [player, transcription]);
+
   if (!media?.src) return null;
 
   return (
     <div className="px-2 py-4" data-testid="media-player">
       <VidstackMediaPlayer
+        ref={player}
         className="my-auto"
-        controls
         src={media.src}
         onCanPlayThrough={(detail, nativeEvent) => {
           mediaRemote.setTarget(nativeEvent.target);
@@ -37,7 +72,8 @@ export const MediaProvider = () => {
         onError={(err) => setDecodeError(err.message)}
       >
         <VidstackMediaProvider />
-        <DefaultAudioLayout icons={defaultLayoutIcons} />
+        <DefaultAudioLayout icons={defaultLayoutIcons} colorScheme={theme} />
+        <DefaultVideoLayout icons={defaultLayoutIcons} colorScheme={theme} />
       </VidstackMediaPlayer>
     </div>
   );
diff --git a/enjoy/src/renderer/components/transcriptions/transcription-edit-button.tsx b/enjoy/src/renderer/components/transcriptions/transcription-edit-button.tsx
index c2dc029b..f7e5353f 100644
--- a/enjoy/src/renderer/components/transcriptions/transcription-edit-button.tsx
+++ b/enjoy/src/renderer/components/transcriptions/transcription-edit-button.tsx
@@ -38,8 +38,8 @@ export const TranscriptionEditButton = (props: {
     // generate text in SRT format from timeline entries
     transcription.result.timeline
       .map(
-        (t: TimelineEntry) =>
-          `${milisecondsToTimestamp(
+        (t: TimelineEntry, index: number) =>
+          `${index + 1}\n${milisecondsToTimestamp(
             t.startTime * 1000
           )} --> ${milisecondsToTimestamp(t.endTime * 1000)}\n${t.text}`
       )
diff --git a/enjoy/src/renderer/context/theme-provider.tsx b/enjoy/src/renderer/context/theme-provider.tsx
index ebc2dcfa..62933dca 100644
--- a/enjoy/src/renderer/context/theme-provider.tsx
+++ b/enjoy/src/renderer/context/theme-provider.tsx
@@ -10,7 +10,7 @@ type ThemeProviderProps = {
 
 type ThemeProviderState = {
   theme: Theme;
-  colorScheme?: Omit<Theme, 'system'>;
+  colorScheme?: Omit<Theme, "system">;
   setTheme: (theme: Theme) => void;
 };
 
@@ -19,7 +19,8 @@ const initialState: ThemeProviderState = {
   setTheme: () => null,
 };
 
-const ThemeProviderContext = createContext<ThemeProviderState>(initialState);
+export const ThemeProviderContext =
+  createContext<ThemeProviderState>(initialState);
 
 export function ThemeProvider({
   children,
@@ -30,7 +31,7 @@ export function ThemeProvider({
   const [theme, setTheme] = useState<Theme>(
     () => (localStorage.getItem(storageKey) as Theme) || defaultTheme
   );
-  const [colorScheme, setColorScheme] = useState<Omit<Theme, 'system'>>();
+  const [colorScheme, setColorScheme] = useState<Omit<Theme, "system">>();
 
   useEffect(() => {
     const root = window.document.documentElement;
diff --git a/enjoy/src/utils.ts b/enjoy/src/utils.ts
index dfac4d7d..c9aa3348 100644
--- a/enjoy/src/utils.ts
+++ b/enjoy/src/utils.ts
@@ -49,7 +49,7 @@ export function milisecondsToTimestamp(ms: number) {
   const hours = Math.floor(ms / 3600000).toString();
   const minutes = Math.floor((ms % 3600000) / 60000).toString();
   const seconds = Math.floor(((ms % 360000) % 60000) / 1000).toString();
-  const milliseconds = Math.round(((ms % 360000) % 60000) % 1000).toString();
+  const milliseconds = Math.floor(ms % 1000).toString();
   return `${hours.padStart(2, "0")}:${minutes.padStart(
     2,
     "0"