Feat: make notes on caption (#544)

* add segment model

* add note model

* db handle segment & note

* add notes & segments handler

* refactor media caption components

* segment & note create

* fix type

* update note column & may sync

* display selected words for note

* refactor selected words

* auto select words when editing note

* refactor

* refactor caption component

* display notes

* refactor notes components

* fix

* refactor segment & notes into context

* destroy note

* update locale

* fix caption switch issue

* fix layout

* refactor caption layout

* remove deprecated code

* may share note

* improve UI

* fix notes list auto update after created

* remove console.log

* add notes page

* refactor note parameters

* refactor components

* mark note on transcription

* handle no notes

* improve style

* improve style

* show context menu on selection text

* fix utils

enjoy/src/renderer/components/posts/index.ts

@@ -1,9 +1,9 @@
 export * from "./posts";
+export * from "./post-actions";
 export * from "./post-audio";
 export * from "./post-card";
 export * from "./post-medium";
-export * from "./post-recording";
-export * from "./post-story";
-
+export * from "./post-note";
 export * from "./post-options";
-export * from "./post-actions";
+export * from "./post-recording";
+export * from "./post-story";

enjoy/src/renderer/components/posts/post-audio.tsx

@@ -1,12 +1,6 @@
-import { useEffect, useState, useRef, useCallback, useContext } from "react";
+import { useEffect, useState, useContext } from "react";
 import { AppSettingsProviderContext } from "@renderer/context";
-import { renderPitchContour } from "@renderer/lib/utils";
-import { extractFrequencies } from "@/utils";
-import WaveSurfer from "wavesurfer.js";
-import { Button, Skeleton } from "@renderer/components/ui";
-import { PlayIcon, PauseIcon } from "lucide-react";
-import { useIntersectionObserver } from "@uidotdev/usehooks";
-import { secondsToTimestamp } from "@renderer/lib/utils";
+import { Button } from "@renderer/components/ui";
 import { MediaPlayer, MediaProvider } from "@vidstack/react";
 import {
   DefaultAudioLayout,
@@ -16,6 +10,7 @@ import { STORAGE_WORKER_ENDPOINT } from "@/constants";
 import { TimelineEntry } from "echogarden/dist/utilities/Timeline.d.js";
 import { t } from "i18next";
 import { XCircleIcon } from "lucide-react";
+import { WavesurferPlayer } from "../widgets";
 
 export const PostAudio = (props: {
   audio: Partial<MediumType>;
@@ -44,6 +39,8 @@ export const PostAudio = (props: {
         targetMd5: audio.md5,
       })
       .then((response) => {
+        const transcription = response?.transcriptions?.[0];
+        if (transcription.targetMd5 !== audio.md5) return;
         setTranscription(response?.transcriptions?.[0]);
       });
   }, [audio.md5]);
@@ -70,7 +67,8 @@ export const PostAudio = (props: {
         <WavesurferPlayer
           currentTime={currentTime}
           setCurrentTime={setCurrentTime}
-          audio={audio}
+          id={audio.id}
+          src={audio.sourceUrl}
           height={height}
           onError={(err) => setError(err.message)}
         />
@@ -103,136 +101,3 @@ export const PostAudio = (props: {
     </div>
   );
 };
-
-const WavesurferPlayer = (props: {
-  audio: Partial<MediumType>;
-  height?: number;
-  currentTime: number;
-  setCurrentTime: (currentTime: number) => void;
-  onError?: (error: Error) => void;
-}) => {
-  const { audio, height = 80, onError, setCurrentTime } = props;
-  const [initialized, setInitialized] = useState(false);
-  const [isPlaying, setIsPlaying] = useState(false);
-  const [wavesurfer, setWavesurfer] = useState(null);
-  const containerRef = useRef();
-  const [ref, entry] = useIntersectionObserver({
-    threshold: 1,
-  });
-  const [duration, setDuration] = useState<number>(0);
-
-  const onPlayClick = useCallback(() => {
-    wavesurfer.isPlaying() ? wavesurfer.pause() : wavesurfer.play();
-  }, [wavesurfer]);
-
-  useEffect(() => {
-    // use the intersection observer to only create the wavesurfer instance
-    // when the player is visible
-    if (!entry?.isIntersecting) return;
-    if (!audio.sourceUrl) return;
-    if (wavesurfer) return;
-
-    const ws = WaveSurfer.create({
-      container: containerRef.current,
-      url: audio.sourceUrl,
-      height,
-      barWidth: 1,
-      cursorWidth: 0,
-      autoCenter: true,
-      autoScroll: true,
-      dragToSeek: true,
-      hideScrollbar: true,
-      minPxPerSec: 100,
-      waveColor: "#ddd",
-      progressColor: "rgba(0, 0, 0, 0.25)",
-    });
-
-    setWavesurfer(ws);
-  }, [audio.sourceUrl, entry]);
-
-  useEffect(() => {
-    if (!wavesurfer) return;
-
-    const subscriptions = [
-      wavesurfer.on("play", () => {
-        setIsPlaying(true);
-      }),
-      wavesurfer.on("pause", () => {
-        setIsPlaying(false);
-      }),
-      wavesurfer.on("timeupdate", (time: number) => {
-        setCurrentTime(time);
-      }),
-      wavesurfer.on("ready", () => {
-        setDuration(wavesurfer.getDuration());
-        const peaks = wavesurfer.getDecodedData().getChannelData(0);
-        const sampleRate = wavesurfer.options.sampleRate;
-        const data = extractFrequencies({ peaks, sampleRate });
-        setTimeout(() => {
-          renderPitchContour({
-            wrapper: wavesurfer.getWrapper(),
-            canvasId: `pitch-contour-${audio.id}-canvas`,
-            labels: new Array(data.length).fill(""),
-            datasets: [
-              {
-                data,
-                cubicInterpolationMode: "monotone",
-                pointRadius: 1,
-              },
-            ],
-          });
-        }, 1000);
-        setInitialized(true);
-      }),
-      wavesurfer.on("error", (err: Error) => {
-        onError(err);
-      }),
-    ];
-
-    return () => {
-      subscriptions.forEach((unsub) => unsub());
-      wavesurfer?.destroy();
-    };
-  }, [wavesurfer]);
-
-  return (
-    <>
-      <div className="flex justify-end">
-        <span className="text-xs text-muted-foreground">
-          {secondsToTimestamp(duration)}
-        </span>
-      </div>
-
-      <div
-        ref={ref}
-        className="bg-background rounded-lg grid grid-cols-9 items-center relative h-[80px]"
-      >
-        {!initialized && (
-          <div className="col-span-9 flex flex-col justify-around h-[80px]">
-            <Skeleton className="h-3 w-full rounded-full" />
-            <Skeleton className="h-3 w-full rounded-full" />
-            <Skeleton className="h-3 w-full rounded-full" />
-          </div>
-        )}
-
-        <div className={`flex justify-center ${initialized ? "" : "hidden"}`}>
-          <Button
-            onClick={onPlayClick}
-            className="aspect-square rounded-full p-2 w-12 h-12 bg-blue-600 hover:bg-blue-500"
-          >
-            {isPlaying ? (
-              <PauseIcon className="w-6 h-6 text-white" />
-            ) : (
-              <PlayIcon className="w-6 h-6 text-white" />
-            )}
-          </Button>
-        </div>
-
-        <div
-          className={`col-span-8 ${initialized ? "" : "hidden"}`}
-          ref={containerRef}
-        ></div>
-      </div>
-    </>
-  );
-};

enjoy/src/renderer/components/posts/post-card.tsx

@@ -6,6 +6,7 @@ import {
   PostMedium,
   PostStory,
   PostOptions,
+  PostNote,
 } from "@renderer/components";
 import { Avatar, AvatarImage, AvatarFallback } from "@renderer/components/ui";
 import { formatDateTime } from "@renderer/lib/utils";
@@ -97,6 +98,15 @@ export const PostCard = (props: {
         </>
       )}
 
+      {post.targetType == "Note" && (
+        <>
+          <div className="text-xs text-muted-foreground">
+            {t("sharedNote")}
+          </div>
+          <PostNote note={post.target as NoteType} />
+        </>
+      )}
+
       <PostActions post={post} />
     </div>
   );

enjoy/src/renderer/components/posts/post-note.tsx

@@ -0,0 +1,20 @@
+import { NoteSemgent } from "@renderer/components";
+
+export const PostNote = (props: { note: NoteType }) => {
+  const { note } = props;
+
+  return (
+    <div className="select-text">
+      <div className="select-text mb-2">{note.content}</div>
+      {note.parameters?.quote && (
+        <div className="mb-2">
+          <span className="text-sm text-muted-foreground select-text border-b border-red-500 border-dashed">
+            {note.parameters.quote}
+          </span>
+        </div>
+      )}
+
+      {note.segment && <NoteSemgent segment={note.segment} notes={[note]} />}
+    </div>
+  );
+};

enjoy/src/renderer/components/posts/posts.tsx

@@ -18,7 +18,7 @@ export const Posts = (props: { userId?: string }) => {
   const { webApi } = useContext(AppSettingsProviderContext);
   const [loading, setLoading] = useState<boolean>(true);
   const [type, setType] = useState<
-    "all" | "recording" | "medium" | "story" | "prompt" | "gpt"
+    "all" | "recording" | "medium" | "story" | "prompt" | "gpt" | "note"
   >("all");
   const [by, setBy] = useState<"all" | "following">("following");
   const [posts, setPosts] = useState<PostType[]>([]);
@@ -109,6 +109,9 @@ export const Posts = (props: { userId?: string }) => {
             <SelectItem key="recording" value="recording">
               {t("recordingType")}
             </SelectItem>
+            <SelectItem key="note" value="note">
+              {t("noteType")}
+            </SelectItem>
             <SelectItem key="prompt" value="prompt">
               {t("promptType")}
             </SelectItem>
@@ -133,10 +136,10 @@ export const Posts = (props: { userId?: string }) => {
 
       <div className="space-y-6">
         {posts.map((post) => (
-          <>
-            <PostCard key={post.id} post={post} handleDelete={handleDelete} />
+          <div key={post.id}>
+            <PostCard post={post} handleDelete={handleDelete} />
             <Separator />
-          </>
+          </div>
         ))}
       </div>