Feat: add camdict (#435)

* add camdict db & logic

* refactor camdict

* refactor media caption

* display camdict result

.gitignore

@@ -129,5 +129,8 @@ ffmpeg-core.wasm
 ffmpeg-core.js
 ffmpeg-core.worker.js
 
+# dict
+cam_dict.refined.sqlite
+
 .vitepress/cache/
 /public/jupyter-notebooks/*.mp3

enjoy/package.json

@@ -21,7 +21,8 @@
     "create-migration": "zx ./src/main/db/create-migration.mjs",
     "download-whisper-model": "zx ./scripts/download-whisper-model.mjs",
     "download-ffmpeg-wasm": "zx ./scripts/download-ffmpeg-wasm.mjs",
-    "download": "yarn run download-whisper-model && yarn run download-ffmpeg-wasm"
+    "download-dictionaries": "zx ./scripts/download-dictionaries.mjs",
+    "download": "yarn run download-whisper-model && yarn run download-ffmpeg-wasm && yarn run download-dictionaries"
   },
   "keywords": [],
   "author": {

enjoy/scripts/download-dictionaries.mjs

@@ -0,0 +1,111 @@
+#!/usr/bin/env zx
+
+import axios from "axios";
+import progress from "progress";
+import { createHash } from "crypto";
+
+const dict = "cam_dict.refined.sqlite";
+const sha = "ff97dbd1e6b5357c4a0b42d3c6b1dce0dbc497dc";
+
+const dir = path.join(process.cwd(), "lib/dictionaries");
+
+console.info(chalk.blue(`=> Download dictionary ${dict}`));
+
+fs.ensureDirSync(dir);
+try {
+  if (fs.statSync(path.join(dir, dict)).isFile()) {
+    console.info(chalk.green(`✅ Dict ${dict} already exists`));
+    const hash = await hashFile(path.join(dir, dict), { algo: "sha1" });
+    if (hash === sha) {
+      console.info(chalk.green(`✅ Dict ${dict} valid`));
+      process.exit(0);
+    } else {
+      console.error(
+        chalk.red(`❌ Dict ${dict} not valid, start to redownload`)
+      );
+      fs.removeSync(path.join(dir, dict));
+    }
+  }
+} catch (err) {
+  if (err && err.code !== "ENOENT") {
+    console.error(chalk.red(`❌ Error: ${err}`));
+    process.exit(1);
+  } else {
+    console.info(chalk.blue(`=> Start to download dict ${dict}`));
+  }
+}
+
+const proxyUrl =
+  process.env.HTTPS_PROXY ||
+  process.env.https_proxy ||
+  process.env.HTTP_PROXY ||
+  process.env.http_proxy;
+
+if (proxyUrl) {
+  const { hostname, port, protocol } = new URL(proxyUrl);
+  axios.defaults.proxy = {
+    host: hostname,
+    port: port,
+    protocol: protocol,
+  };
+}
+
+const dictUrlPrefix = "https://enjoy-storage.baizhiheizi.com";
+
+function hashFile(path, options) {
+  const algo = options.algo || "sha1";
+  return new Promise((resolve, reject) => {
+    const hash = createHash(algo);
+    const stream = fs.createReadStream(path);
+    stream.on("error", reject);
+    stream.on("data", (chunk) => hash.update(chunk));
+    stream.on("end", () => resolve(hash.digest("hex")));
+  });
+}
+
+const download = async (url, dest) => {
+  console.info(chalk.blue(`=> Start to download from ${url} to ${dest}`));
+  return axios
+    .get(url, { responseType: "stream" })
+    .then((response) => {
+      const totalLength = response.headers["content-length"];
+
+      const progressBar = new progress(`-> downloading [:bar] :percent :etas`, {
+        width: 40,
+        complete: "=",
+        incomplete: " ",
+        renderThrottle: 1,
+        total: parseInt(totalLength),
+      });
+
+      response.data.on("data", (chunk) => {
+        progressBar.tick(chunk.length);
+      });
+
+      response.data.pipe(fs.createWriteStream(dest)).on("close", async () => {
+        console.info(chalk.green(`✅ Dict ${dict} downloaded successfully`));
+        const hash = await hashFile(path.join(dir, dict), { algo: "sha1" });
+        if (hash === sha) {
+          console.info(chalk.green(`✅ Dict ${dict} valid`));
+          process.exit(0);
+        } else {
+          console.error(
+            chalk.red(
+              `❌ Dict ${dict} not valid, please try again using command \`yarn workspace enjoy download-dictionaries\``
+            )
+          );
+          process.exit(1);
+        }
+      });
+    })
+    .catch((err) => {
+      console.error(
+        chalk.red(
+          `❌ Failed to download ${url}: ${err}.\nPlease try again using command \`yarn workspace enjoy download-dictionaries\``
+        )
+      );
+      process.exit(1);
+    });
+};
+
+await download(`${dictUrlPrefix}/${dict}`, path.join(dir, dict));

enjoy/src/i18n/en.json

@@ -511,5 +511,8 @@
   "reTranslate": "re-translate",
   "analyzeSetence": "analyze setenece",
   "useAIAssistantToAnalyze": "Use AI assistant to analyze",
-  "reAnalyze": "re-analyze"
+  "reAnalyze": "re-analyze",
+  "AiDictionary": "AI dictionary",
+  "AiTranslate": "AI translate",
+  "cambridgeDictionary": "Cambridge dictionary"
 }

enjoy/src/i18n/zh-CN.json

@@ -510,5 +510,8 @@
   "reTranslate": "重新翻译",
   "analyzeSetence": "分析句子",
   "useAIAssistantToAnalyze": "使用智能助手分析",
-  "reAnalyze": "重新分析"
+  "reAnalyze": "重新分析",
+  "AiDictionary": "智能词典",
+  "AiTranslate": "智能翻译",
+  "cambridgeDictionary": "剑桥词典"
 }

enjoy/src/main/camdict.ts

@@ -0,0 +1,77 @@
+import { ipcMain, app } from "electron";
+import path from "path";
+import log from "@main/logger";
+import url from "url";
+import { Sequelize, DataType } from "sequelize-typescript";
+
+const __filename = url.fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const logger = log.scope("camdict");
+
+class Camdict {
+  public dbPath = path.join(
+    __dirname,
+    "lib",
+    "dictionaries",
+    "cam_dict.refined.sqlite"
+  );
+  private sequelize: Sequelize;
+  private db: any;
+
+  async init() {
+    if (this.db) return;
+
+    try {
+      this.sequelize = new Sequelize({
+        dialect: "sqlite",
+        storage: this.dbPath,
+      });
+      this.sequelize.sync();
+      this.sequelize.authenticate();
+      this.db = this.sequelize.define(
+        "Camdict",
+        {
+          id: {
+            type: DataType.INTEGER,
+            primaryKey: true,
+          },
+          oid: {
+            type: DataType.STRING,
+          },
+          word: {
+            type: DataType.STRING,
+          },
+          posItems: {
+            type: DataType.JSON,
+          },
+        },
+        {
+          modelName: "Camdict",
+          tableName: "camdict",
+          underscored: true,
+          timestamps: true,
+        }
+      );
+    } catch (err) {
+      logger.error("Failed to initialize camdict", err);
+    }
+  }
+
+  async lookup(word: string) {
+    await this.init();
+
+    const item = await this.db?.findOne({
+      where: { word: word.trim().toLowerCase() },
+    });
+    return item?.toJSON();
+  }
+
+  registerIpcHandlers() {
+    ipcMain.handle("camdict-lookup", async (_event, word: string) => {
+      return this.lookup(word);
+    });
+  }
+}
+
+export default new Camdict();

enjoy/src/main/window.ts

@@ -22,6 +22,7 @@ import Ffmpeg from "@main/ffmpeg";
 import { Waveform } from "./waveform";
 import url from "url";
 import echogarden from "./echogarden";
+import camdict from "./camdict";
 
 const __filename = url.fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -47,6 +48,8 @@ main.init = () => {
   // Prepare local database
   db.registerIpcHandlers();
 
+  camdict.registerIpcHandlers();
+
   // Prepare Settings
   settings.registerIpcHandlers();
 

enjoy/src/preload.ts

@@ -178,6 +178,11 @@ contextBridge.exposeInMainWorld("__ENJOY_APP__", {
       ipcRenderer.removeAllListeners("db-on-transaction");
     },
   },
+  camdict: {
+    lookup: (word: string) => {
+      return ipcRenderer.invoke("camdict-lookup", word);
+    }
+  },
   audios: {
     findAll: (params: {
       offset: number | undefined;

enjoy/src/renderer/components/meanings/meaning-card.tsx

@@ -32,7 +32,7 @@ export const MeaningCard = (props: {
           </span>
         )}
         {pronunciation && (
-          <span className="text-sm mr-2">/{pronunciation}/</span>
+          <span className="text-sm font-code mr-2">/{pronunciation}/</span>
         )}
         {lemma && lemma !== word && <span className="text-sm">({lemma})</span>}
       </div>

enjoy/src/renderer/components/medias/index.ts

@@ -1,5 +1,6 @@
 export * from "./media-player-controls";
 export * from "./media-caption";
+export * from "./media-caption-tabs";
 export * from "./media-info-panel";
 export * from "./media-recordings";
 export * from "./media-current-recording";

enjoy/src/renderer/components/medias/media-caption-tabs.tsx

@@ -0,0 +1,504 @@
+import { useEffect, useState, useContext } from "react";
+import {
+  AppSettingsProviderContext,
+  MediaPlayerProviderContext,
+} from "@renderer/context";
+import {
+  Button,
+  toast,
+  Tabs,
+  TabsList,
+  TabsTrigger,
+  TabsContent,
+  Separator,
+} from "@renderer/components/ui";
+import { ConversationShortcuts } from "@renderer/components";
+import { t } from "i18next";
+import { BotIcon } from "lucide-react";
+import { TimelineEntry } from "echogarden/dist/utilities/Timeline.d.js";
+import { useAiCommand, useCamdict } from "@renderer/hooks";
+import { LoaderIcon, Volume2Icon } from "lucide-react";
+import { convertIpaToNormal } from "@/utils";
+import { md5 } from "js-md5";
+import Markdown from "react-markdown";
+
+/*
+ * Tabs below the caption text.
+ * It provides the translation, analysis, and note features.
+ */
+export const MediaCaptionTabs = (props: {
+  caption: TimelineEntry;
+  selectedIndices: number[];
+  toggleRegion: (index: number) => void;
+}) => {
+  const { caption, selectedIndices, toggleRegion } = props;
+
+  const [tab, setTab] = useState<string>("selected");
+
+  if (!caption) return null;
+
+  return (
+    <Tabs value={tab} onValueChange={(value) => setTab(value)} className="">
+      <TabsList className="grid grid-cols-4 gap-4 rounded-none sticky top-0 px-4 mb-4">
+        <TabsTrigger value="selected">{t("captionTabs.selected")}</TabsTrigger>
+        <TabsTrigger value="translation">
+          {t("captionTabs.translation")}
+        </TabsTrigger>
+        <TabsTrigger value="analysis">{t("captionTabs.analysis")}</TabsTrigger>
+        <TabsTrigger value="note">{t("captionTabs.note")}</TabsTrigger>
+      </TabsList>
+
+      <div className="px-4 pb-4 min-h-32">
+        <SelectedTabContent
+          caption={caption}
+          selectedIndices={selectedIndices}
+          toggleRegion={toggleRegion}
+        />
+
+        <TranslationTabContent text={caption.text} />
+
+        <AnalysisTabContent text={caption.text} />
+
+        <TabsContent value="note">
+          <div className="text-muted-foreground text-center py-4">
+            Comming soon
+          </div>
+        </TabsContent>
+      </div>
+    </Tabs>
+  );
+};
+
+const AIButton = (props: {
+  prompt: string;
+  onReply?: (replies: MessageType[]) => void;
+  tooltip: string;
+}) => {
+  const { prompt, onReply, tooltip } = props;
+  return (
+    <ConversationShortcuts
+      prompt={prompt}
+      onReply={onReply}
+      title={tooltip}
+      trigger={
+        <Button
+          data-tooltip-id="media-player-tooltip"
+          data-tooltip-content={tooltip}
+          variant="outline"
+          size="sm"
+          className="p-0 w-8 h-8 rounded-full"
+        >
+          <BotIcon className="w-5 h-5" />
+        </Button>
+      }
+    />
+  );
+};
+
+const SelectedTabContent = (props: {
+  caption: TimelineEntry;
+  selectedIndices: number[];
+  toggleRegion: (index: number) => void;
+}) => {
+  const { selectedIndices, caption, toggleRegion } = props;
+
+  const { transcription } = useContext(MediaPlayerProviderContext);
+  const { webApi } = useContext(AppSettingsProviderContext);
+
+  const [lookingUp, setLookingUp] = useState<boolean>(false);
+  const [lookupResult, setLookupResult] = useState<LookupType>();
+
+  const lookup = () => {
+    if (selectedIndices.length === 0) return;
+
+    const word = selectedIndices
+      .map((index) => caption.timeline[index]?.text || "")
+      .join(" ");
+    if (!word) return;
+
+    setLookingUp(true);
+    lookupWord({
+      word,
+      context: caption.text,
+      sourceId: transcription.targetId,
+      sourceType: transcription.targetType,
+    })
+      .then((res) => {
+        if (res?.meaning) {
+          setLookupResult(res);
+        }
+      })
+      .catch((error) => {
+        toast.error(error.message);
+      })
+      .finally(() => {
+        setLookingUp(false);
+      });
+  };
+
+  const { lookupWord } = useAiCommand();
+  const { result: camdictResult } = useCamdict(
+    selectedIndices
+      .map((index) => caption?.timeline?.[index]?.text || "")
+      .join(" ")
+      .trim()
+  );
+
+  /*
+   * If the selected indices are changed, then reset the lookup result.
+   */
+  useEffect(() => {
+    if (!caption) return;
+    if (!selectedIndices) return;
+
+    const word = selectedIndices
+      .map((index) => caption.timeline[index]?.text || "")
+      .join(" ");
+
+    if (!word) return;
+
+    webApi
+      .lookup({
+        word,
+        context: caption.text,
+        sourceId: transcription.targetId,
+        sourceType: transcription.targetType,
+      })
+      .then((res) => {
+        if (res?.meaning) {
+          setLookupResult(res);
+        } else {
+          setLookupResult(null);
+        }
+      });
+  }, [caption, selectedIndices]);
+
+  if (selectedIndices.length === 0)
+    return (
+      <TabsContent value="selected">
+        <div className="text-sm text-muted-foreground py-4">
+          {t("clickAnyWordToSelect")}
+        </div>
+      </TabsContent>
+    );
+
+  return (
+    <TabsContent value="selected">
+      <div className="flex flex-wrap items-center space-x-2 select-text mb-4">
+        {selectedIndices.map((index, i) => {
+          const word = caption.timeline[index];
+          if (!word) return;
+          return (
+            <div key={index}>
+              <div className="font-serif text-lg font-semibold tracking-tight">
+                {word.text}
+              </div>
+              <div className="text-sm text-serif text-muted-foreground">
+                <span
+                  className={`mr-2 font-code ${
+                    i === 0 ? "before:content-['/']" : ""
+                  }
+                        ${
+                          i === selectedIndices.length - 1
+                            ? "after:content-['/']"
+                            : ""
+                        }`}
+                >
+                  {word.timeline
+                    .map((t) =>
+                      t.timeline.map((s) => convertIpaToNormal(s.text)).join("")
+                    )
+                    .join("")}
+                </span>
+              </div>
+            </div>
+          );
+        })}
+      </div>
+
+      {camdictResult && (
+        <>
+          <Separator className="my-2" />
+          <div className="text-sm italic text-muted-foreground mb-2">
+            {t("cambridgeDictionary")}
+          </div>
+          <div className="select-text">
+            {camdictResult.posItems.map((posItem, index) => (
+              <div key={index} className="mb-4">
+                <div className="flex items-center space-x-4 mb-2">
+                  <div className="italic text-sm text-muted-foreground">
+                    {posItem.type}
+                  </div>
+
+                  {posItem.pronunciations.map((pron, i) => (
+                    <div
+                      key={`pron-${i}`}
+                      className="flex items-center space-x-2"
+                    >
+                      <span className="uppercase text-xs font-serif text-muted-foreground">
+                        [{pron.region}]
+                      </span>
+                      <span className="text-sm font-code">
+                        /{pron.pronunciation}/
+                      </span>
+                      {pron.audio && (
+                        <div>
+                          <Button
+                            variant="ghost"
+                            size="icon"
+                            className="rounded-full p-0 w-6 h-6"
+                            onClick={() => {
+                              const audio = document.getElementById(
+                                `${posItem.type}-${pron.region}`
+                              ) as HTMLAudioElement;
+                              if (audio) {
+                                audio.play();
+                              }
+                            }}
+                          >
+                            <Volume2Icon className="w-4 h-4" />
+                          </Button>
+                          <audio
+                            className="hidden"
+                            id={`${posItem.type}-${pron.region}`}
+                            src={pron.audio}
+                          />
+                        </div>
+                      )}
+                    </div>
+                  ))}
+                </div>
+                <ul className="list-disc pl-4">
+                  {posItem.definitions.map((def, i) => (
+                    <li key={`pos-${i}`} className="">
+                      {def.definition}
+                    </li>
+                  ))}
+                </ul>
+              </div>
+            ))}
+          </div>
+        </>
+      )}
+
+      <Separator className="my-2" />
+      <div className="text-sm italic text-muted-foreground mb-2">
+        {t("AiDictionary")}
+      </div>
+      {lookupResult ? (
+        <div className="mb-4 select-text">
+          <div className="mb-2">
+            {lookupResult.meaning?.pos && (
+              <span className="italic text-sm text-muted-foreground mr-2">
+                {lookupResult.meaning.pos}
+              </span>
+            )}
+            {lookupResult.meaning?.pronunciation && (
+              <span className="text-sm font-code mr-2">
+                /{lookupResult.meaning.pronunciation}/
+              </span>
+            )}
+            {lookupResult.meaning?.lemma &&
+              lookupResult.meaning.lemma !== lookupResult.meaning.word && (
+                <span className="text-sm">({lookupResult.meaning.lemma})</span>
+              )}
+          </div>
+          <div className="text-serif">{lookupResult.meaning.translation}</div>
+          <div className="text-serif">{lookupResult.meaning.definition}</div>
+        </div>
+      ) : (
+        <div className="flex items-center space-x-2 py-2">
+          <Button size="sm" disabled={lookingUp} onClick={lookup}>
+            {lookingUp && <LoaderIcon className="animate-spin w-4 h-4 mr-2" />}
+            <span>{t("AiTranslate")}</span>
+          </Button>
+        </div>
+      )}
+
+      <div className="flex items-center justify-end py-2">
+        <Button
+          size="sm"
+          variant="secondary"
+          onClick={() => toggleRegion(selectedIndices[0])}
+        >
+          {t("cancel")}
+        </Button>
+      </div>
+    </TabsContent>
+  );
+};
+
+/*
+ * Translation tab content.
+ */
+const TranslationTabContent = (props: { text: string }) => {
+  const { text } = props;
+  const { EnjoyApp } = useContext(AppSettingsProviderContext);
+  const [translation, setTranslation] = useState<string>();
+  const [translating, setTranslating] = useState<boolean>(false);
+  const { translate } = useAiCommand();
+
+  const translateSetence = async () => {
+    if (translating) return;
+
+    setTranslating(true);
+    translate(text)
+      .then((result) => {
+        if (result) {
+          setTranslation(result);
+        }
+      })
+      .catch((err) => t("translationFailed", { error: err.message }))
+      .finally(() => {
+        setTranslating(false);
+      });
+  };
+
+  /*
+   * If the caption is changed, then reset the translation.
+   * Also, check if the translation is cached, then use it.
+   */
+  useEffect(() => {
+    EnjoyApp.cacheObjects.get(`translate-${md5(text)}`).then((cached) => {
+      setTranslation(cached);
+    });
+  }, [text]);
+
+  return (
+    <TabsContent value="translation">
+      {translation ? (
+        <>
+          <Markdown className="select-text prose prose-sm prose-h3:text-base max-w-full mb-4">
+            {translation}
+          </Markdown>
+
+          <div className="flex items-center justify-end">
+            <Button
+              variant="secondary"
+              size="sm"
+              disabled={translating}
+              onClick={translateSetence}
+            >
+              {translating && (
+                <LoaderIcon className="animate-spin w-4 h-4 mr-2" />
+              )}
+              {t("reTranslate")}
+            </Button>
+          </div>
+        </>
+      ) : (
+        <div className="flex items-center justify-center space-x-2 py-4">
+          <Button
+            size="sm"
+            disabled={translating}
+            onClick={() => translateSetence()}
+          >
+            {translating && (
+              <LoaderIcon className="animate-spin w-4 h-4 mr-2" />
+            )}
+            <span>{t("translateSetence")}</span>
+          </Button>
+        </div>
+      )}
+    </TabsContent>
+  );
+};
+
+const AnalysisTabContent = (props: { text: string }) => {
+  const { text } = props;
+  const { EnjoyApp } = useContext(AppSettingsProviderContext);
+  const [analyzing, setAnalyzing] = useState<boolean>(false);
+  const [analysisResult, setAnalysisResult] = useState<string>();
+
+  const { analyzeText } = useAiCommand();
+
+  const analyzeSetence = async () => {
+    if (analyzing) return;
+
+    setAnalyzing(true);
+    analyzeText(text, `analyze-${md5(text)}`)
+      .then((result) => {
+        if (result) {
+          setAnalysisResult(result);
+        }
+      })
+      .catch((err) => t("analysisFailed", { error: err.message }))
+      .finally(() => {
+        setAnalyzing(false);
+      });
+  };
+
+  /*
+   * If the caption is changed, then reset the analysis.
+   * Also, check if the translation is cached, then use it.
+   */
+  useEffect(() => {
+    EnjoyApp.cacheObjects.get(`analyze-${md5(text)}`).then((cached) => {
+      setAnalysisResult(cached);
+    });
+  }, [text]);
+
+  return (
+    <TabsContent value="analysis">
+      {analysisResult ? (
+        <>
+          <Markdown
+            className="select-text prose prose-sm prose-h3:text-base max-w-full mb-4"
+            components={{
+              a({ node, children, ...props }) {
+                try {
+                  new URL(props.href ?? "");
+                  props.target = "_blank";
+                  props.rel = "noopener noreferrer";
+                } catch (e) {}
+
+                return <a {...props}>{children}</a>;
+              },
+            }}
+          >
+            {analysisResult}
+          </Markdown>
+
+          <div className="flex items-center space-x-2 justify-end">
+            <Button
+              variant="secondary"
+              size="sm"
+              disabled={analyzing}
+              onClick={analyzeSetence}
+            >
+              {analyzing && (
+                <LoaderIcon className="animate-spin w-4 h-4 mr-2" />
+              )}
+              {t("reAnalyze")}
+            </Button>
+            <AIButton
+              prompt={text as string}
+              onReply={(replies) => {
+                const result = replies.map((m) => m.content).join("\n");
+                setAnalysisResult(result);
+                EnjoyApp.cacheObjects.set(`analyze-${md5(text)}`, result);
+              }}
+              tooltip={t("useAIAssistantToAnalyze")}
+            />
+          </div>
+        </>
+      ) : (
+        <div className="flex items-center justify-center space-x-2 py-4">
+          <Button size="sm" disabled={analyzing} onClick={analyzeSetence}>
+            {analyzing && <LoaderIcon className="animate-spin w-4 h-4 mr-2" />}
+            <span>{t("analyzeSetence")}</span>
+          </Button>
+          <AIButton
+            prompt={text as string}
+            onReply={(replies) => {
+              const result = replies.map((m) => m.content).join("\n");
+              setAnalysisResult(result);
+              EnjoyApp.cacheObjects.set(`analyze-${md5(text)}`, result);
+            }}
+            tooltip={t("useAIAssistantToAnalyze")}
+          />
+        </div>
+      )}
+    </TabsContent>
+  );
+};

enjoy/src/renderer/components/medias/media-caption.tsx

@@ -1,28 +1,16 @@
 import { useEffect, useState, useContext } from "react";
-import {
-  AppSettingsProviderContext,
-  MediaPlayerProviderContext,
-} from "@renderer/context";
+import { MediaPlayerProviderContext } from "@renderer/context";
 import cloneDeep from "lodash/cloneDeep";
-import {
-  Button,
-  toast,
-  ScrollArea,
-  Tabs,
-  TabsList,
-  TabsTrigger,
-  TabsContent,
-} from "@renderer/components/ui";
-import { ConversationShortcuts } from "@renderer/components";
+import { Button, toast, ScrollArea } from "@renderer/components/ui";
+import { ConversationShortcuts, MediaCaptionTabs } from "@renderer/components";
 import { t } from "i18next";
 import { BotIcon, CopyIcon, CheckIcon, SpeechIcon } from "lucide-react";
-import { Timeline } from "echogarden/dist/utilities/Timeline.d.js";
-import { useAiCommand } from "@renderer/hooks";
-import { LoaderIcon } from "lucide-react";
+import {
+  Timeline,
+  TimelineEntry,
+} from "echogarden/dist/utilities/Timeline.d.js";
 import { convertIpaToNormal } from "@/utils";
 import { useCopyToClipboard } from "@uidotdev/usehooks";
-import { md5 } from "js-md5";
-import Markdown from "react-markdown";
 
 export const MediaCaption = () => {
   const {
@@ -45,9 +33,7 @@ export const MediaCaption = () => {
   const [_, copyToClipboard] = useCopyToClipboard();
   const [copied, setCopied] = useState<boolean>(false);
 
-  const caption = (transcription?.result?.timeline as Timeline)?.[
-    currentSegmentIndex
-  ];
+  const [caption, setCaption] = useState<TimelineEntry | null>(null);
 
   const toggleMultiSelect = (event: KeyboardEvent) => {
     setMultiSelecting(event.shiftKey && event.type === "keydown");
@@ -262,7 +248,11 @@ export const MediaCaption = () => {
     };
   }, [editingRegion]);
 
-  useEffect(() => {}, [caption]);
+  useEffect(() => {
+    setCaption(
+      (transcription?.result?.timeline as Timeline)?.[currentSegmentIndex]
+    );
+  }, [currentSegmentIndex, transcription]);
 
   useEffect(() => {
     document.addEventListener("keydown", (event: KeyboardEvent) =>
@@ -364,7 +354,8 @@ export const MediaCaption = () => {
               ))}
         </div>
 
-        <CaptionTabs
+        <MediaCaptionTabs
+          caption={caption}
           selectedIndices={selectedIndices}
           toggleRegion={toggleRegion}
         />
@@ -382,9 +373,19 @@ export const MediaCaption = () => {
           <SpeechIcon className="w-4 h-4" />
         </Button>
 
-        <AIButton
+        <ConversationShortcuts
           prompt={caption.text as string}
-          tooltip={t("sendToAIAssistant")}
+          trigger={
+            <Button
+              data-tooltip-id="media-player-tooltip"
+              data-tooltip-content={t("sendToAIAssistant")}
+              variant="outline"
+              size="sm"
+              className="p-0 w-8 h-8 rounded-full"
+            >
+              <BotIcon className="w-5 h-5" />
+            </Button>
+          }
         />
 
         <Button
@@ -415,369 +416,3 @@ export const MediaCaption = () => {
     </div>
   );
 };
-
-/*
- * Tabs below the caption text.
- * It provides the translation, analysis, and note features.
- */
-const CaptionTabs = (props: {
-  selectedIndices: number[];
-  toggleRegion: (index: number) => void;
-}) => {
-  const { selectedIndices, toggleRegion } = props;
-  const { transcription, currentSegmentIndex } = useContext(
-    MediaPlayerProviderContext
-  );
-  const { EnjoyApp, webApi } = useContext(AppSettingsProviderContext);
-
-  const [translation, setTranslation] = useState<string>();
-  const [translating, setTranslating] = useState<boolean>(false);
-
-  const [lookingUp, setLookingUp] = useState<boolean>(false);
-  const [lookupResult, setLookupResult] = useState<LookupType>();
-
-  const [analyzing, setAnalyzing] = useState<boolean>(false);
-  const [analysisResult, setAnalysisResult] = useState<string>();
-
-  const [hash, setHash] = useState<string>();
-
-  const [tab, setTab] = useState<string>("selected");
-
-  const { translate, lookupWord, analyzeText } = useAiCommand();
-  const caption = (transcription?.result?.timeline as Timeline)?.[
-    currentSegmentIndex
-  ];
-
-  const lookup = () => {
-    if (selectedIndices.length === 0) return;
-
-    const word = selectedIndices
-      .map((index) => caption.timeline[index]?.text || "")
-      .join(" ");
-    if (!word) return;
-
-    setLookingUp(true);
-    lookupWord({
-      word,
-      context: caption.text,
-      sourceId: transcription.targetId,
-      sourceType: transcription.targetType,
-    })
-      .then((res) => {
-        if (res?.meaning) {
-          setLookupResult(res);
-        }
-      })
-      .catch((error) => {
-        toast.error(error.message);
-      })
-      .finally(() => {
-        setLookingUp(false);
-      });
-  };
-
-  const translateSetence = async () => {
-    if (translating) return;
-
-    setTranslating(true);
-    translate(caption.text, `translate-${hash}`)
-      .then((result) => {
-        if (result) {
-          setTranslation(result);
-        }
-      })
-      .catch((err) => t("translationFailed", { error: err.message }))
-      .finally(() => {
-        setTranslating(false);
-      });
-  };
-
-  const analyzeSetence = async () => {
-    if (analyzing) return;
-
-    setAnalyzing(true);
-    analyzeText(caption.text, `analyze-${hash}`)
-      .then((result) => {
-        if (result) {
-          setAnalysisResult(result);
-        }
-      })
-      .catch((err) => t("analysisFailed", { error: err.message }))
-      .finally(() => {
-        setAnalyzing(false);
-      });
-  };
-
-  /*
-   * If the selected indices are changed, then reset the lookup result.
-   */
-  useEffect(() => {
-    if (!caption) return;
-    if (!selectedIndices) return;
-
-    const word = selectedIndices
-      .map((index) => caption.timeline[index]?.text || "")
-      .join(" ");
-
-    if (!word) return;
-
-    webApi
-      .lookup({
-        word,
-        context: caption.text,
-        sourceId: transcription.targetId,
-        sourceType: transcription.targetType,
-      })
-      .then((res) => {
-        if (res?.meaning) {
-          setLookupResult(res);
-        } else {
-          setLookupResult(null);
-        }
-      });
-  }, [caption, selectedIndices]);
-
-  /*
-   * If the caption is changed, then reset the translation and lookup result.
-   * Also, check if the translation is cached, then use it.
-   */
-  useEffect(() => {
-    if (!caption) return;
-
-    const md5Hash = md5.create();
-    md5Hash.update(caption.text);
-    setHash(md5Hash.hex());
-
-    EnjoyApp.cacheObjects.get(`translate-${md5Hash.hex()}`).then((cached) => {
-      setTranslation(cached);
-    });
-
-    EnjoyApp.cacheObjects.get(`analyze-${md5Hash.hex()}`).then((cached) => {
-      setAnalysisResult(cached);
-    });
-  }, [caption]);
-
-  return (
-    <Tabs value={tab} onValueChange={(value) => setTab(value)} className="">
-      <TabsList className="grid grid-cols-4 gap-4 rounded-none sticky top-0 px-4 mb-4">
-        <TabsTrigger value="selected">{t("captionTabs.selected")}</TabsTrigger>
-        <TabsTrigger value="translation">
-          {t("captionTabs.translation")}
-        </TabsTrigger>
-        <TabsTrigger value="analysis">{t("captionTabs.analysis")}</TabsTrigger>
-        <TabsTrigger value="note">{t("captionTabs.note")}</TabsTrigger>
-      </TabsList>
-
-      <div className="px-4 pb-4 min-h-32">
-        <TabsContent value="selected">
-          {selectedIndices.length > 0 ? (
-            <>
-              <div className="flex flex-wrap items-center space-x-2 select-text mb-4">
-                {selectedIndices.map((index, i) => {
-                  const word = caption.timeline[index];
-                  if (!word) return;
-                  return (
-                    <div key={index}>
-                      <div className="font-serif text-lg font-semibold tracking-tight">
-                        {word.text}
-                      </div>
-                      <div className="text-sm text-serif text-muted-foreground">
-                        <span
-                          className={`mr-2 font-code ${
-                            i === 0 ? "before:content-['/']" : ""
-                          }
-                        ${
-                          i === selectedIndices.length - 1
-                            ? "after:content-['/']"
-                            : ""
-                        }`}
-                        >
-                          {word.timeline
-                            .map((t) =>
-                              t.timeline
-                                .map((s) => convertIpaToNormal(s.text))
-                                .join("")
-                            )
-                            .join(" · ")}
-                        </span>
-                      </div>
-                    </div>
-                  );
-                })}
-              </div>
-
-              {lookupResult && (
-                <div className="py-2 select-text">
-                  <div className="text-serif">
-                    {lookupResult.meaning.translation}
-                  </div>
-                  <div className="text-serif">
-                    {lookupResult.meaning.definition}
-                  </div>
-                </div>
-              )}
-
-              <div className="flex items-center space-x-2 py-2">
-                {!lookupResult && (
-                  <Button size="sm" disabled={lookingUp} onClick={lookup}>
-                    {lookingUp && (
-                      <LoaderIcon className="animate-spin w-4 h-4 mr-2" />
-                    )}
-                    <span>{t("translate")}</span>
-                  </Button>
-                )}
-                <Button
-                  size="sm"
-                  variant="secondary"
-                  onClick={() => toggleRegion(selectedIndices[0])}
-                >
-                  {t("cancel")}
-                </Button>
-              </div>
-            </>
-          ) : (
-            <div className="text-sm text-muted-foreground py-4">
-              {t("clickAnyWordToSelect")}
-            </div>
-          )}
-        </TabsContent>
-
-        <TabsContent value="translation">
-          {translation ? (
-            <>
-              <Markdown className="select-text prose prose-sm prose-h3:text-base max-w-full mb-4">
-                {translation}
-              </Markdown>
-
-              <div className="flex items-center justify-end">
-                <Button
-                  variant="secondary"
-                  size="sm"
-                  disabled={translating}
-                  onClick={translateSetence}
-                >
-                  {translating && (
-                    <LoaderIcon className="animate-spin w-4 h-4 mr-2" />
-                  )}
-                  {t("reTranslate")}
-                </Button>
-              </div>
-            </>
-          ) : (
-            <div className="flex items-center justify-center space-x-2 py-4">
-              <Button
-                size="sm"
-                disabled={translating}
-                onClick={() => translateSetence()}
-              >
-                {translating && (
-                  <LoaderIcon className="animate-spin w-4 h-4 mr-2" />
-                )}
-                <span>{t("translateSetence")}</span>
-              </Button>
-            </div>
-          )}
-        </TabsContent>
-
-        <TabsContent value="analysis">
-          {analysisResult ? (
-            <>
-              <Markdown
-                className="select-text prose prose-sm prose-h3:text-base max-w-full mb-4"
-                components={{
-                  a({ node, children, ...props }) {
-                    try {
-                      new URL(props.href ?? "");
-                      props.target = "_blank";
-                      props.rel = "noopener noreferrer";
-                    } catch (e) {}
-
-                    return <a {...props}>{children}</a>;
-                  },
-                }}
-              >
-                {analysisResult}
-              </Markdown>
-
-              <div className="flex items-center space-x-2 justify-end">
-                <Button
-                  variant="secondary"
-                  size="sm"
-                  disabled={analyzing}
-                  onClick={analyzeSetence}
-                >
-                  {analyzing && (
-                    <LoaderIcon className="animate-spin w-4 h-4 mr-2" />
-                  )}
-                  {t("reAnalyze")}
-                </Button>
-                <AIButton
-                  prompt={caption.text as string}
-                  onReply={(replies) => {
-                    const result = replies.map((m) => m.content).join("\n");
-                    setAnalysisResult(result);
-                    EnjoyApp.cacheObjects.set(`analyze-${hash}`, result);
-                  }}
-                  tooltip={t("useAIAssistantToAnalyze")}
-                />
-              </div>
-            </>
-          ) : (
-            <div className="flex items-center justify-center space-x-2 py-4">
-              <Button size="sm" disabled={analyzing} onClick={analyzeSetence}>
-                {analyzing && (
-                  <LoaderIcon className="animate-spin w-4 h-4 mr-2" />
-                )}
-                <span>{t("analyzeSetence")}</span>
-              </Button>
-              <AIButton
-                prompt={caption.text as string}
-                onReply={(replies) => {
-                  const result = replies.map((m) => m.content).join("\n");
-                  setAnalysisResult(result);
-                  EnjoyApp.cacheObjects.set(`analyze-${hash}`, result);
-                }}
-                tooltip={t("useAIAssistantToAnalyze")}
-              />
-            </div>
-          )}
-        </TabsContent>
-
-        <TabsContent value="note">
-          <div className="text-muted-foreground text-center py-4">
-            Comming soon
-          </div>
-        </TabsContent>
-      </div>
-    </Tabs>
-  );
-};
-
-const AIButton = (props: {
-  prompt: string;
-  onReply?: (replies: MessageType[]) => void;
-  tooltip: string;
-}) => {
-  const { prompt, onReply, tooltip } = props;
-  const [asking, setAsking] = useState<boolean>(false);
-  return (
-    <ConversationShortcuts
-      open={asking}
-      onOpenChange={setAsking}
-      prompt={prompt}
-      onReply={onReply}
-      title={tooltip}
-      trigger={
-        <Button
-          data-tooltip-id="media-player-tooltip"
-          data-tooltip-content={tooltip}
-          variant="outline"
-          size="sm"
-          className="p-0 w-8 h-8 rounded-full"
-        >
-          <BotIcon className="w-5 h-5" />
-        </Button>
-      }
-    />
-  );
-};

enjoy/src/renderer/hooks/index.ts

@@ -8,3 +8,5 @@ export * from './use-conversation';
 
 export * from './use-audio';
 export * from './use-video';
+
+export * from './use-camdict';

enjoy/src/renderer/hooks/use-camdict.tsx

@@ -0,0 +1,21 @@
+import { useEffect, useContext, useState } from "react";
+import {
+  AppSettingsProviderContext,
+} from "@renderer/context";
+
+export const useCamdict = (word: string) => {
+  const { EnjoyApp } = useContext(AppSettingsProviderContext);
+  const [result, setResult] = useState<CamdictWordType>(null);
+
+  useEffect(() => {
+    if (!word) return;
+
+    EnjoyApp.camdict.lookup(word).then((res) => {
+      setResult(res);
+    });
+  }, [word]);
+
+  return {
+    result,
+  };
+};

enjoy/src/types/camdict.d.ts

@@ -0,0 +1,21 @@
+type CamdictWordType = {
+  id: number;
+  oid: string;
+  word: string;
+  posItems: CamdictPosItemType[];
+  updatedAt: Date;
+  createdAt: Date;
+};
+
+type CamdictPosItemType = {
+  type: string;
+  definitions: {
+    definition: string;
+    examples?: string[];
+  }[];
+  pronunciations: {
+    audio: string;
+    pronunciation: string;
+    region: string;
+  }[];
+};

enjoy/src/types/enjoy-app.d.ts

@@ -111,6 +111,9 @@ type EnjoyAppType = {
     ) => Promise<void>;
     removeListeners: () => Promise<void>;
   };
+  camdict: {
+    lookup: (word: string) => Promise<CamdictWordType | null>;
+  };
   audios: {
     findAll: (params: any) => Promise<AudioType[]>;
     findOne: (params: any) => Promise<AudioType>;

enjoy/vite.main.config.ts

@@ -55,6 +55,10 @@ export default defineConfig((env) => {
             }/${os.platform()}/*`,
             dest: "lib/youtubedr",
           },
+          {
+            src: "lib/dictionaries/*",
+            dest: "lib/dictionaries",
+          },
           {
             src: "src/main/db/migrations/*",
             dest: "migrations",