diff --git a/.gitignore b/.gitignore index c0c35973..4d366ed2 100644 --- a/.gitignore +++ b/.gitignore @@ -129,5 +129,8 @@ ffmpeg-core.wasm ffmpeg-core.js ffmpeg-core.worker.js +# dict +cam_dict.refined.sqlite + .vitepress/cache/ /public/jupyter-notebooks/*.mp3 diff --git a/enjoy/lib/dictionaries/.keep b/enjoy/lib/dictionaries/.keep new file mode 100644 index 00000000..e69de29b diff --git a/enjoy/package.json b/enjoy/package.json index a2ff74ff..b8de3ff9 100644 --- a/enjoy/package.json +++ b/enjoy/package.json @@ -21,7 +21,8 @@ "create-migration": "zx ./src/main/db/create-migration.mjs", "download-whisper-model": "zx ./scripts/download-whisper-model.mjs", "download-ffmpeg-wasm": "zx ./scripts/download-ffmpeg-wasm.mjs", - "download": "yarn run download-whisper-model && yarn run download-ffmpeg-wasm" + "download-dictionaries": "zx ./scripts/download-dictionaries.mjs", + "download": "yarn run download-whisper-model && yarn run download-ffmpeg-wasm && yarn run download-dictionaries" }, "keywords": [], "author": { diff --git a/enjoy/scripts/download-dictionaries.mjs b/enjoy/scripts/download-dictionaries.mjs new file mode 100755 index 00000000..54adbe7b --- /dev/null +++ b/enjoy/scripts/download-dictionaries.mjs @@ -0,0 +1,111 @@ +#!/usr/bin/env zx + +import axios from "axios"; +import progress from "progress"; +import { createHash } from "crypto"; + +const dict = "cam_dict.refined.sqlite"; +const sha = "ff97dbd1e6b5357c4a0b42d3c6b1dce0dbc497dc"; + +const dir = path.join(process.cwd(), "lib/dictionaries"); + +console.info(chalk.blue(`=> Download dictionary ${dict}`)); + +fs.ensureDirSync(dir); +try { + if (fs.statSync(path.join(dir, dict)).isFile()) { + console.info(chalk.green(`✅ Dict ${dict} already exists`)); + const hash = await hashFile(path.join(dir, dict), { algo: "sha1" }); + if (hash === sha) { + console.info(chalk.green(`✅ Dict ${dict} valid`)); + process.exit(0); + } else { + console.error( + chalk.red(`❌ Dict ${dict} not valid, start to redownload`) + ); + fs.removeSync(path.join(dir, dict)); + } + } +} catch (err) { + if (err && err.code !== "ENOENT") { + console.error(chalk.red(`❌ Error: ${err}`)); + process.exit(1); + } else { + console.info(chalk.blue(`=> Start to download dict ${dict}`)); + } +} + +const proxyUrl = + process.env.HTTPS_PROXY || + process.env.https_proxy || + process.env.HTTP_PROXY || + process.env.http_proxy; + +if (proxyUrl) { + const { hostname, port, protocol } = new URL(proxyUrl); + axios.defaults.proxy = { + host: hostname, + port: port, + protocol: protocol, + }; +} + +const dictUrlPrefix = "https://enjoy-storage.baizhiheizi.com"; + +function hashFile(path, options) { + const algo = options.algo || "sha1"; + return new Promise((resolve, reject) => { + const hash = createHash(algo); + const stream = fs.createReadStream(path); + stream.on("error", reject); + stream.on("data", (chunk) => hash.update(chunk)); + stream.on("end", () => resolve(hash.digest("hex"))); + }); +} + +const download = async (url, dest) => { + console.info(chalk.blue(`=> Start to download from ${url} to ${dest}`)); + return axios + .get(url, { responseType: "stream" }) + .then((response) => { + const totalLength = response.headers["content-length"]; + + const progressBar = new progress(`-> downloading [:bar] :percent :etas`, { + width: 40, + complete: "=", + incomplete: " ", + renderThrottle: 1, + total: parseInt(totalLength), + }); + + response.data.on("data", (chunk) => { + progressBar.tick(chunk.length); + }); + + response.data.pipe(fs.createWriteStream(dest)).on("close", async () => { + console.info(chalk.green(`✅ Dict ${dict} downloaded successfully`)); + const hash = await hashFile(path.join(dir, dict), { algo: "sha1" }); + if (hash === sha) { + console.info(chalk.green(`✅ Dict ${dict} valid`)); + process.exit(0); + } else { + console.error( + chalk.red( + `❌ Dict ${dict} not valid, please try again using command \`yarn workspace enjoy download-dictionaries\`` + ) + ); + process.exit(1); + } + }); + }) + .catch((err) => { + console.error( + chalk.red( + `❌ Failed to download ${url}: ${err}.\nPlease try again using command \`yarn workspace enjoy download-dictionaries\`` + ) + ); + process.exit(1); + }); +}; + +await download(`${dictUrlPrefix}/${dict}`, path.join(dir, dict)); diff --git a/enjoy/src/i18n/en.json b/enjoy/src/i18n/en.json index bd124b26..c6ff64d5 100644 --- a/enjoy/src/i18n/en.json +++ b/enjoy/src/i18n/en.json @@ -511,5 +511,8 @@ "reTranslate": "re-translate", "analyzeSetence": "analyze setenece", "useAIAssistantToAnalyze": "Use AI assistant to analyze", - "reAnalyze": "re-analyze" + "reAnalyze": "re-analyze", + "AiDictionary": "AI dictionary", + "AiTranslate": "AI translate", + "cambridgeDictionary": "Cambridge dictionary" } diff --git a/enjoy/src/i18n/zh-CN.json b/enjoy/src/i18n/zh-CN.json index 5019e240..6fb6f325 100644 --- a/enjoy/src/i18n/zh-CN.json +++ b/enjoy/src/i18n/zh-CN.json @@ -510,5 +510,8 @@ "reTranslate": "重新翻译", "analyzeSetence": "分析句子", "useAIAssistantToAnalyze": "使用智能助手分析", - "reAnalyze": "重新分析" + "reAnalyze": "重新分析", + "AiDictionary": "智能词典", + "AiTranslate": "智能翻译", + "cambridgeDictionary": "剑桥词典" } diff --git a/enjoy/src/main/camdict.ts b/enjoy/src/main/camdict.ts new file mode 100644 index 00000000..951f1eee --- /dev/null +++ b/enjoy/src/main/camdict.ts @@ -0,0 +1,77 @@ +import { ipcMain, app } from "electron"; +import path from "path"; +import log from "@main/logger"; +import url from "url"; +import { Sequelize, DataType } from "sequelize-typescript"; + +const __filename = url.fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +const logger = log.scope("camdict"); + +class Camdict { + public dbPath = path.join( + __dirname, + "lib", + "dictionaries", + "cam_dict.refined.sqlite" + ); + private sequelize: Sequelize; + private db: any; + + async init() { + if (this.db) return; + + try { + this.sequelize = new Sequelize({ + dialect: "sqlite", + storage: this.dbPath, + }); + this.sequelize.sync(); + this.sequelize.authenticate(); + this.db = this.sequelize.define( + "Camdict", + { + id: { + type: DataType.INTEGER, + primaryKey: true, + }, + oid: { + type: DataType.STRING, + }, + word: { + type: DataType.STRING, + }, + posItems: { + type: DataType.JSON, + }, + }, + { + modelName: "Camdict", + tableName: "camdict", + underscored: true, + timestamps: true, + } + ); + } catch (err) { + logger.error("Failed to initialize camdict", err); + } + } + + async lookup(word: string) { + await this.init(); + + const item = await this.db?.findOne({ + where: { word: word.trim().toLowerCase() }, + }); + return item?.toJSON(); + } + + registerIpcHandlers() { + ipcMain.handle("camdict-lookup", async (_event, word: string) => { + return this.lookup(word); + }); + } +} + +export default new Camdict(); diff --git a/enjoy/src/main/window.ts b/enjoy/src/main/window.ts index 91df82df..0a923afd 100644 --- a/enjoy/src/main/window.ts +++ b/enjoy/src/main/window.ts @@ -22,6 +22,7 @@ import Ffmpeg from "@main/ffmpeg"; import { Waveform } from "./waveform"; import url from "url"; import echogarden from "./echogarden"; +import camdict from "./camdict"; const __filename = url.fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -47,6 +48,8 @@ main.init = () => { // Prepare local database db.registerIpcHandlers(); + camdict.registerIpcHandlers(); + // Prepare Settings settings.registerIpcHandlers(); diff --git a/enjoy/src/preload.ts b/enjoy/src/preload.ts index 525f6473..d1f4a9d3 100644 --- a/enjoy/src/preload.ts +++ b/enjoy/src/preload.ts @@ -178,6 +178,11 @@ contextBridge.exposeInMainWorld("__ENJOY_APP__", { ipcRenderer.removeAllListeners("db-on-transaction"); }, }, + camdict: { + lookup: (word: string) => { + return ipcRenderer.invoke("camdict-lookup", word); + } + }, audios: { findAll: (params: { offset: number | undefined; diff --git a/enjoy/src/renderer/components/meanings/meaning-card.tsx b/enjoy/src/renderer/components/meanings/meaning-card.tsx index 362a4149..c7f211ee 100644 --- a/enjoy/src/renderer/components/meanings/meaning-card.tsx +++ b/enjoy/src/renderer/components/meanings/meaning-card.tsx @@ -32,7 +32,7 @@ export const MeaningCard = (props: { )} {pronunciation && ( - /{pronunciation}/ + /{pronunciation}/ )} {lemma && lemma !== word && ({lemma})} diff --git a/enjoy/src/renderer/components/medias/index.ts b/enjoy/src/renderer/components/medias/index.ts index ba93a3a5..28f0e2aa 100644 --- a/enjoy/src/renderer/components/medias/index.ts +++ b/enjoy/src/renderer/components/medias/index.ts @@ -1,5 +1,6 @@ export * from "./media-player-controls"; export * from "./media-caption"; +export * from "./media-caption-tabs"; export * from "./media-info-panel"; export * from "./media-recordings"; export * from "./media-current-recording"; diff --git a/enjoy/src/renderer/components/medias/media-caption-tabs.tsx b/enjoy/src/renderer/components/medias/media-caption-tabs.tsx new file mode 100644 index 00000000..8c638835 --- /dev/null +++ b/enjoy/src/renderer/components/medias/media-caption-tabs.tsx @@ -0,0 +1,504 @@ +import { useEffect, useState, useContext } from "react"; +import { + AppSettingsProviderContext, + MediaPlayerProviderContext, +} from "@renderer/context"; +import { + Button, + toast, + Tabs, + TabsList, + TabsTrigger, + TabsContent, + Separator, +} from "@renderer/components/ui"; +import { ConversationShortcuts } from "@renderer/components"; +import { t } from "i18next"; +import { BotIcon } from "lucide-react"; +import { TimelineEntry } from "echogarden/dist/utilities/Timeline.d.js"; +import { useAiCommand, useCamdict } from "@renderer/hooks"; +import { LoaderIcon, Volume2Icon } from "lucide-react"; +import { convertIpaToNormal } from "@/utils"; +import { md5 } from "js-md5"; +import Markdown from "react-markdown"; + +/* + * Tabs below the caption text. + * It provides the translation, analysis, and note features. + */ +export const MediaCaptionTabs = (props: { + caption: TimelineEntry; + selectedIndices: number[]; + toggleRegion: (index: number) => void; +}) => { + const { caption, selectedIndices, toggleRegion } = props; + + const [tab, setTab] = useState("selected"); + + if (!caption) return null; + + return ( + setTab(value)} className=""> + + {t("captionTabs.selected")} + + {t("captionTabs.translation")} + + {t("captionTabs.analysis")} + {t("captionTabs.note")} + + +
+ + + + + + + +
+ Comming soon +
+ +
+ + ); +}; + +const AIButton = (props: { + prompt: string; + onReply?: (replies: MessageType[]) => void; + tooltip: string; +}) => { + const { prompt, onReply, tooltip } = props; + return ( + + + + } + /> + ); +}; + +const SelectedTabContent = (props: { + caption: TimelineEntry; + selectedIndices: number[]; + toggleRegion: (index: number) => void; +}) => { + const { selectedIndices, caption, toggleRegion } = props; + + const { transcription } = useContext(MediaPlayerProviderContext); + const { webApi } = useContext(AppSettingsProviderContext); + + const [lookingUp, setLookingUp] = useState(false); + const [lookupResult, setLookupResult] = useState(); + + const lookup = () => { + if (selectedIndices.length === 0) return; + + const word = selectedIndices + .map((index) => caption.timeline[index]?.text || "") + .join(" "); + if (!word) return; + + setLookingUp(true); + lookupWord({ + word, + context: caption.text, + sourceId: transcription.targetId, + sourceType: transcription.targetType, + }) + .then((res) => { + if (res?.meaning) { + setLookupResult(res); + } + }) + .catch((error) => { + toast.error(error.message); + }) + .finally(() => { + setLookingUp(false); + }); + }; + + const { lookupWord } = useAiCommand(); + const { result: camdictResult } = useCamdict( + selectedIndices + .map((index) => caption?.timeline?.[index]?.text || "") + .join(" ") + .trim() + ); + + /* + * If the selected indices are changed, then reset the lookup result. + */ + useEffect(() => { + if (!caption) return; + if (!selectedIndices) return; + + const word = selectedIndices + .map((index) => caption.timeline[index]?.text || "") + .join(" "); + + if (!word) return; + + webApi + .lookup({ + word, + context: caption.text, + sourceId: transcription.targetId, + sourceType: transcription.targetType, + }) + .then((res) => { + if (res?.meaning) { + setLookupResult(res); + } else { + setLookupResult(null); + } + }); + }, [caption, selectedIndices]); + + if (selectedIndices.length === 0) + return ( + +
+ {t("clickAnyWordToSelect")} +
+ + ); + + return ( + +
+ {selectedIndices.map((index, i) => { + const word = caption.timeline[index]; + if (!word) return; + return ( +
+
+ {word.text} +
+
+ + {word.timeline + .map((t) => + t.timeline.map((s) => convertIpaToNormal(s.text)).join("") + ) + .join("")} + +
+
+ ); + })} +
+ + {camdictResult && ( + <> + +
+ {t("cambridgeDictionary")} +
+
+ {camdictResult.posItems.map((posItem, index) => ( +
+
+
+ {posItem.type} +
+ + {posItem.pronunciations.map((pron, i) => ( +
+ + [{pron.region}] + + + /{pron.pronunciation}/ + + {pron.audio && ( +
+ +
+ )} +
+ ))} +
+
    + {posItem.definitions.map((def, i) => ( +
  • + {def.definition} +
    • + ))} +
+
+ ))} +
+ + )} + + +
+ {t("AiDictionary")} +
+ {lookupResult ? ( +
+
+ {lookupResult.meaning?.pos && ( + + {lookupResult.meaning.pos} + + )} + {lookupResult.meaning?.pronunciation && ( + + /{lookupResult.meaning.pronunciation}/ + + )} + {lookupResult.meaning?.lemma && + lookupResult.meaning.lemma !== lookupResult.meaning.word && ( + ({lookupResult.meaning.lemma}) + )} +
+
{lookupResult.meaning.translation}
+
{lookupResult.meaning.definition}
+
+ ) : ( +
+ +
+ )} + +
+ +
+ + ); +}; + +/* + * Translation tab content. + */ +const TranslationTabContent = (props: { text: string }) => { + const { text } = props; + const { EnjoyApp } = useContext(AppSettingsProviderContext); + const [translation, setTranslation] = useState(); + const [translating, setTranslating] = useState(false); + const { translate } = useAiCommand(); + + const translateSetence = async () => { + if (translating) return; + + setTranslating(true); + translate(text) + .then((result) => { + if (result) { + setTranslation(result); + } + }) + .catch((err) => t("translationFailed", { error: err.message })) + .finally(() => { + setTranslating(false); + }); + }; + + /* + * If the caption is changed, then reset the translation. + * Also, check if the translation is cached, then use it. + */ + useEffect(() => { + EnjoyApp.cacheObjects.get(`translate-${md5(text)}`).then((cached) => { + setTranslation(cached); + }); + }, [text]); + + return ( + + {translation ? ( + <> + + {translation} + + +
+ +
+ + ) : ( +
+ +
+ )} + + ); +}; + +const AnalysisTabContent = (props: { text: string }) => { + const { text } = props; + const { EnjoyApp } = useContext(AppSettingsProviderContext); + const [analyzing, setAnalyzing] = useState(false); + const [analysisResult, setAnalysisResult] = useState(); + + const { analyzeText } = useAiCommand(); + + const analyzeSetence = async () => { + if (analyzing) return; + + setAnalyzing(true); + analyzeText(text, `analyze-${md5(text)}`) + .then((result) => { + if (result) { + setAnalysisResult(result); + } + }) + .catch((err) => t("analysisFailed", { error: err.message })) + .finally(() => { + setAnalyzing(false); + }); + }; + + /* + * If the caption is changed, then reset the analysis. + * Also, check if the translation is cached, then use it. + */ + useEffect(() => { + EnjoyApp.cacheObjects.get(`analyze-${md5(text)}`).then((cached) => { + setAnalysisResult(cached); + }); + }, [text]); + + return ( + + {analysisResult ? ( + <> + {children}; + }, + }} + > + {analysisResult} + + +
+ + { + const result = replies.map((m) => m.content).join("\n"); + setAnalysisResult(result); + EnjoyApp.cacheObjects.set(`analyze-${md5(text)}`, result); + }} + tooltip={t("useAIAssistantToAnalyze")} + /> +
+ + ) : ( +
+ + { + const result = replies.map((m) => m.content).join("\n"); + setAnalysisResult(result); + EnjoyApp.cacheObjects.set(`analyze-${md5(text)}`, result); + }} + tooltip={t("useAIAssistantToAnalyze")} + /> +
+ )} + + ); +}; diff --git a/enjoy/src/renderer/components/medias/media-caption.tsx b/enjoy/src/renderer/components/medias/media-caption.tsx index 62b1f4be..d5fbfcc8 100644 --- a/enjoy/src/renderer/components/medias/media-caption.tsx +++ b/enjoy/src/renderer/components/medias/media-caption.tsx @@ -1,28 +1,16 @@ import { useEffect, useState, useContext } from "react"; -import { - AppSettingsProviderContext, - MediaPlayerProviderContext, -} from "@renderer/context"; +import { MediaPlayerProviderContext } from "@renderer/context"; import cloneDeep from "lodash/cloneDeep"; -import { - Button, - toast, - ScrollArea, - Tabs, - TabsList, - TabsTrigger, - TabsContent, -} from "@renderer/components/ui"; -import { ConversationShortcuts } from "@renderer/components"; +import { Button, toast, ScrollArea } from "@renderer/components/ui"; +import { ConversationShortcuts, MediaCaptionTabs } from "@renderer/components"; import { t } from "i18next"; import { BotIcon, CopyIcon, CheckIcon, SpeechIcon } from "lucide-react"; -import { Timeline } from "echogarden/dist/utilities/Timeline.d.js"; -import { useAiCommand } from "@renderer/hooks"; -import { LoaderIcon } from "lucide-react"; +import { + Timeline, + TimelineEntry, +} from "echogarden/dist/utilities/Timeline.d.js"; import { convertIpaToNormal } from "@/utils"; import { useCopyToClipboard } from "@uidotdev/usehooks"; -import { md5 } from "js-md5"; -import Markdown from "react-markdown"; export const MediaCaption = () => { const { @@ -45,9 +33,7 @@ export const MediaCaption = () => { const [_, copyToClipboard] = useCopyToClipboard(); const [copied, setCopied] = useState(false); - const caption = (transcription?.result?.timeline as Timeline)?.[ - currentSegmentIndex - ]; + const [caption, setCaption] = useState(null); const toggleMultiSelect = (event: KeyboardEvent) => { setMultiSelecting(event.shiftKey && event.type === "keydown"); @@ -262,7 +248,11 @@ export const MediaCaption = () => { }; }, [editingRegion]); - useEffect(() => {}, [caption]); + useEffect(() => { + setCaption( + (transcription?.result?.timeline as Timeline)?.[currentSegmentIndex] + ); + }, [currentSegmentIndex, transcription]); useEffect(() => { document.addEventListener("keydown", (event: KeyboardEvent) => @@ -364,7 +354,8 @@ export const MediaCaption = () => { ))} - @@ -382,9 +373,19 @@ export const MediaCaption = () => { - + + + } /> - )} - - - - ) : ( -
- {t("clickAnyWordToSelect")} -
- )} - - - - {translation ? ( - <> - - {translation} - - -
- -
- - ) : ( -
- -
- )} - - - - {analysisResult ? ( - <> - {children}; - }, - }} - > - {analysisResult} - - -
- - { - const result = replies.map((m) => m.content).join("\n"); - setAnalysisResult(result); - EnjoyApp.cacheObjects.set(`analyze-${hash}`, result); - }} - tooltip={t("useAIAssistantToAnalyze")} - /> -
- - ) : ( -
- - { - const result = replies.map((m) => m.content).join("\n"); - setAnalysisResult(result); - EnjoyApp.cacheObjects.set(`analyze-${hash}`, result); - }} - tooltip={t("useAIAssistantToAnalyze")} - /> -
- )} - - - -
- Comming soon -
- - - - ); -}; - -const AIButton = (props: { - prompt: string; - onReply?: (replies: MessageType[]) => void; - tooltip: string; -}) => { - const { prompt, onReply, tooltip } = props; - const [asking, setAsking] = useState(false); - return ( - - - - } - /> - ); -}; diff --git a/enjoy/src/renderer/hooks/index.ts b/enjoy/src/renderer/hooks/index.ts index e752f1b9..b56f5c28 100644 --- a/enjoy/src/renderer/hooks/index.ts +++ b/enjoy/src/renderer/hooks/index.ts @@ -8,3 +8,5 @@ export * from './use-conversation'; export * from './use-audio'; export * from './use-video'; + +export * from './use-camdict'; diff --git a/enjoy/src/renderer/hooks/use-camdict.tsx b/enjoy/src/renderer/hooks/use-camdict.tsx new file mode 100644 index 00000000..2fff55cc --- /dev/null +++ b/enjoy/src/renderer/hooks/use-camdict.tsx @@ -0,0 +1,21 @@ +import { useEffect, useContext, useState } from "react"; +import { + AppSettingsProviderContext, +} from "@renderer/context"; + +export const useCamdict = (word: string) => { + const { EnjoyApp } = useContext(AppSettingsProviderContext); + const [result, setResult] = useState(null); + + useEffect(() => { + if (!word) return; + + EnjoyApp.camdict.lookup(word).then((res) => { + setResult(res); + }); + }, [word]); + + return { + result, + }; +}; diff --git a/enjoy/src/types/camdict.d.ts b/enjoy/src/types/camdict.d.ts new file mode 100644 index 00000000..58914c65 --- /dev/null +++ b/enjoy/src/types/camdict.d.ts @@ -0,0 +1,21 @@ +type CamdictWordType = { + id: number; + oid: string; + word: string; + posItems: CamdictPosItemType[]; + updatedAt: Date; + createdAt: Date; +}; + +type CamdictPosItemType = { + type: string; + definitions: { + definition: string; + examples?: string[]; + }[]; + pronunciations: { + audio: string; + pronunciation: string; + region: string; + }[]; +}; diff --git a/enjoy/src/types/enjoy-app.d.ts b/enjoy/src/types/enjoy-app.d.ts index 01791e29..0346a331 100644 --- a/enjoy/src/types/enjoy-app.d.ts +++ b/enjoy/src/types/enjoy-app.d.ts @@ -111,6 +111,9 @@ type EnjoyAppType = { ) => Promise; removeListeners: () => Promise; }; + camdict: { + lookup: (word: string) => Promise; + }; audios: { findAll: (params: any) => Promise; findOne: (params: any) => Promise; diff --git a/enjoy/vite.main.config.ts b/enjoy/vite.main.config.ts index 870b9b66..9c71a32a 100644 --- a/enjoy/vite.main.config.ts +++ b/enjoy/vite.main.config.ts @@ -55,6 +55,10 @@ export default defineConfig((env) => { }/${os.platform()}/*`, dest: "lib/youtubedr", }, + { + src: "lib/dictionaries/*", + dest: "lib/dictionaries", + }, { src: "src/main/db/migrations/*", dest: "migrations",