From d8231ca97cb939387f40c96e4d016d4302c55b08 Mon Sep 17 00:00:00 2001 From: an-lee Date: Wed, 29 May 2024 15:13:52 +0800 Subject: [PATCH] Feat: Settings for learning lang (#641) * refactor settings * refactor constants * add settings for native/learning language * setup langugage for transcribe * use 2 letter code for echogarden * AI commands support multiple language * update languages constant * fix sentry error * fix context menu * show camdict when only learning English * add en-GB * recording assess support multiple languages * fix ai command * refactor --- enjoy/src/api/client.ts | 1 + enjoy/src/commands/analyze.command.ts | 25 +- enjoy/src/commands/extract-story.command.ts | 4 +- enjoy/src/commands/ipa.command.ts | 4 +- enjoy/src/commands/lookup.command.ts | 15 +- enjoy/src/commands/summarize-topic.command.ts | 8 +- enjoy/src/commands/translate.command.ts | 4 +- .../gpt-presets.ts} | 353 +----------------- enjoy/src/constants/index.ts | 57 +++ enjoy/src/constants/ipa.ts | 215 +++++++++++ enjoy/src/constants/languages.json | 38 ++ enjoy/src/constants/whisper-models.json | 79 ++++ enjoy/src/i18n/en.json | 2 + enjoy/src/i18n/zh-CN.json | 2 + enjoy/src/main.ts | 6 +- .../main/db/handlers/recordings-handler.ts | 4 +- enjoy/src/main/db/models/recording.ts | 3 +- enjoy/src/main/settings.ts | 11 +- enjoy/src/main/window.ts | 12 +- enjoy/src/preload.ts | 10 +- enjoy/src/renderer.ts | 2 +- .../tab-content-translation.tsx | 14 +- .../renderer/components/preferences/index.ts | 5 +- .../learning-language-settings.tsx | 56 +++ .../preferences/native-language-settings.tsx | 56 +++ .../components/preferences/preferences.tsx | 11 +- .../recordings/recording-detail.tsx | 4 +- .../components/widgets/lookup-widget.tsx | 10 +- .../context/app-settings-provider.tsx | 40 +- enjoy/src/renderer/hooks/use-ai-command.tsx | 30 +- enjoy/src/renderer/hooks/use-transcribe.tsx | 10 +- enjoy/src/renderer/pages/conversations.tsx | 4 +- enjoy/src/types/enjoy-app.d.ts | 4 +- 33 files changed, 679 insertions(+), 420 deletions(-) rename enjoy/src/{constants.ts => constants/gpt-presets.ts} (61%) create mode 100644 enjoy/src/constants/index.ts create mode 100644 enjoy/src/constants/ipa.ts create mode 100644 enjoy/src/constants/languages.json create mode 100644 enjoy/src/constants/whisper-models.json create mode 100644 enjoy/src/renderer/components/preferences/learning-language-settings.tsx create mode 100644 enjoy/src/renderer/components/preferences/native-language-settings.tsx diff --git a/enjoy/src/api/client.ts b/enjoy/src/api/client.ts index 51d4b7ff..96a3ee10 100644 --- a/enjoy/src/api/client.ts +++ b/enjoy/src/api/client.ts @@ -309,6 +309,7 @@ export class Client { context: string; sourceId?: string; sourceType?: string; + nativeLanguage?: string; }): Promise { return this.api.post("/api/lookups", decamelizeKeys(params)); } diff --git a/enjoy/src/commands/analyze.command.ts b/enjoy/src/commands/analyze.command.ts index f6ebc883..678575f3 100644 --- a/enjoy/src/commands/analyze.command.ts +++ b/enjoy/src/commands/analyze.command.ts @@ -1,8 +1,13 @@ import { ChatPromptTemplate } from "@langchain/core/prompts"; import { textCommand } from "./text.command"; +import { LANGUAGES } from "@/constants"; export const analyzeCommand = async ( text: string, + params: { + learningLanguage: string; + nativeLanguage: string; + }, options: { key: string; modelName?: string; @@ -12,21 +17,25 @@ export const analyzeCommand = async ( ): Promise => { if (!text) throw new Error("Text is required"); + const { learningLanguage, nativeLanguage } = params; const prompt = await ChatPromptTemplate.fromMessages([ ["system", SYSTEM_PROMPT], ["human", text], - ]).format({}); + ]).format({ + learning_language: LANGUAGES.find((l) => l.code === learningLanguage).name, + native_language: LANGUAGES.find((l) => l.code === nativeLanguage).name, + }); return textCommand(prompt, options); }; -const SYSTEM_PROMPT = `你是我的英语教练，我将提供英语文本，你将帮助我分析文本的句子结构、语法和词汇/短语，并对文本进行详细解释。请用中文回答，并按以下格式返回结果： +const SYSTEM_PROMPT = `I speak {native_language}. You're my {learning_language} coach, I'll provide {learning_language} text, you'll help me analyze the sentence structure, grammar, and vocabulary/phrases, and provide a detailed explanation of the text. Please return the results in the following format(but in {native_language}): - ### 句子结构 - (解释句子的每个元素) +### Sentence Structure +(Explain each element of the sentence) - ### 语法 - (解释句子的语法) +### Grammar +(Explain the grammar of the sentence) - ### 词汇/短语 - (解释使用的关键词汇和短语)`; +### Vocabulary/Phrases +(Explain the key vocabulary and phrases used)`; diff --git a/enjoy/src/commands/extract-story.command.ts b/enjoy/src/commands/extract-story.command.ts index 1168c27a..b2367773 100644 --- a/enjoy/src/commands/extract-story.command.ts +++ b/enjoy/src/commands/extract-story.command.ts @@ -1,9 +1,11 @@ import { ChatPromptTemplate } from "@langchain/core/prompts"; import { z } from "zod"; import { jsonCommand } from "./json.command"; +import { LANGUAGES } from "@/constants"; export const extractStoryCommand = async ( text: string, + learningLanguage: string, options: { key: string; modelName?: string; @@ -20,7 +22,7 @@ export const extractStoryCommand = async ( ["system", EXTRACT_STORY_PROMPT], ["human", "{text}"], ]).format({ - learning_language: "English", + learning_language: LANGUAGES.find((l) => l.code === learningLanguage).name, text, }); diff --git a/enjoy/src/commands/ipa.command.ts b/enjoy/src/commands/ipa.command.ts index 1f04bdd2..1575f189 100644 --- a/enjoy/src/commands/ipa.command.ts +++ b/enjoy/src/commands/ipa.command.ts @@ -16,8 +16,8 @@ export const ipaCommand = async ( const schema = z.object({ words: z.array( z.object({ - word: z.string().nonempty(), - ipa: z.string().nonempty(), + word: z.string().min(1), + ipa: z.string().min(1), }) ), }); diff --git a/enjoy/src/commands/lookup.command.ts b/enjoy/src/commands/lookup.command.ts index 6564a734..215dd83c 100644 --- a/enjoy/src/commands/lookup.command.ts +++ b/enjoy/src/commands/lookup.command.ts @@ -1,12 +1,15 @@ import { ChatPromptTemplate } from "@langchain/core/prompts"; import { z } from "zod"; import { jsonCommand } from "./json.command"; +import { LANGUAGES } from "@/constants"; export const lookupCommand = async ( params: { word: string; context: string; meaningOptions?: Partial[]; + learningLanguage?: string; + nativeLanguage?: string; }, options: { key: string; @@ -24,7 +27,13 @@ export const lookupCommand = async ( translation?: string; lemma?: string; }> => { - const { word, context, meaningOptions } = params; + const { + word, + context, + meaningOptions, + learningLanguage = "en-US", + nativeLanguage = "zh-CN", + } = params; const schema = z.object({ id: z.string().optional(), @@ -41,8 +50,8 @@ export const lookupCommand = async ( ["system", DICITIONARY_PROMPT], ["human", "{input}"], ]).format({ - learning_language: "English", - native_language: "Chinese", + learning_language: LANGUAGES.find((l) => l.code === learningLanguage).name, + native_language: LANGUAGES.find((l) => l.code === nativeLanguage).name, input: JSON.stringify({ word, context, diff --git a/enjoy/src/commands/summarize-topic.command.ts b/enjoy/src/commands/summarize-topic.command.ts index 74031ecc..2c7d26f4 100644 --- a/enjoy/src/commands/summarize-topic.command.ts +++ b/enjoy/src/commands/summarize-topic.command.ts @@ -1,8 +1,10 @@ import { ChatPromptTemplate } from "@langchain/core/prompts"; import { textCommand } from "./text.command"; +import { LANGUAGES } from "@/constants"; export const summarizeTopicCommand = async ( text: string, + learningLanguage: string, options: { key: string; modelName?: string; @@ -15,10 +17,12 @@ export const summarizeTopicCommand = async ( const prompt = await ChatPromptTemplate.fromMessages([ ["system", SYSTEM_PROMPT], ["human", text], - ]).format({}); + ]).format({ + learning_language: LANGUAGES.find((l) => l.code === learningLanguage).name, + }); return textCommand(prompt, options); }; const SYSTEM_PROMPT = - "Please generate a four to five word title summarizing our conversation without any lead-in, punctuation, quotation marks, periods, symbols, bold text, or additional text. Remove enclosing quotation marks."; + "Please generate a four to five words title summarizing our conversation in {learning_language} without any lead-in, punctuation, quotation marks, periods, symbols, bold text, or additional text. Remove enclosing quotation marks."; diff --git a/enjoy/src/commands/translate.command.ts b/enjoy/src/commands/translate.command.ts index b6460590..00f94e40 100644 --- a/enjoy/src/commands/translate.command.ts +++ b/enjoy/src/commands/translate.command.ts @@ -1,8 +1,10 @@ import { ChatPromptTemplate } from "@langchain/core/prompts"; import { textCommand } from "./text.command"; +import { LANGUAGES } from "@/constants"; export const translateCommand = async ( text: string, + nativeLanguage: string, options: { key: string; modelName?: string; @@ -16,7 +18,7 @@ export const translateCommand = async ( ["system", SYSTEM_PROMPT], ["human", TRANSLATION_PROMPT], ]).format({ - native_language: "Chinese", + native_language: LANGUAGES.find((l) => l.code === nativeLanguage).name, text, }); diff --git a/enjoy/src/constants.ts b/enjoy/src/constants/gpt-presets.ts similarity index 61% rename from enjoy/src/constants.ts rename to enjoy/src/constants/gpt-presets.ts index 6da15024..9809cf2b 100644 --- a/enjoy/src/constants.ts +++ b/enjoy/src/constants/gpt-presets.ts @@ -1,137 +1,4 @@ -export const DATABASE_NAME = "enjoy_database"; -export const LIBRARY_PATH_SUFFIX = "EnjoyLibrary"; - -export const STORAGE_WORKER_ENDPOINT = "https://storage.enjoy.bot"; -export const STORAGE_WORKER_ENDPOINTS = [ - "https://storage.enjoy.bot", - "https://enjoy-storage.baizhiheizi.com", -]; - -export const AI_WORKER_ENDPOINT = "https://ai-worker.enjoy.bot"; - -export const WEB_API_URL = "https://enjoy.bot"; -export const WEB_API_URLS = ["https://enjoy.bot", "https://enjoy-web.fly.dev"]; - -export const REPO_URL = "https://github.com/xiaolai/everyone-can-use-english"; - -export const SENTRY_DSN = - "https://d51056d7af7d14eae446c0c15b4f3d31@o1168905.ingest.us.sentry.io/4506969353289728"; - -export const MAGIC_TOKEN_REGEX = - /\b(Mrs|Ms|Mr|Dr|Prof|St|[a-zA-Z]{1,2}|\d{1,2})\.\b/g; -export const END_OF_SENTENCE_REGEX = /[^\.!,\?][\.!\?]/g; - -export const FFMPEG_TRIM_SILENCE_OPTIONS = [ - "-af", - "silenceremove=1:start_duration=1:start_threshold=-50dB:detection=peak,aformat=dblp,areverse,silenceremove=start_periods=1:start_duration=1:start_threshold=-50dB:detection=peak,aformat=dblp,areverse", -]; - -export const FFMPEG_CONVERT_WAV_OPTIONS = [ - "-ar", - "16000", - "-ac", - "1", - "-c:a", - "pcm_s16le", -]; - -// https://hf-mirror.com/ggerganov/whisper.cpp/tree/main -export const WHISPER_MODELS_OPTIONS = [ - { - type: "tiny", - name: "ggml-tiny.bin", - size: "75 MB", - sha: "bd577a113a864445d4c299885e0cb97d4ba92b5f", - url: "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-tiny.bin", - }, - { - type: "tiny.en", - name: "ggml-tiny.en.bin", - size: "75 MB", - sha: "c78c86eb1a8faa21b369bcd33207cc90d64ae9df", - url: "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin", - }, - { - type: "base", - name: "ggml-base.bin", - size: "142 MB", - sha: "465707469ff3a37a2b9b8d8f89f2f99de7299dac", - url: "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-base.bin", - }, - { - type: "base.en", - name: "ggml-base.en.bin", - size: "142 MB", - sha: "137c40403d78fd54d454da0f9bd998f78703390c", - url: "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin", - }, - { - type: "small", - name: "ggml-small.bin", - size: "466 MB", - sha: "55356645c2b361a969dfd0ef2c5a50d530afd8d5", - url: "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-small.bin", - }, - { - type: "small.en", - name: "ggml-small.en.bin", - size: "466 MB", - sha: "db8a495a91d927739e50b3fc1cc4c6b8f6c2d022", - url: "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-small.en.bin", - }, - { - type: "medium", - name: "ggml-medium.bin", - size: "1.5 GB", - sha: "fd9727b6e1217c2f614f9b698455c4ffd82463b4", - url: "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-medium.bin", - }, - { - type: "medium.en", - name: "ggml-medium.en.bin", - size: "1.5 GB", - sha: "8c30f0e44ce9560643ebd10bbe50cd20eafd3723", - url: "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-medium.en.bin", - }, - { - type: "large-v1", - name: "ggml-large-v1.bin", - size: "2.9 GB", - sha: "b1caaf735c4cc1429223d5a74f0f4d0b9b59a299", - url: "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-large-v1.bin", - }, - { - type: "large-v2", - name: "ggml-large-v2.bin", - size: "2.9 GB", - sha: "0f4c8e34f21cf1a914c59d8b3ce882345ad349d6", - url: "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-large-v2.bin", - }, - { - type: "large", - name: "ggml-large-v3.bin", - size: "2.9 GB", - sha: "ad82bf6a9043ceed055076d0fd39f5f186ff8062", - url: "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-large-v3.bin", - }, -]; - -export const AudioFormats = ["mp3", "wav", "ogg", "flac", "m4a", "wma", "aac"]; - -export const VideoFormats = ["mp4", "mkv", "avi", "mov", "wmv", "flv", "webm"]; - -export const PROCESS_TIMEOUT = 1000 * 60 * 15; - -export const AI_GATEWAY_ENDPOINT = - "https://gateway.ai.cloudflare.com/v1/11d43ab275eb7e1b271ba4089ecc3864/enjoy"; - -export const NOT_SUPPORT_JSON_FORMAT_MODELS = [ - "gpt-4-vision-preview", - "gpt-4", - "gpt-4-32k", -]; - -export const CONVERSATION_PRESETS = [ +export const GPT_PRESETS = [ { key: "english-coach", name: "英语教练", @@ -368,220 +235,4 @@ export const CONVERSATION_PRESETS = [ }, }, }, -]; - -export const IPA_CONSONANTS: { [key: string]: string[] } = { - plosive: [ - "p", - "b", - "t", - "d", - "ʈ", - "ɖ", - "c", - "ɟ", - "k", - "g", - "q", - "ɢ", - "ʔ", - /* extensions */ "ɡ", - ], - nasal: ["m", "ɱ", "n", "ɳ", "ɲ", "ŋ", "ɴ", "n̩"], - trill: ["ʙ", "r", "ʀ"], - tapOrFlap: ["ⱱ", "ɾ", "ɽ"], - fricative: [ - "ɸ", - "β", - "f", - "v", - "θ", - "ð", - "s", - "z", - "ʃ", - "ʒ", - "ʂ", - "ʐ", - "ç", - "ʝ", - "x", - "ɣ", - "χ", - "ʁ", - "ħ", - "ʕ", - "h", - "ɦ", - ], - lateralFricative: ["ɬ", "ɮ"], - affricate: ["tʃ", "ʈʃ", "dʒ"], // very incomplete, there are many others - approximant: ["ʋ", "ɹ", "ɻ", "j", "ɰ", /* extensions */ "w"], - lateralApproximant: ["l", "ɭ", "ʎ", "ʟ"], -}; - -export const IPA_VOWELS: { [key: string]: string[] } = { - close: ["i", "yɨ", "ʉɯ", "u", "iː"], - closeOther: ["ɪ", "ʏ", "ʊ", "ɨ", "ᵻ"], - closeMid: ["e", "ø", "ɘ", "ɵ", "ɤ", "o", "ə", "oː"], - openMid: ["ɛ", "œ", "ɜ", "ɞ", "ʌ", "ɔ", "ɜː", "uː", "ɔː", "ɛː"], - open: ["æ", "a", "ɶ", "ɐ", "ɑ", "ɒ", "ɑː"], - rhotic: ["◌˞", "ɚ", "ɝ", "ɹ̩"], - diphtongs: [ - "eɪ", - "əʊ", - "oʊ", - "aɪ", - "ɔɪ", - "aʊ", - "iə", - "ɜr", - "ɑr", - "ɔr", - "oʊr", - "oːɹ", - "ir", - "ɪɹ", - "ɔːɹ", - "ɑːɹ", - "ʊɹ", - "ʊr", - "ɛr", - "ɛɹ", - "əl", - "aɪɚ", - "aɪə", - ], -}; - -export const IPA_MAPPINGS: { [key: string]: string } = { - p: "p", - b: "b", - t: "t", - d: "d", - ʈ: "t", - ɖ: "d", - c: "k", - ɟ: "g", - k: "k", - g: "g", - q: "k", - ɢ: "g", - ʔ: "t", - ɡ: "g", - m: "m", - ɱ: "m", - n: "n", - ɳ: "n", - ɲ: "j", - ŋ: "ŋ", - ɴ: "ŋ", - n̩: "n", - ʙ: "r", - r: "r", - ʀ: "r", - ⱱ: "", - ɾ: "t", - ɽ: "r", - ɸ: "f", - β: "v", - f: "f", - v: "v", - θ: "θ", - ð: "ð", - s: "s", - z: "z", - ʃ: "ʃ", - ʒ: "ʒ", - ʂ: "s", - ʐ: "z", - ç: "", - ʝ: "j", - x: "k", - ɣ: "g", - χ: "h", - ʁ: "r", - ħ: "h", - ʕ: "", - h: "h", - ɦ: "h", - ɬ: "", - ɮ: "", - tʃ: "tʃ", - ʈʃ: "tʃ", - dʒ: "dʒ", - ʋ: "v", - ɹ: "r", - ɻ: "r", - j: "j", - ɰ: "w", - w: "w", - l: "l", - ɭ: "l", - ʎ: "j", - ʟ: "l", - i: "i", - yɨ: "iː", - ʉɯ: "uː", - u: "uː", - iː: "iː", - ɪ: "ɪ", - ʏ: "ɪ", - ʊ: "ʊ", - ɨ: "i", - ᵻ: "i:", - e: "e", - ø: "e", - ɘ: "ə", - ɵ: "ə", - ɤ: "ɑː", - o: "o", - ə: "ə", - oː: "oː", - ɛ: "e", - œ: "æ", - ɜ: "ɝ", - ɞ: "əː", - ʌ: "ʌ", - ɔ: "ɔ", - ɜː: "ɝː", - uː: "uː", - ɔː: "ɔː", - ɛː: "e:", - eː: "i:", - æ: "æ", - a: "ɑ", - ɶ: "ɑ", - ɐ: "ə", - ɑ: "ɑ", - ɒ: "ɑː", - ɑː: "ɑː", - "◌˞": "", - ɚ: "ɚ", - ɝ: "ɝ", - ɹ̩: "r", - eɪ: "eɪ", - əʊ: "oʊ", - oʊ: "oʊ", - aɪ: "aɪ", - ɔɪ: "ɔɪ", - aʊ: "aʊ", - iə: "iə", - ɜr: "ɜr", - ɑr: "ɑr", - ɔr: "ɔr", - oʊr: "oʊr", - oːɹ: "ɔːr", - ir: "ir", - ɪɹ: "ɪr", - ɔːɹ: "ɔːr", - ɑːɹ: "ɑːr", - ʊɹ: "ʊr", - ʊr: "ʊr", - ɛr: "er", - ɛɹ: "er", - əl: "əl", - aɪɚ: "aɪ", - aɪə: "aɪə", - ts: "tz", -}; +]; \ No newline at end of file diff --git a/enjoy/src/constants/index.ts b/enjoy/src/constants/index.ts new file mode 100644 index 00000000..21230db8 --- /dev/null +++ b/enjoy/src/constants/index.ts @@ -0,0 +1,57 @@ +export * from './gpt-presets'; +export * from './ipa'; + +// https://hf-mirror.com/ggerganov/whisper.cpp/tree/main +import whisperModels from './whisper-models.json'; +export const WHISPER_MODELS_OPTIONS = whisperModels; + +import languages from './languages.json'; +export const LANGUAGES = languages; + +export const DATABASE_NAME = "enjoy_database"; +export const LIBRARY_PATH_SUFFIX = "EnjoyLibrary"; + +export const STORAGE_WORKER_ENDPOINT = "https://storage.enjoy.bot"; +export const STORAGE_WORKER_ENDPOINTS = [ + "https://storage.enjoy.bot", + "https://enjoy-storage.baizhiheizi.com", +]; + +export const AI_WORKER_ENDPOINT = "https://ai-worker.enjoy.bot"; + +export const WEB_API_URL = "https://enjoy.bot"; + +export const REPO_URL = "https://github.com/xiaolai/everyone-can-use-english"; + +export const SENTRY_DSN = + "https://d51056d7af7d14eae446c0c15b4f3d31@o1168905.ingest.us.sentry.io/4506969353289728"; + +export const MAGIC_TOKEN_REGEX = + /\b(Mrs|Ms|Mr|Dr|Prof|St|[a-zA-Z]{1,2}|\d{1,2})\.\b/g; +export const END_OF_SENTENCE_REGEX = /[^\.!,\?][\.!\?]/g; + +export const FFMPEG_TRIM_SILENCE_OPTIONS = [ + "-af", + "silenceremove=1:start_duration=1:start_threshold=-50dB:detection=peak,aformat=dblp,areverse,silenceremove=start_periods=1:start_duration=1:start_threshold=-50dB:detection=peak,aformat=dblp,areverse", +]; + +export const FFMPEG_CONVERT_WAV_OPTIONS = [ + "-ar", + "16000", + "-ac", + "1", + "-c:a", + "pcm_s16le", +]; + +export const AudioFormats = ["mp3", "wav", "ogg", "flac", "m4a", "wma", "aac"]; + +export const VideoFormats = ["mp4", "mkv", "avi", "mov", "wmv", "flv", "webm"]; + +export const PROCESS_TIMEOUT = 1000 * 60 * 15; + +export const NOT_SUPPORT_JSON_FORMAT_MODELS = [ + "gpt-4-vision-preview", + "gpt-4", + "gpt-4-32k", +]; \ No newline at end of file diff --git a/enjoy/src/constants/ipa.ts b/enjoy/src/constants/ipa.ts new file mode 100644 index 00000000..72c1a142 --- /dev/null +++ b/enjoy/src/constants/ipa.ts @@ -0,0 +1,215 @@ +export const IPA_CONSONANTS: { [key: string]: string[] } = { + plosive: [ + "p", + "b", + "t", + "d", + "ʈ", + "ɖ", + "c", + "ɟ", + "k", + "g", + "q", + "ɢ", + "ʔ", + /* extensions */ "ɡ", + ], + nasal: ["m", "ɱ", "n", "ɳ", "ɲ", "ŋ", "ɴ", "n̩"], + trill: ["ʙ", "r", "ʀ"], + tapOrFlap: ["ⱱ", "ɾ", "ɽ"], + fricative: [ + "ɸ", + "β", + "f", + "v", + "θ", + "ð", + "s", + "z", + "ʃ", + "ʒ", + "ʂ", + "ʐ", + "ç", + "ʝ", + "x", + "ɣ", + "χ", + "ʁ", + "ħ", + "ʕ", + "h", + "ɦ", + ], + lateralFricative: ["ɬ", "ɮ"], + affricate: ["tʃ", "ʈʃ", "dʒ"], // very incomplete, there are many others + approximant: ["ʋ", "ɹ", "ɻ", "j", "ɰ", /* extensions */ "w"], + lateralApproximant: ["l", "ɭ", "ʎ", "ʟ"], +}; + +export const IPA_VOWELS: { [key: string]: string[] } = { + close: ["i", "yɨ", "ʉɯ", "u", "iː"], + closeOther: ["ɪ", "ʏ", "ʊ", "ɨ", "ᵻ"], + closeMid: ["e", "ø", "ɘ", "ɵ", "ɤ", "o", "ə", "oː"], + openMid: ["ɛ", "œ", "ɜ", "ɞ", "ʌ", "ɔ", "ɜː", "uː", "ɔː", "ɛː"], + open: ["æ", "a", "ɶ", "ɐ", "ɑ", "ɒ", "ɑː"], + rhotic: ["◌˞", "ɚ", "ɝ", "ɹ̩"], + diphtongs: [ + "eɪ", + "əʊ", + "oʊ", + "aɪ", + "ɔɪ", + "aʊ", + "iə", + "ɜr", + "ɑr", + "ɔr", + "oʊr", + "oːɹ", + "ir", + "ɪɹ", + "ɔːɹ", + "ɑːɹ", + "ʊɹ", + "ʊr", + "ɛr", + "ɛɹ", + "əl", + "aɪɚ", + "aɪə", + ], +}; + +export const IPA_MAPPINGS: { [key: string]: string } = { + p: "p", + b: "b", + t: "t", + d: "d", + ʈ: "t", + ɖ: "d", + c: "k", + ɟ: "g", + k: "k", + g: "g", + q: "k", + ɢ: "g", + ʔ: "t", + ɡ: "g", + m: "m", + ɱ: "m", + n: "n", + ɳ: "n", + ɲ: "j", + ŋ: "ŋ", + ɴ: "ŋ", + n̩: "n", + ʙ: "r", + r: "r", + ʀ: "r", + ⱱ: "", + ɾ: "t", + ɽ: "r", + ɸ: "f", + β: "v", + f: "f", + v: "v", + θ: "θ", + ð: "ð", + s: "s", + z: "z", + ʃ: "ʃ", + ʒ: "ʒ", + ʂ: "s", + ʐ: "z", + ç: "", + ʝ: "j", + x: "k", + ɣ: "g", + χ: "h", + ʁ: "r", + ħ: "h", + ʕ: "", + h: "h", + ɦ: "h", + ɬ: "", + ɮ: "", + tʃ: "tʃ", + ʈʃ: "tʃ", + dʒ: "dʒ", + ʋ: "v", + ɹ: "r", + ɻ: "r", + j: "j", + ɰ: "w", + w: "w", + l: "l", + ɭ: "l", + ʎ: "j", + ʟ: "l", + i: "i", + yɨ: "iː", + ʉɯ: "uː", + u: "uː", + iː: "iː", + ɪ: "ɪ", + ʏ: "ɪ", + ʊ: "ʊ", + ɨ: "i", + ᵻ: "i:", + e: "e", + ø: "e", + ɘ: "ə", + ɵ: "ə", + ɤ: "ɑː", + o: "o", + ə: "ə", + oː: "oː", + ɛ: "e", + œ: "æ", + ɜ: "ɝ", + ɞ: "əː", + ʌ: "ʌ", + ɔ: "ɔ", + ɜː: "ɝː", + uː: "uː", + ɔː: "ɔː", + ɛː: "e:", + eː: "i:", + æ: "æ", + a: "ɑ", + ɶ: "ɑ", + ɐ: "ə", + ɑ: "ɑ", + ɒ: "ɑː", + ɑː: "ɑː", + "◌˞": "", + ɚ: "ɚ", + ɝ: "ɝ", + ɹ̩: "r", + eɪ: "eɪ", + əʊ: "oʊ", + oʊ: "oʊ", + aɪ: "aɪ", + ɔɪ: "ɔɪ", + aʊ: "aʊ", + iə: "iə", + ɜr: "ɜr", + ɑr: "ɑr", + ɔr: "ɔr", + oʊr: "oʊr", + oːɹ: "ɔːr", + ir: "ir", + ɪɹ: "ɪr", + ɔːɹ: "ɔːr", + ɑːɹ: "ɑːr", + ʊɹ: "ʊr", + ʊr: "ʊr", + ɛr: "er", + ɛɹ: "er", + əl: "əl", + aɪɚ: "aɪ", + aɪə: "aɪə", + ts: "tz", +}; diff --git a/enjoy/src/constants/languages.json b/enjoy/src/constants/languages.json new file mode 100644 index 00000000..b781de0d --- /dev/null +++ b/enjoy/src/constants/languages.json @@ -0,0 +1,38 @@ +[ + { + "code": "en-US", + "name": "English (United States)" + }, + { + "code": "en-GB", + "name": "English (United Kingdom)" + }, + { + "code": "zh-CN", + "name": "简体中文" + }, + { + "code": "jp-JP", + "name": "日本語" + }, + { + "code": "ko-KR", + "name": "한국인" + }, + { + "code": "es-ES", + "name": "Española" + }, + { + "code": "it-IT", + "name": "Italiana" + }, + { + "code": "de-DE", + "name": "Deutsch" + }, + { + "code": "fr-FR", + "name": "Français" + } +] diff --git a/enjoy/src/constants/whisper-models.json b/enjoy/src/constants/whisper-models.json new file mode 100644 index 00000000..6b778aa6 --- /dev/null +++ b/enjoy/src/constants/whisper-models.json @@ -0,0 +1,79 @@ +[ + { + "type": "tiny", + "name": "ggml-tiny.bin", + "size": "75 MB", + "sha": "bd577a113a864445d4c299885e0cb97d4ba92b5f", + "url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-tiny.bin" + }, + { + "type": "tiny.en", + "name": "ggml-tiny.en.bin", + "size": "75 MB", + "sha": "c78c86eb1a8faa21b369bcd33207cc90d64ae9df", + "url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin" + }, + { + "type": "base", + "name": "ggml-base.bin", + "size": "142 MB", + "sha": "465707469ff3a37a2b9b8d8f89f2f99de7299dac", + "url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-base.bin" + }, + { + "type": "base.en", + "name": "ggml-base.en.bin", + "size": "142 MB", + "sha": "137c40403d78fd54d454da0f9bd998f78703390c", + "url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin" + }, + { + "type": "small", + "name": "ggml-small.bin", + "size": "466 MB", + "sha": "55356645c2b361a969dfd0ef2c5a50d530afd8d5", + "url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-small.bin" + }, + { + "type": "small.en", + "name": "ggml-small.en.bin", + "size": "466 MB", + "sha": "db8a495a91d927739e50b3fc1cc4c6b8f6c2d022", + "url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-small.en.bin" + }, + { + "type": "medium", + "name": "ggml-medium.bin", + "size": "1.5 GB", + "sha": "fd9727b6e1217c2f614f9b698455c4ffd82463b4", + "url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-medium.bin" + }, + { + "type": "medium.en", + "name": "ggml-medium.en.bin", + "size": "1.5 GB", + "sha": "8c30f0e44ce9560643ebd10bbe50cd20eafd3723", + "url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-medium.en.bin" + }, + { + "type": "large-v1", + "name": "ggml-large-v1.bin", + "size": "2.9 GB", + "sha": "b1caaf735c4cc1429223d5a74f0f4d0b9b59a299", + "url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-large-v1.bin" + }, + { + "type": "large-v2", + "name": "ggml-large-v2.bin", + "size": "2.9 GB", + "sha": "0f4c8e34f21cf1a914c59d8b3ce882345ad349d6", + "url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-large-v2.bin" + }, + { + "type": "large", + "name": "ggml-large-v3.bin", + "size": "2.9 GB", + "sha": "ad82bf6a9043ceed055076d0fd39f5f186ff8062", + "url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-large-v3.bin" + } +] diff --git a/enjoy/src/i18n/en.json b/enjoy/src/i18n/en.json index 6fce7d07..4eeff4a0 100644 --- a/enjoy/src/i18n/en.json +++ b/enjoy/src/i18n/en.json @@ -339,6 +339,8 @@ "advancedSettings": "Advanced settings", "advanced": "Advanced", "language": "Language", + "nativeLanguage": "Native Language", + "learningLanguage": "Learning Language", "editEmail": "Edit email", "editUserName": "Edit user name", "userName": "User name", diff --git a/enjoy/src/i18n/zh-CN.json b/enjoy/src/i18n/zh-CN.json index 57b6d96f..e302a783 100644 --- a/enjoy/src/i18n/zh-CN.json +++ b/enjoy/src/i18n/zh-CN.json @@ -339,6 +339,8 @@ "advancedSettings": "高级设置", "advanced": "高级设置", "language": "语言", + "nativeLanguage": "母语", + "learningLanguage": "学习语言", "editEmail": "修改邮箱地址", "editUserName": "修改用户名", "userName": "用户名", diff --git a/enjoy/src/main.ts b/enjoy/src/main.ts index df8568bf..4b19dc4f 100644 --- a/enjoy/src/main.ts +++ b/enjoy/src/main.ts @@ -8,7 +8,7 @@ import mainWindow from "@main/window"; import ElectronSquirrelStartup from "electron-squirrel-startup"; import contextMenu from "electron-context-menu"; import { t } from "i18next"; -import * as Sentry from "@sentry/electron"; +import * as Sentry from "@sentry/electron/main"; import { SENTRY_DSN } from "@/constants"; import { updateElectronApp, UpdateSourceType } from "update-electron-app"; @@ -67,9 +67,7 @@ contextMenu({ }, { label: t("aiTranslate"), - visible: - parameters.selectionText.trim().length > 0 && - parameters.selectionText.trim().includes(" "), + visible: parameters.selectionText.trim().length > 0, click: () => { const { x, y, selectionText } = parameters; browserWindow.webContents.send("on-translate", selectionText, { x, y }); diff --git a/enjoy/src/main/db/handlers/recordings-handler.ts b/enjoy/src/main/db/handlers/recordings-handler.ts index 8291ce7f..2cc6aa85 100644 --- a/enjoy/src/main/db/handlers/recordings-handler.ts +++ b/enjoy/src/main/db/handlers/recordings-handler.ts @@ -178,7 +178,7 @@ class RecordingsHandler { return await recording.upload(); } - private async assess(event: IpcMainEvent, id: string) { + private async assess(event: IpcMainEvent, id: string, language?: string) { const recording = await Recording.findOne({ where: { id, @@ -193,7 +193,7 @@ class RecordingsHandler { } return recording - .assess() + .assess(language) .then((res) => { return res; }) diff --git a/enjoy/src/main/db/models/recording.ts b/enjoy/src/main/db/models/recording.ts index 955a93a0..003c3bf3 100644 --- a/enjoy/src/main/db/models/recording.ts +++ b/enjoy/src/main/db/models/recording.ts @@ -146,7 +146,7 @@ export class Recording extends Model { }); } - async assess() { + async assess(language?: string) { const assessment = await PronunciationAssessment.findOne({ where: { targetId: this.id, targetType: "Recording" }, }); @@ -171,6 +171,7 @@ export class Recording extends Model { const result = await sdk.pronunciationAssessment({ filePath: this.filePath, reference: this.referenceText, + language, }); const resultJson = camelcaseKeys( diff --git a/enjoy/src/main/settings.ts b/enjoy/src/main/settings.ts index e768eec5..9aa49da1 100644 --- a/enjoy/src/main/settings.ts +++ b/enjoy/src/main/settings.ts @@ -34,7 +34,7 @@ const libraryPath = () => { settings.setSync( "library", process.env.LIBRARY_PATH || - path.join(app.getPath("documents"), LIBRARY_PATH_SUFFIX) + path.join(app.getPath("documents"), LIBRARY_PATH_SUFFIX) ); } else if (path.parse(_library).base !== LIBRARY_PATH_SUFFIX) { settings.setSync("library", path.join(_library, LIBRARY_PATH_SUFFIX)); @@ -92,9 +92,16 @@ const userDataPath = () => { return userData; }; - export default { registerIpcHandlers: () => { + ipcMain.handle("settings-get", (_event, key) => { + return settings.getSync(key); + }); + + ipcMain.handle("settings-set", (_event, key, value) => { + settings.setSync(key, value); + }); + ipcMain.handle("settings-get-library", (_event) => { libraryPath(); return settings.getSync("library"); diff --git a/enjoy/src/main/window.ts b/enjoy/src/main/window.ts index 751f5e9a..31f77364 100644 --- a/enjoy/src/main/window.ts +++ b/enjoy/src/main/window.ts @@ -16,7 +16,7 @@ import whisper from "@main/whisper"; import fs from "fs-extra"; import "@main/i18n"; import log from "@main/logger"; -import { WEB_API_URL, REPO_URL, WEB_API_URLS } from "@/constants"; +import { WEB_API_URL, REPO_URL } from "@/constants"; import { AudibleProvider, TedProvider, YoutubeProvider } from "@main/providers"; import Ffmpeg from "@main/ffmpeg"; import { Waveform } from "./waveform"; @@ -140,10 +140,6 @@ main.init = () => { } = bounds; const { navigatable = false } = options || {}; - const OAUTH_URL_REGEX = new RegExp( - `^("${WEB_API_URLS.map((url) => `${url}/oauth`).join("|")})` - ); - logger.debug("view-load", url); const view = new WebContentsView(); mainWindow.contentView.addChildView(view); @@ -191,10 +187,6 @@ main.init = () => { }); logger.debug("will-redirect", detail.url); - if (detail.url.match(OAUTH_URL_REGEX)) { - logger.debug("prevent redirect", detail.url); - detail.preventDefault(); - } }); view.webContents.on("will-navigate", (detail) => { @@ -204,7 +196,7 @@ main.init = () => { }); logger.debug("will-navigate", detail.url); - if (!navigatable || detail.url.match(OAUTH_URL_REGEX)) { + if (!navigatable) { logger.debug("prevent navigation", detail.url); detail.preventDefault(); } diff --git a/enjoy/src/preload.ts b/enjoy/src/preload.ts index 802288d6..785ce462 100644 --- a/enjoy/src/preload.ts +++ b/enjoy/src/preload.ts @@ -156,6 +156,12 @@ contextBridge.exposeInMainWorld("__ENJOY_APP__", { ipcRenderer.invoke("dialog-show-error-box", title, content), }, settings: { + get: (key: string) => { + return ipcRenderer.invoke("settings-get", key); + }, + set: (key: string, value: any) => { + return ipcRenderer.invoke("settings-set", key, value); + }, getLibrary: () => { return ipcRenderer.invoke("settings-get-library"); }, @@ -310,8 +316,8 @@ contextBridge.exposeInMainWorld("__ENJOY_APP__", { upload: (id: string) => { return ipcRenderer.invoke("recordings-upload", id); }, - assess: (id: string) => { - return ipcRenderer.invoke("recordings-assess", id); + assess: (id: string, language?: string) => { + return ipcRenderer.invoke("recordings-assess", id, language); }, stats: (params: { from: string; to: string }) => { return ipcRenderer.invoke("recordings-stats", params); diff --git a/enjoy/src/renderer.ts b/enjoy/src/renderer.ts index d6e184dd..9a985b65 100644 --- a/enjoy/src/renderer.ts +++ b/enjoy/src/renderer.ts @@ -28,7 +28,7 @@ import "./index.css"; import "./renderer/index"; -import * as Sentry from "@sentry/electron"; +import * as Sentry from "@sentry/electron/renderer"; import { SENTRY_DSN } from "@/constants"; Sentry.init({ diff --git a/enjoy/src/renderer/components/medias/media-captions/tab-content-translation.tsx b/enjoy/src/renderer/components/medias/media-captions/tab-content-translation.tsx index 00400b14..1c3beb90 100644 --- a/enjoy/src/renderer/components/medias/media-captions/tab-content-translation.tsx +++ b/enjoy/src/renderer/components/medias/media-captions/tab-content-translation.tsx @@ -1,5 +1,8 @@ import { useContext } from "react"; -import { MediaPlayerProviderContext } from "@renderer/context"; +import { + AppSettingsProviderContext, + MediaPlayerProviderContext, +} from "@renderer/context"; import { TabsContent, Separator } from "@renderer/components/ui"; import { t } from "i18next"; import { TimelineEntry } from "echogarden/dist/utilities/Timeline"; @@ -38,6 +41,7 @@ const SelectedWords = (props: { const { selectedIndices, caption } = props; const { transcription, ipaMappings } = useContext(MediaPlayerProviderContext); + const { learningLanguage } = useContext(AppSettingsProviderContext); const word = selectedIndices .map((index) => caption.timeline[index]?.text || "") @@ -92,8 +96,12 @@ const SelectedWords = (props: { })} - - + {learningLanguage.startsWith("en") && ( + <> + + + + )} { + const { learningLanguage, switchLearningLanguage } = useContext( + AppSettingsProviderContext + ); + + return ( +
+
+
{t("learningLanguage")}
+
+ {LANGUAGES.find((lang) => lang.code === learningLanguage)?.name} +
+
+ +
+
+ +
+
+
+ ); +}; diff --git a/enjoy/src/renderer/components/preferences/native-language-settings.tsx b/enjoy/src/renderer/components/preferences/native-language-settings.tsx new file mode 100644 index 00000000..791d9adf --- /dev/null +++ b/enjoy/src/renderer/components/preferences/native-language-settings.tsx @@ -0,0 +1,56 @@ +import { t } from "i18next"; +import { + Select, + SelectTrigger, + SelectItem, + SelectValue, + SelectContent, +} from "@renderer/components/ui"; +import { AppSettingsProviderContext } from "@renderer/context"; +import { useContext } from "react"; +import { LANGUAGES } from "@/constants"; + +export const NativeLanguageSettings = () => { + const { nativeLanguage, switchNativeLanguage } = useContext( + AppSettingsProviderContext + ); + + return ( +
+
+
{t("nativeLanguage")}
+
+ {LANGUAGES.find((lang) => lang.code === nativeLanguage)?.name} +
+
+ +
+
+ +
+
+
+ ); +}; diff --git a/enjoy/src/renderer/components/preferences/preferences.tsx b/enjoy/src/renderer/components/preferences/preferences.tsx index f3e9d551..9bebe700 100644 --- a/enjoy/src/renderer/components/preferences/preferences.tsx +++ b/enjoy/src/renderer/components/preferences/preferences.tsx @@ -14,6 +14,8 @@ import { GoogleGenerativeAiSettings, ResetSettings, ResetAllSettings, + NativeLanguageSettings, + LearningLanguageSettings, } from "@renderer/components"; import { useState } from "react"; import { Tooltip } from "react-tooltip"; @@ -29,6 +31,10 @@ export const Preferences = () => {
{t("basicSettings")}
+ + + + @@ -108,8 +114,9 @@ export const Preferences = () => { key={tab.value} variant={activeTab === tab.value ? "default" : "ghost"} size="sm" - className={`capitilized w-full justify-start mb-2 ${activeTab === tab.value ? "" : "hover:bg-muted" - }`} + className={`capitilized w-full justify-start mb-2 ${ + activeTab === tab.value ? "" : "hover:bg-muted" + }`} onClick={() => setActiveTab(tab.value)} > {tab.label} diff --git a/enjoy/src/renderer/components/recordings/recording-detail.tsx b/enjoy/src/renderer/components/recordings/recording-detail.tsx index bb5f731a..5c8f3c43 100644 --- a/enjoy/src/renderer/components/recordings/recording-detail.tsx +++ b/enjoy/src/renderer/components/recordings/recording-detail.tsx @@ -21,12 +21,12 @@ export const RecordingDetail = (props: { recording: RecordingType }) => { }>(); const [isPlaying, setIsPlaying] = useState(false); - const { EnjoyApp } = useContext(AppSettingsProviderContext); + const { EnjoyApp, learningLanguage } = useContext(AppSettingsProviderContext); const [assessing, setAssessing] = useState(false); const assess = () => { setAssessing(true); - EnjoyApp.recordings.assess(recording.id).finally(() => { + EnjoyApp.recordings.assess(recording.id, learningLanguage).finally(() => { setAssessing(false); }); }; diff --git a/enjoy/src/renderer/components/widgets/lookup-widget.tsx b/enjoy/src/renderer/components/widgets/lookup-widget.tsx index fcaeb6b8..81136580 100644 --- a/enjoy/src/renderer/components/widgets/lookup-widget.tsx +++ b/enjoy/src/renderer/components/widgets/lookup-widget.tsx @@ -15,7 +15,7 @@ import { t } from "i18next"; import { md5 } from "js-md5"; export const LookupWidget = () => { - const { EnjoyApp } = useContext(AppSettingsProviderContext); + const { EnjoyApp, learningLanguage } = useContext(AppSettingsProviderContext); const [open, setOpen] = useState(false); const [selected, setSelected] = useState<{ word: string; @@ -82,8 +82,12 @@ export const LookupWidget = () => { {selected?.word}
- - + {learningLanguage.startsWith("en") && ( + <> + + + + )} void; + nativeLanguage?: string; + switchNativeLanguage?: (lang: string) => void; + learningLanguage?: string; + switchLearningLanguage?: (lang: string) => void; proxy?: ProxyConfigType; setProxy?: (config: ProxyConfigType) => Promise; ahoy?: typeof ahoy; @@ -42,6 +46,8 @@ export const AppSettingsProvider = ({ const [user, setUser] = useState(null); const [libraryPath, setLibraryPath] = useState(""); const [language, setLanguage] = useState<"en" | "zh-CN">(); + const [nativeLanguage, setNativeLanguage] = useState("zh-CN"); + const [learningLanguage, setLearningLanguage] = useState("en-US"); const [proxy, setProxy] = useState(); const EnjoyApp = window.__ENJOY_APP__; @@ -49,7 +55,7 @@ export const AppSettingsProvider = ({ fetchVersion(); fetchUser(); fetchLibraryPath(); - fetchLanguage(); + fetchLanguages(); fetchProxyConfig(); }, []); @@ -73,10 +79,18 @@ export const AppSettingsProvider = ({ }); }, [apiUrl]); - const fetchLanguage = async () => { + const fetchLanguages = async () => { const language = await EnjoyApp.settings.getLanguage(); setLanguage(language as "en" | "zh-CN"); i18n.changeLanguage(language); + + const _nativeLanguage = + (await EnjoyApp.settings.get("nativeLanguage")) || "zh-CN"; + setNativeLanguage(_nativeLanguage); + + const _learningLanguage = + (await EnjoyApp.settings.get("learningLanguage")) || "en-US"; + setLearningLanguage(_learningLanguage); }; const switchLanguage = (language: "en" | "zh-CN") => { @@ -86,6 +100,22 @@ export const AppSettingsProvider = ({ }); }; + const switchNativeLanguage = (lang: string) => { + if (LANGUAGES.findIndex((l) => l.code == lang) < 0) return; + if (lang == learningLanguage) return; + + setNativeLanguage(lang); + EnjoyApp.settings.set("nativeLanguage", lang); + }; + + const switchLearningLanguage = (lang: string) => { + if (LANGUAGES.findIndex((l) => l.code == lang) < 0) return; + if (lang == nativeLanguage) return; + + EnjoyApp.settings.set("learningLanguage", lang); + setLearningLanguage(lang); + }; + const fetchVersion = async () => { const version = EnjoyApp.app.version; setVersion(version); @@ -148,6 +178,10 @@ export const AppSettingsProvider = ({ value={{ language, switchLanguage, + nativeLanguage, + switchNativeLanguage, + learningLanguage, + switchLearningLanguage, EnjoyApp, version, webApi, diff --git a/enjoy/src/renderer/hooks/use-ai-command.tsx b/enjoy/src/renderer/hooks/use-ai-command.tsx index 4ce2987a..61f8d9df 100644 --- a/enjoy/src/renderer/hooks/use-ai-command.tsx +++ b/enjoy/src/renderer/hooks/use-ai-command.tsx @@ -13,7 +13,9 @@ import { } from "@commands"; export const useAiCommand = () => { - const { EnjoyApp, webApi } = useContext(AppSettingsProviderContext); + const { EnjoyApp, webApi, nativeLanguage, learningLanguage } = useContext( + AppSettingsProviderContext + ); const { currentEngine } = useContext(AISettingsProviderContext); const lookupWord = async (params: { @@ -34,6 +36,7 @@ export const useAiCommand = () => { context, sourceId, sourceType, + nativeLanguage, }); if (lookup.meaning && !force) { @@ -48,6 +51,8 @@ export const useAiCommand = () => { word, context, meaningOptions: lookup.meaningOptions, + nativeLanguage, + learningLanguage, }, { key: currentEngine.key, @@ -74,7 +79,7 @@ export const useAiCommand = () => { }; const extractStory = async (story: StoryType) => { - const res = await extractStoryCommand(story.content, { + const res = await extractStoryCommand(story.content, learningLanguage, { key: currentEngine.key, modelName: currentEngine.models.extractStory || currentEngine.models.default, @@ -92,7 +97,7 @@ export const useAiCommand = () => { text: string, cacheKey?: string ): Promise => { - return translateCommand(text, { + return translateCommand(text, nativeLanguage, { key: currentEngine.key, modelName: currentEngine.models.translate || currentEngine.models.default, baseUrl: currentEngine.baseUrl, @@ -105,11 +110,18 @@ export const useAiCommand = () => { }; const analyzeText = async (text: string, cacheKey?: string) => { - const res = await analyzeCommand(text, { - key: currentEngine.key, - modelName: currentEngine.models.analyze || currentEngine.models.default, - baseUrl: currentEngine.baseUrl, - }); + const res = await analyzeCommand( + text, + { + learningLanguage, + nativeLanguage, + }, + { + key: currentEngine.key, + modelName: currentEngine.models.analyze || currentEngine.models.default, + baseUrl: currentEngine.baseUrl, + } + ); if (cacheKey) { EnjoyApp.cacheObjects.set(cacheKey, res); @@ -126,7 +138,7 @@ export const useAiCommand = () => { }; const summarizeTopic = async (text: string) => { - return summarizeTopicCommand(text, { + return summarizeTopicCommand(text, learningLanguage, { key: currentEngine.key, modelName: currentEngine.models.default, baseUrl: currentEngine.baseUrl, diff --git a/enjoy/src/renderer/hooks/use-transcribe.tsx b/enjoy/src/renderer/hooks/use-transcribe.tsx index 0e2a18fa..7b881c87 100644 --- a/enjoy/src/renderer/hooks/use-transcribe.tsx +++ b/enjoy/src/renderer/hooks/use-transcribe.tsx @@ -12,7 +12,9 @@ import { AlignmentResult } from "echogarden/dist/api/API.d.js"; import { useAiCommand } from "./use-ai-command"; export const useTranscribe = () => { - const { EnjoyApp, user, webApi } = useContext(AppSettingsProviderContext); + const { EnjoyApp, user, webApi, learningLanguage } = useContext( + AppSettingsProviderContext + ); const { whisperConfig, openai } = useContext(AISettingsProviderContext); const { punctuateText } = useAiCommand(); @@ -47,7 +49,7 @@ export const useTranscribe = () => { targetId, targetType, originalText, - language = "english", + language = learningLanguage.split("-")[0], } = params || {}; const blob = await (await fetch(url)).blob(); @@ -169,8 +171,8 @@ export const useTranscribe = () => { const audioConfig = sdk.AudioConfig.fromWavFileInput( new File([blob], "audio.wav") ); - // setting the recognition language to English. - config.speechRecognitionLanguage = "en-US"; + // setting the recognition language to learning language, such as 'en-US'. + config.speechRecognitionLanguage = learningLanguage; config.requestWordLevelTimestamps(); config.outputFormat = sdk.OutputFormat.Detailed; diff --git a/enjoy/src/renderer/pages/conversations.tsx b/enjoy/src/renderer/pages/conversations.tsx index 75f7594c..464d673a 100644 --- a/enjoy/src/renderer/pages/conversations.tsx +++ b/enjoy/src/renderer/pages/conversations.tsx @@ -25,7 +25,7 @@ import { AISettingsProviderContext, } from "@renderer/context"; import { conversationsReducer } from "@renderer/reducers"; -import { CONVERSATION_PRESETS } from "@/constants"; +import { GPT_PRESETS } from "@/constants"; export default () => { const [searchParams] = useSearchParams(); @@ -132,7 +132,7 @@ export default () => { }; const preparePresets = async () => { - let presets = CONVERSATION_PRESETS; + let presets = GPT_PRESETS; let defaultGptPreset = { key: "custom", engine: "enjoyai", diff --git a/enjoy/src/types/enjoy-app.d.ts b/enjoy/src/types/enjoy-app.d.ts index 60de1365..bc79b5d7 100644 --- a/enjoy/src/types/enjoy-app.d.ts +++ b/enjoy/src/types/enjoy-app.d.ts @@ -105,6 +105,8 @@ type EnjoyAppType = { showErrorBox: (title: string, content: string) => Promise; }; settings: { + get: (key: string) => Promise; + set: (key: string, value: any) => Promise; getLibrary: () => Promise; setLibrary: (library: string) => Promise; getUser: () => Promise; @@ -173,7 +175,7 @@ type EnjoyAppType = { update: (id: string, params: any) => Promise; destroy: (id: string) => Promise; upload: (id: string) => Promise; - assess: (id: string) => Promise; + assess: (id: string, language?: string) => Promise; stats: (params: { from: string; to: string }) => Promise<{ count: number; duration: number;