Refactor whisper (#1131)
* refactor transcribe hook * Fix stt setting * remove deprecated files * fix transcribe api * may select model when transcribe using local * refactor * upgrade deps * update echogarden check try to fix CI * update echogarden check * update CI
This commit is contained in:
13
.github/workflows/test-enjoy-app.yml
vendored
13
.github/workflows/test-enjoy-app.yml
vendored
@@ -20,7 +20,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [macos-12, macos-14, windows-latest]
|
||||
os: [macos-12, macos-14, windows-latest, ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -47,17 +47,24 @@ jobs:
|
||||
run: yarn enjoy:package
|
||||
|
||||
- name: Run main tests with xvfb-run on Ubuntu
|
||||
continue-on-error: true
|
||||
if: contains(matrix.os, 'ubuntu')
|
||||
env:
|
||||
DEBUG: pw:browser*
|
||||
run: |
|
||||
xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn enjoy:test:main
|
||||
|
||||
- name: Run main tests on macOS or Windows
|
||||
if: contains(matrix.os, 'macos') || contains(matrix.os, 'windows')
|
||||
- name: Run main tests on Windows
|
||||
continue-on-error: true
|
||||
if: contains(matrix.os, 'windows')
|
||||
run: yarn enjoy:test:main
|
||||
|
||||
- name: Run main tests on macOS
|
||||
if: contains(matrix.os, 'macos')
|
||||
run: yarn enjoy:test:main
|
||||
|
||||
- name: Run renderer tests with xvfb-run on Ubuntu
|
||||
continue-on-error: true
|
||||
if: contains(matrix.os, 'ubuntu')
|
||||
run: |
|
||||
xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn enjoy:test:renderer
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
"dependencies": {
|
||||
"@nuxtjs/seo": "^2.0.0-rc.23",
|
||||
"nuxt": "^3.13.2",
|
||||
"nuxt-og-image": "^3.0.4",
|
||||
"nuxt-og-image": "^3.0.6",
|
||||
"vue": "^3.5.12",
|
||||
"vue-router": "^4.4.5"
|
||||
},
|
||||
@@ -20,6 +20,6 @@
|
||||
"autoprefixer": "^10.4.20",
|
||||
"postcss": "^8.4.47",
|
||||
"sass": "^1.79.5",
|
||||
"tailwindcss": "^3.4.13"
|
||||
"tailwindcss": "^3.4.14"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,9 +60,17 @@ test.afterAll(async () => {
|
||||
await electronApp.close();
|
||||
});
|
||||
|
||||
test("validate whisper command", async () => {
|
||||
test("validate echogarden command", async () => {
|
||||
const res = await page.evaluate(() => {
|
||||
return window.__ENJOY_APP__.whisper.check();
|
||||
return window.__ENJOY_APP__.echogarden.check({
|
||||
engine: "whisper",
|
||||
whisper: {
|
||||
model: "tiny.en",
|
||||
language: "en",
|
||||
encoderProvider: "cpu",
|
||||
decoderProvider: "cpu",
|
||||
},
|
||||
});
|
||||
});
|
||||
console.info(res.log);
|
||||
expect(res.success).toBeTruthy();
|
||||
@@ -75,13 +83,6 @@ test("valid ffmpeg command", async () => {
|
||||
expect(res).toBeTruthy();
|
||||
});
|
||||
|
||||
test("validate echogarden align command", async () => {
|
||||
const res = await page.evaluate(() => {
|
||||
return window.__ENJOY_APP__.echogarden.check();
|
||||
});
|
||||
expect(res).toBeTruthy();
|
||||
});
|
||||
|
||||
test("should setup default library path", async () => {
|
||||
const settings = fs.readJsonSync(path.join(resultDir, "settings.json"));
|
||||
expect(settings.library).not.toBeNull();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Binary file not shown.
File diff suppressed because it is too large
Load Diff
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -19,9 +19,8 @@
|
||||
"test:main": "yarn run playwright test e2e/main.spec.ts",
|
||||
"test:renderer": "yarn run playwright test e2e/renderer.spec.ts",
|
||||
"create-migration": "zx ./src/main/db/create-migration.mjs",
|
||||
"download-whisper-model": "zx ./scripts/download-whisper-model.mjs",
|
||||
"download-dictionaries": "zx ./scripts/download-dictionaries.mjs",
|
||||
"download": "yarn run download-whisper-model && yarn run download-dictionaries"
|
||||
"download": "yarn run download-dictionaries"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": {
|
||||
@@ -41,7 +40,7 @@
|
||||
"@electron-forge/plugin-vite": "<7.5.0",
|
||||
"@electron-forge/publisher-github": "<7.5.0",
|
||||
"@electron/fuses": "^1.8.0",
|
||||
"@playwright/test": "^1.48.0",
|
||||
"@playwright/test": "^1.48.1",
|
||||
"@tailwindcss/typography": "^0.5.15",
|
||||
"@types/ahoy.js": "^0.4.2",
|
||||
"@types/autosize": "^4.0.3",
|
||||
@@ -66,7 +65,7 @@
|
||||
"@typescript-eslint/parser": "^8.9.0",
|
||||
"@vitejs/plugin-react": "^4.3.2",
|
||||
"autoprefixer": "^10.4.20",
|
||||
"electron": "^32.2.0",
|
||||
"electron": "^33.0.1",
|
||||
"electron-devtools-installer": "^3.2.0",
|
||||
"electron-playwright-helpers": "^1.7.1",
|
||||
"eslint": "^9.12.0",
|
||||
@@ -78,10 +77,10 @@
|
||||
"prop-types": "^15.8.1",
|
||||
"tailwind-merge": "^2.5.4",
|
||||
"tailwind-scrollbar": "^3.1.0",
|
||||
"tailwindcss": "^3.4.13",
|
||||
"tailwindcss": "^3.4.14",
|
||||
"tailwindcss-animate": "^1.0.7",
|
||||
"ts-node": "^10.9.2",
|
||||
"tslib": "^2.7.0",
|
||||
"tslib": "^2.8.0",
|
||||
"typescript": "^5.6.3",
|
||||
"vite": "^5.4.9",
|
||||
"vite-plugin-static-copy": "^2.0.0",
|
||||
@@ -120,7 +119,7 @@
|
||||
"@radix-ui/react-toast": "^1.2.2",
|
||||
"@radix-ui/react-toggle": "^1.1.0",
|
||||
"@radix-ui/react-tooltip": "^1.1.3",
|
||||
"@rails/actioncable": "7.2.100",
|
||||
"@rails/actioncable": "7.2.101",
|
||||
"@uidotdev/usehooks": "^2.4.1",
|
||||
"@vidstack/react": "^1.12.11",
|
||||
"ahoy.js": "^0.4.4",
|
||||
@@ -128,7 +127,7 @@
|
||||
"axios": "^1.7.7",
|
||||
"camelcase": "^8.0.0",
|
||||
"camelcase-keys": "^9.1.3",
|
||||
"chart.js": "^4.4.4",
|
||||
"chart.js": "^4.4.5",
|
||||
"cheerio": "^1.0.0",
|
||||
"class-variance-authority": "^0.7.0",
|
||||
"clsx": "^2.1.1",
|
||||
@@ -140,7 +139,7 @@
|
||||
"dayjs": "^1.11.13",
|
||||
"decamelize": "^6.0.0",
|
||||
"decamelize-keys": "^2.0.1",
|
||||
"echogarden": "^1.8.1",
|
||||
"echogarden": "^1.8.4",
|
||||
"electron-context-menu": "^4.0.4",
|
||||
"electron-log": "^5.2.0",
|
||||
"electron-settings": "^4.0.4",
|
||||
@@ -156,9 +155,9 @@
|
||||
"langchain": "^0.3.2",
|
||||
"lodash": "^4.17.21",
|
||||
"lru-cache": "^11.0.1",
|
||||
"lucide-react": "^0.452.0",
|
||||
"lucide-react": "^0.453.0",
|
||||
"mark.js": "^8.11.1",
|
||||
"microsoft-cognitiveservices-speech-sdk": "^1.40.0",
|
||||
"microsoft-cognitiveservices-speech-sdk": "^1.41.0",
|
||||
"mustache": "^4.2.0",
|
||||
"next-themes": "^0.3.0",
|
||||
"openai": "^4.67.3",
|
||||
|
||||
@@ -1,113 +0,0 @@
|
||||
#!/usr/bin/env zx
|
||||
|
||||
import axios from "axios";
|
||||
import progress from "progress";
|
||||
import { createHash } from "crypto";
|
||||
|
||||
const model = "ggml-tiny.en.bin";
|
||||
const sha = "c78c86eb1a8faa21b369bcd33207cc90d64ae9df";
|
||||
|
||||
const dir = path.join(process.cwd(), "lib/whisper.cpp/models");
|
||||
|
||||
console.info(chalk.blue(`=> Download whisper model ${model}`));
|
||||
|
||||
fs.ensureDirSync(dir);
|
||||
try {
|
||||
if (fs.statSync(path.join(dir, model)).isFile()) {
|
||||
console.info(chalk.green(`✅ Model ${model} already exists`));
|
||||
const hash = await hashFile(path.join(dir, model), { algo: "sha1" });
|
||||
if (hash === sha) {
|
||||
console.info(chalk.green(`✅ Model ${model} valid`));
|
||||
process.exit(0);
|
||||
} else {
|
||||
console.error(
|
||||
chalk.red(`❌ Model ${model} not valid, start to redownload`)
|
||||
);
|
||||
fs.removeSync(path.join(dir, model));
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (err && err.code !== "ENOENT") {
|
||||
console.error(chalk.red(`❌ Error: ${err}`));
|
||||
process.exit(1);
|
||||
} else {
|
||||
console.info(chalk.blue(`=> Start to download model ${model}`));
|
||||
}
|
||||
}
|
||||
|
||||
const proxyUrl =
|
||||
process.env.HTTPS_PROXY ||
|
||||
process.env.https_proxy ||
|
||||
process.env.HTTP_PROXY ||
|
||||
process.env.http_proxy;
|
||||
|
||||
if (proxyUrl) {
|
||||
const { hostname, port, protocol } = new URL(proxyUrl);
|
||||
axios.defaults.proxy = {
|
||||
host: hostname,
|
||||
port: port,
|
||||
protocol: protocol,
|
||||
};
|
||||
}
|
||||
|
||||
// const modelUrlPrefix =
|
||||
// "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main";
|
||||
const modelUrlPrefix = "https://enjoy-storage.baizhiheizi.com";
|
||||
|
||||
function hashFile(path, options) {
|
||||
const algo = options.algo || "sha1";
|
||||
return new Promise((resolve, reject) => {
|
||||
const hash = createHash(algo);
|
||||
const stream = fs.createReadStream(path);
|
||||
stream.on("error", reject);
|
||||
stream.on("data", (chunk) => hash.update(chunk));
|
||||
stream.on("end", () => resolve(hash.digest("hex")));
|
||||
});
|
||||
}
|
||||
|
||||
const download = async (url, dest) => {
|
||||
console.info(chalk.blue(`=> Start to download from ${url} to ${dest}`));
|
||||
return axios
|
||||
.get(url, { responseType: "stream" })
|
||||
.then((response) => {
|
||||
const totalLength = response.headers["content-length"];
|
||||
|
||||
const progressBar = new progress(`-> downloading [:bar] :percent :etas`, {
|
||||
width: 40,
|
||||
complete: "=",
|
||||
incomplete: " ",
|
||||
renderThrottle: 1,
|
||||
total: parseInt(totalLength),
|
||||
});
|
||||
|
||||
response.data.on("data", (chunk) => {
|
||||
progressBar.tick(chunk.length);
|
||||
});
|
||||
|
||||
response.data.pipe(fs.createWriteStream(dest)).on("close", async () => {
|
||||
console.info(chalk.green(`✅ Model ${model} downloaded successfully`));
|
||||
const hash = await hashFile(path.join(dir, model), { algo: "sha1" });
|
||||
if (hash === sha) {
|
||||
console.info(chalk.green(`✅ Model ${model} valid`));
|
||||
process.exit(0);
|
||||
} else {
|
||||
console.error(
|
||||
chalk.red(
|
||||
`❌ Model ${model} not valid, please try again using command \`yarn workspace enjoy download-whisper-model\``
|
||||
)
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
})
|
||||
.catch((err) => {
|
||||
console.error(
|
||||
chalk.red(
|
||||
`❌ Failed to download ${url}: ${err}.\nPlease try again using command \`yarn workspace enjoy download-whisper-model\``
|
||||
)
|
||||
);
|
||||
process.exit(1);
|
||||
});
|
||||
};
|
||||
|
||||
await download(`${modelUrlPrefix}/${model}`, path.join(dir, model));
|
||||
@@ -2,9 +2,20 @@ export * from "./chat-agent-templates";
|
||||
export * from "./gpt-presets";
|
||||
export * from "./ipa";
|
||||
|
||||
// https://hf-mirror.com/ggerganov/whisper.cpp/tree/main
|
||||
import whisperModels from "./whisper-models.json";
|
||||
export const WHISPER_MODELS_OPTIONS = whisperModels;
|
||||
export const WHISPER_MODELS = [
|
||||
"tiny",
|
||||
"tiny.en",
|
||||
"base",
|
||||
"base.en",
|
||||
"small",
|
||||
"small.en",
|
||||
"medium",
|
||||
"medium.en",
|
||||
"large-v1",
|
||||
"large-v2",
|
||||
"large-v3",
|
||||
"large-v3-turbo",
|
||||
];
|
||||
|
||||
import languages from "./languages.json";
|
||||
export const LANGUAGES = languages;
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
[
|
||||
{
|
||||
"type": "tiny",
|
||||
"name": "ggml-tiny.bin",
|
||||
"size": "75 MB",
|
||||
"sha": "bd577a113a864445d4c299885e0cb97d4ba92b5f",
|
||||
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-tiny.bin"
|
||||
},
|
||||
{
|
||||
"type": "tiny.en",
|
||||
"name": "ggml-tiny.en.bin",
|
||||
"size": "75 MB",
|
||||
"sha": "c78c86eb1a8faa21b369bcd33207cc90d64ae9df",
|
||||
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin"
|
||||
},
|
||||
{
|
||||
"type": "base",
|
||||
"name": "ggml-base.bin",
|
||||
"size": "142 MB",
|
||||
"sha": "465707469ff3a37a2b9b8d8f89f2f99de7299dac",
|
||||
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-base.bin"
|
||||
},
|
||||
{
|
||||
"type": "base.en",
|
||||
"name": "ggml-base.en.bin",
|
||||
"size": "142 MB",
|
||||
"sha": "137c40403d78fd54d454da0f9bd998f78703390c",
|
||||
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin"
|
||||
},
|
||||
{
|
||||
"type": "small",
|
||||
"name": "ggml-small.bin",
|
||||
"size": "466 MB",
|
||||
"sha": "55356645c2b361a969dfd0ef2c5a50d530afd8d5",
|
||||
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-small.bin"
|
||||
},
|
||||
{
|
||||
"type": "small.en",
|
||||
"name": "ggml-small.en.bin",
|
||||
"size": "466 MB",
|
||||
"sha": "db8a495a91d927739e50b3fc1cc4c6b8f6c2d022",
|
||||
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-small.en.bin"
|
||||
},
|
||||
{
|
||||
"type": "medium",
|
||||
"name": "ggml-medium.bin",
|
||||
"size": "1.5 GB",
|
||||
"sha": "fd9727b6e1217c2f614f9b698455c4ffd82463b4",
|
||||
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-medium.bin"
|
||||
},
|
||||
{
|
||||
"type": "medium.en",
|
||||
"name": "ggml-medium.en.bin",
|
||||
"size": "1.5 GB",
|
||||
"sha": "8c30f0e44ce9560643ebd10bbe50cd20eafd3723",
|
||||
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-medium.en.bin"
|
||||
},
|
||||
{
|
||||
"type": "large-v1",
|
||||
"name": "ggml-large-v1.bin",
|
||||
"size": "2.9 GB",
|
||||
"sha": "b1caaf735c4cc1429223d5a74f0f4d0b9b59a299",
|
||||
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-large-v1.bin"
|
||||
},
|
||||
{
|
||||
"type": "large-v2",
|
||||
"name": "ggml-large-v2.bin",
|
||||
"size": "2.9 GB",
|
||||
"sha": "0f4c8e34f21cf1a914c59d8b3ce882345ad349d6",
|
||||
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-large-v2.bin"
|
||||
},
|
||||
{
|
||||
"type": "large",
|
||||
"name": "ggml-large-v3.bin",
|
||||
"size": "2.9 GB",
|
||||
"sha": "ad82bf6a9043ceed055076d0fd39f5f186ff8062",
|
||||
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-large-v3.bin"
|
||||
}
|
||||
]
|
||||
@@ -450,6 +450,10 @@
|
||||
"whisperIsWorkingGood": "Whisper is working good",
|
||||
"whisperIsNotWorking": "Whisper is not working",
|
||||
"whisperTranscribeFailed": "Whisper transcribe failed. Maybe your machine is not compatbile with it. Please use other STT service. {{error}}",
|
||||
"openaiTranscribeFailed": "OpenAI transcribe failed. {{error}}",
|
||||
"cloudflareTranscribeFailed": "Cloudflare transcribe failed. {{error}}",
|
||||
"azureTranscribeFailed": "Azure transcribe failed. {{error}}",
|
||||
"transcribeFailed": "Transcribe failed.",
|
||||
"checkingWhisperModel": "Checking whisper model",
|
||||
"whisperModelIsWorkingGood": "Whisper model is working good",
|
||||
"whisperModelIsNotWorking": "Whisper model is not working",
|
||||
|
||||
@@ -450,6 +450,10 @@
|
||||
"whisperIsWorkingGood": "Whisper 正常工作",
|
||||
"whisperIsNotWorking": "Whisper 无法正常工作,请尝试更换模型后重试,或联系开发者",
|
||||
"whisperTranscribeFailed": "Whisper 语音转文本失败。您的设备可能不兼容该功能,请选择其他语音转文本服务。{{error}}",
|
||||
"openaiTranscribeFailed": "OpenAI 语音转文本失败。{{error}}",
|
||||
"cloudflareTranscribeFailed": "Cloudflare 语音转文本失败。{{error}}",
|
||||
"azureTranscribeFailed": "Azure 语音转文本失败。{{error}}",
|
||||
"transcribeFailed": "语音转文本失败。",
|
||||
"checkingWhisperModel": "正在检查 Whisper 模型",
|
||||
"whisperModelIsWorkingGood": "Whisper 模型正常工作",
|
||||
"whisperModelIsNotWorking": "Whisper 模型无法正常工作,请尝试更换模型后重试,或联系开发者",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { ipcMain } from "electron";
|
||||
import * as Echogarden from "echogarden/dist/api/API.js";
|
||||
import { AlignmentOptions } from "echogarden/dist/api/API";
|
||||
import { AlignmentOptions, RecognitionOptions } from "echogarden/dist/api/API";
|
||||
import {
|
||||
encodeRawAudioToWave,
|
||||
decodeWaveToRawAudio,
|
||||
@@ -22,11 +22,19 @@ import settings from "@main/settings";
|
||||
import fs from "fs-extra";
|
||||
import ffmpegPath from "ffmpeg-static";
|
||||
import { enjoyUrlToPath, pathToEnjoyUrl } from "./utils";
|
||||
import { UserSetting } from "./db/models";
|
||||
import { UserSettingKeyEnum } from "@/types/enums";
|
||||
import { WHISPER_MODELS } from "@/constants";
|
||||
import { WhisperOptions } from "echogarden/dist/recognition/WhisperSTT.js";
|
||||
|
||||
Echogarden.setGlobalOption(
|
||||
"ffmpegPath",
|
||||
ffmpegPath.replace("app.asar", "app.asar.unpacked")
|
||||
);
|
||||
Echogarden.setGlobalOption(
|
||||
"packageBaseURL",
|
||||
"https://hf-mirror.com/echogarden/echogarden-packages/resolve/main/"
|
||||
);
|
||||
|
||||
const __filename = url.fileURLToPath(import.meta.url);
|
||||
/*
|
||||
@@ -38,6 +46,7 @@ const __dirname = path
|
||||
|
||||
const logger = log.scope("echogarden");
|
||||
class EchogardenWrapper {
|
||||
public recognize: typeof Echogarden.recognize;
|
||||
public align: typeof Echogarden.align;
|
||||
public alignSegments: typeof Echogarden.alignSegments;
|
||||
public denoise: typeof Echogarden.denoise;
|
||||
@@ -50,6 +59,7 @@ class EchogardenWrapper {
|
||||
public wordTimelineToSegmentSentenceTimeline: typeof wordTimelineToSegmentSentenceTimeline;
|
||||
|
||||
constructor() {
|
||||
this.recognize = Echogarden.recognize;
|
||||
this.align = Echogarden.align;
|
||||
this.alignSegments = Echogarden.alignSegments;
|
||||
this.denoise = Echogarden.denoise;
|
||||
@@ -63,14 +73,27 @@ class EchogardenWrapper {
|
||||
wordTimelineToSegmentSentenceTimeline;
|
||||
}
|
||||
|
||||
async check() {
|
||||
async check(
|
||||
options: RecognitionOptions = {
|
||||
engine: "whisper",
|
||||
whisper: {
|
||||
model: "tiny.en",
|
||||
language: "en",
|
||||
} as WhisperOptions,
|
||||
}
|
||||
) {
|
||||
const sampleFile = path.join(__dirname, "samples", "jfk.wav");
|
||||
try {
|
||||
const result = await this.align(
|
||||
sampleFile,
|
||||
"And so my fellow Americans ask not what your country can do for you",
|
||||
{}
|
||||
);
|
||||
const whisperModel = await UserSetting.get(UserSettingKeyEnum.WHISPER);
|
||||
if (WHISPER_MODELS.includes(whisperModel)) {
|
||||
options.whisper.model = whisperModel;
|
||||
}
|
||||
} catch (e) {
|
||||
logger.error(e);
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await this.recognize(sampleFile, options);
|
||||
logger.info(result);
|
||||
fs.writeJsonSync(
|
||||
path.join(settings.cachePath(), "echogarden-check.json"),
|
||||
@@ -78,10 +101,10 @@ class EchogardenWrapper {
|
||||
{ spaces: 2 }
|
||||
);
|
||||
|
||||
return true;
|
||||
return { success: true, log: "" };
|
||||
} catch (e) {
|
||||
logger.error(e);
|
||||
return false;
|
||||
return { success: false, log: e.message };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,6 +125,20 @@ class EchogardenWrapper {
|
||||
}
|
||||
|
||||
registerIpcHandlers() {
|
||||
ipcMain.handle(
|
||||
"echogarden-recognize",
|
||||
async (_event, url: string, options: RecognitionOptions) => {
|
||||
logger.debug("echogarden-recognize:", options);
|
||||
try {
|
||||
const input = enjoyUrlToPath(url);
|
||||
return await this.recognize(input, options);
|
||||
} catch (err) {
|
||||
logger.error(err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
ipcMain.handle(
|
||||
"echogarden-align",
|
||||
async (
|
||||
@@ -129,6 +166,9 @@ class EchogardenWrapper {
|
||||
options: AlignmentOptions
|
||||
) => {
|
||||
logger.debug("echogarden-align-segments:", timeline, options);
|
||||
if (typeof input === "string") {
|
||||
input = enjoyUrlToPath(input);
|
||||
}
|
||||
try {
|
||||
const rawAudio = await this.ensureRawAudio(input, 16000);
|
||||
return await this.alignSegments(rawAudio, timeline, options);
|
||||
@@ -182,8 +222,8 @@ class EchogardenWrapper {
|
||||
}
|
||||
);
|
||||
|
||||
ipcMain.handle("echogarden-check", async (_event) => {
|
||||
return this.check();
|
||||
ipcMain.handle("echogarden-check", async (_event, options: any) => {
|
||||
return this.check(options);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,330 +0,0 @@
|
||||
import { ipcMain } from "electron";
|
||||
import settings from "@main/settings";
|
||||
import path from "path";
|
||||
import { WHISPER_MODELS_OPTIONS, PROCESS_TIMEOUT } from "@/constants";
|
||||
import { exec, spawn } from "child_process";
|
||||
import fs from "fs-extra";
|
||||
import log from "@main/logger";
|
||||
import url from "url";
|
||||
import { enjoyUrlToPath } from "./utils";
|
||||
import { t } from "i18next";
|
||||
import { UserSetting } from "@main/db/models";
|
||||
import db from "@main/db";
|
||||
import { UserSettingKeyEnum } from "@/types/enums";
|
||||
|
||||
const __filename = url.fileURLToPath(import.meta.url);
|
||||
/*
|
||||
* whipser bin file will be in /app.asar.unpacked instead of /app.asar
|
||||
*/
|
||||
const __dirname = path
|
||||
.dirname(__filename)
|
||||
.replace("app.asar", "app.asar.unpacked");
|
||||
|
||||
const logger = log.scope("whisper");
|
||||
|
||||
class Whipser {
|
||||
private binMain: string;
|
||||
private bundledModelsDir: string;
|
||||
public config: WhisperConfigType;
|
||||
private abortController: AbortController;
|
||||
|
||||
constructor() {
|
||||
const customWhisperPath = path.join(
|
||||
settings.libraryPath(),
|
||||
"whisper",
|
||||
"main"
|
||||
);
|
||||
this.bundledModelsDir = path.join(__dirname, "lib", "whisper", "models");
|
||||
if (fs.existsSync(customWhisperPath)) {
|
||||
this.binMain = customWhisperPath;
|
||||
} else {
|
||||
this.binMain = path.join(__dirname, "lib", "whisper", "main");
|
||||
}
|
||||
this.initialize();
|
||||
}
|
||||
|
||||
async initialize() {
|
||||
const models = [];
|
||||
|
||||
const bundledModels = fs.readdirSync(this.bundledModelsDir);
|
||||
for (const file of bundledModels) {
|
||||
const model = WHISPER_MODELS_OPTIONS.find((m) => m.name == file);
|
||||
if (!model) continue;
|
||||
|
||||
models.push({
|
||||
...model,
|
||||
savePath: path.join(this.bundledModelsDir, file),
|
||||
});
|
||||
}
|
||||
|
||||
const dir = path.join(settings.libraryPath(), "whisper", "models");
|
||||
fs.ensureDirSync(dir);
|
||||
const files = fs.readdirSync(dir);
|
||||
for (const file of files) {
|
||||
const model = WHISPER_MODELS_OPTIONS.find((m) => m.name == file);
|
||||
if (!model) continue;
|
||||
|
||||
models.push({
|
||||
...model,
|
||||
savePath: path.join(dir, file),
|
||||
});
|
||||
}
|
||||
|
||||
if (db.connection) {
|
||||
const whisperConfig = (await UserSetting.get(
|
||||
UserSettingKeyEnum.WHISPER
|
||||
)) as string;
|
||||
this.config = {
|
||||
model: whisperConfig || models[0].name,
|
||||
availableModels: models,
|
||||
modelsPath: dir,
|
||||
};
|
||||
} else {
|
||||
this.config = {
|
||||
model: models[0].name,
|
||||
availableModels: models,
|
||||
modelsPath: dir,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
currentModel() {
|
||||
if (!this.config.availableModels) return;
|
||||
|
||||
let model: WhisperConfigType["availableModels"][0];
|
||||
if (this.config.model) {
|
||||
model = (this.config.availableModels || []).find(
|
||||
(m) => m.name === this.config.model
|
||||
);
|
||||
}
|
||||
if (!model) {
|
||||
model = this.config.availableModels[0];
|
||||
this.config = Object.assign({}, this.config, { model: model.name });
|
||||
UserSetting.set(UserSettingKeyEnum.WHISPER, model.name);
|
||||
}
|
||||
|
||||
return model;
|
||||
}
|
||||
|
||||
async check() {
|
||||
this.abortController?.abort();
|
||||
this.abortController = new AbortController();
|
||||
|
||||
const model = this.currentModel();
|
||||
logger.debug(`Checking whisper model: ${model.savePath}`);
|
||||
|
||||
const sampleFile = path.join(__dirname, "samples", "jfk.wav");
|
||||
const tmpDir = settings.cachePath();
|
||||
const outputFile = path.join(tmpDir, "jfk.json");
|
||||
fs.rmSync(outputFile, { force: true });
|
||||
return new Promise((resolve, _reject) => {
|
||||
const commands = [
|
||||
`"${this.binMain}"`,
|
||||
`--file "${sampleFile}"`,
|
||||
`--model "${model.savePath}"`,
|
||||
"--output-json",
|
||||
`--output-file "${path.join(tmpDir, "jfk")}"`,
|
||||
];
|
||||
logger.debug(`Checking whisper command: ${commands.join(" ")}`);
|
||||
exec(
|
||||
commands.join(" "),
|
||||
{
|
||||
timeout: PROCESS_TIMEOUT,
|
||||
signal: this.abortController.signal,
|
||||
},
|
||||
(error, stdout, stderr) => {
|
||||
if (error) {
|
||||
logger.error("error", error);
|
||||
}
|
||||
|
||||
if (stderr) {
|
||||
logger.info("stderr", stderr);
|
||||
}
|
||||
|
||||
if (stdout) {
|
||||
logger.debug(stdout);
|
||||
}
|
||||
|
||||
resolve({
|
||||
success: fs.existsSync(outputFile),
|
||||
log: `${error?.message || ""}\n${stderr}\n${stdout}`,
|
||||
});
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/* Ensure the file is in wav format
|
||||
* and 16kHz sample rate
|
||||
*/
|
||||
async transcribe(
|
||||
params: {
|
||||
file?: string;
|
||||
blob?: {
|
||||
type: string;
|
||||
arrayBuffer: ArrayBuffer;
|
||||
};
|
||||
},
|
||||
options?: {
|
||||
language?: string;
|
||||
force?: boolean;
|
||||
extra?: string[];
|
||||
onProgress?: (progress: number) => void;
|
||||
}
|
||||
): Promise<Partial<WhisperOutputType>> {
|
||||
logger.debug("transcribing from local");
|
||||
|
||||
this.abortController?.abort();
|
||||
this.abortController = new AbortController();
|
||||
|
||||
const { blob } = params;
|
||||
let { file } = params;
|
||||
|
||||
if (file) {
|
||||
file = enjoyUrlToPath(file);
|
||||
} else if (blob) {
|
||||
const format = blob.type.split("/")[1];
|
||||
if (format !== "wav") {
|
||||
throw new Error("Only wav format is supported");
|
||||
}
|
||||
|
||||
file = path.join(settings.cachePath(), `${Date.now()}.${format}`);
|
||||
await fs.outputFile(file, Buffer.from(blob.arrayBuffer));
|
||||
} else {
|
||||
throw new Error("No file or blob provided");
|
||||
}
|
||||
|
||||
const { force = false, extra = [], language, onProgress } = options || {};
|
||||
|
||||
const model = this.currentModel();
|
||||
if (language && !language.startsWith("en") && model.name.includes("en")) {
|
||||
throw new Error(`Model ${model.name} does not support ${language}`);
|
||||
}
|
||||
|
||||
const filename = path.basename(file, path.extname(file));
|
||||
const tmpDir = settings.cachePath();
|
||||
const outputFile = path.join(tmpDir, filename + ".json");
|
||||
|
||||
logger.info(`Trying to transcribe ${file} to ${outputFile}`);
|
||||
if (fs.pathExistsSync(outputFile) && !force) {
|
||||
logger.info(`File ${outputFile} already exists`);
|
||||
return fs.readJson(outputFile);
|
||||
}
|
||||
|
||||
const commandArguments = [
|
||||
"--file",
|
||||
file,
|
||||
"--model",
|
||||
model.savePath,
|
||||
"--output-json",
|
||||
"--output-file",
|
||||
path.join(tmpDir, filename),
|
||||
"--print-progress",
|
||||
"--language",
|
||||
model.name.includes("en") ? "en" : language?.split("-")?.[0] || "auto",
|
||||
// `--split-on-word`,
|
||||
// `--max-len`,
|
||||
// "1",
|
||||
...extra,
|
||||
];
|
||||
|
||||
logger.info(
|
||||
`Running command: ${this.binMain} ${commandArguments.join(" ")}`
|
||||
);
|
||||
|
||||
const command = spawn(this.binMain, commandArguments, {
|
||||
timeout: PROCESS_TIMEOUT,
|
||||
signal: this.abortController.signal,
|
||||
});
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
command.stdout.on("data", (data) => {
|
||||
logger.debug(`stdout: ${data}`);
|
||||
});
|
||||
|
||||
command.stderr.on("data", (data) => {
|
||||
const output = data.toString();
|
||||
logger.info(`stderr: ${output}`);
|
||||
if (output.startsWith("whisper_print_progress_callback")) {
|
||||
const progress = parseInt(output.match(/\d+%/)?.[0] || "0");
|
||||
if (typeof progress === "number" && onProgress) onProgress(progress);
|
||||
}
|
||||
});
|
||||
|
||||
command.on("exit", (code) => {
|
||||
logger.info(`transcribe process exited with code ${code}`);
|
||||
});
|
||||
|
||||
command.on("error", (err) => {
|
||||
logger.error("transcribe error", err.message);
|
||||
reject(err);
|
||||
});
|
||||
|
||||
command.on("close", (code) => {
|
||||
if (code === 0 && fs.pathExistsSync(outputFile)) {
|
||||
resolve(fs.readJson(outputFile));
|
||||
} else {
|
||||
reject(new Error("Transcription failed"));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
abort() {
|
||||
this.abortController?.abort();
|
||||
}
|
||||
|
||||
registerIpcHandlers() {
|
||||
ipcMain.handle("whisper-config", async () => {
|
||||
await this.initialize();
|
||||
return this.config;
|
||||
});
|
||||
|
||||
ipcMain.handle("whisper-set-model", async (_event, model) => {
|
||||
const originalModel = this.config.model;
|
||||
this.config.model = model;
|
||||
|
||||
return this.check()
|
||||
.then(({ success, log }) => {
|
||||
if (success) {
|
||||
return Object.assign({}, this.config, { ready: true });
|
||||
} else {
|
||||
throw new Error(log);
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
this.config.model = originalModel;
|
||||
throw err;
|
||||
})
|
||||
.finally(() => {
|
||||
UserSetting.set(UserSettingKeyEnum.WHISPER, this.config.model);
|
||||
});
|
||||
});
|
||||
|
||||
ipcMain.handle("whisper-check", async (_event) => {
|
||||
return await this.check();
|
||||
});
|
||||
|
||||
ipcMain.handle("whisper-transcribe", async (event, params, options) => {
|
||||
return this.transcribe(params, {
|
||||
...options,
|
||||
onProgress: (progress) => {
|
||||
event.sender.send("whisper-on-progress", progress);
|
||||
},
|
||||
})
|
||||
.then((result) => {
|
||||
return result;
|
||||
})
|
||||
.catch((err) => {
|
||||
logger.error(err);
|
||||
throw t("whisperTranscribeFailed", { error: err.message });
|
||||
});
|
||||
});
|
||||
|
||||
ipcMain.handle("whisper-abort", async (_event) => {
|
||||
return await this.abort();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export default new Whipser();
|
||||
@@ -12,7 +12,6 @@ import path from "path";
|
||||
import db from "@main/db";
|
||||
import settings from "@main/settings";
|
||||
import downloader from "@main/downloader";
|
||||
import whisper from "@main/whisper";
|
||||
import fs from "fs-extra";
|
||||
import log from "@main/logger";
|
||||
import { REPO_URL, WS_URL } from "@/constants";
|
||||
@@ -62,9 +61,6 @@ main.init = async () => {
|
||||
// echogarden
|
||||
echogarden.registerIpcHandlers();
|
||||
|
||||
// Whisper
|
||||
whisper.registerIpcHandlers();
|
||||
|
||||
// Waveform
|
||||
waveform.registerIpcHandlers();
|
||||
|
||||
@@ -411,7 +407,6 @@ ${log}
|
||||
segments: path.join(settings.userDataPath(), "segments"),
|
||||
speeches: path.join(settings.userDataPath(), "speeches"),
|
||||
recordings: path.join(settings.userDataPath(), "recordings"),
|
||||
whisper: path.join(settings.libraryPath(), "whisper"),
|
||||
waveforms: path.join(settings.libraryPath(), "waveforms"),
|
||||
logs: path.join(settings.libraryPath(), "logs"),
|
||||
cache: settings.cachePath(),
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import { contextBridge, ipcRenderer, IpcRendererEvent } from "electron";
|
||||
import { version } from "../package.json";
|
||||
import { Timeline } from "echogarden/dist/utilities/Timeline";
|
||||
import { RecognitionOptions } from "echogarden/dist/api/API";
|
||||
|
||||
contextBridge.exposeInMainWorld("__ENJOY_APP__", {
|
||||
app: {
|
||||
@@ -472,6 +473,9 @@ contextBridge.exposeInMainWorld("__ENJOY_APP__", {
|
||||
},
|
||||
},
|
||||
echogarden: {
|
||||
recognize: (input: string, options: RecognitionOptions) => {
|
||||
return ipcRenderer.invoke("echogarden-recognize", input, options);
|
||||
},
|
||||
align: (input: string, transcript: string, options: any) => {
|
||||
return ipcRenderer.invoke("echogarden-align", input, transcript, options);
|
||||
},
|
||||
@@ -502,42 +506,6 @@ contextBridge.exposeInMainWorld("__ENJOY_APP__", {
|
||||
return ipcRenderer.invoke("echogarden-check");
|
||||
},
|
||||
},
|
||||
whisper: {
|
||||
config: () => {
|
||||
return ipcRenderer.invoke("whisper-config");
|
||||
},
|
||||
setModel: (model: string) => {
|
||||
return ipcRenderer.invoke("whisper-set-model", model);
|
||||
},
|
||||
check: () => {
|
||||
return ipcRenderer.invoke("whisper-check");
|
||||
},
|
||||
transcribe: (
|
||||
params: {
|
||||
file?: string;
|
||||
blob?: {
|
||||
type: string;
|
||||
arrayBuffer: ArrayBuffer;
|
||||
};
|
||||
},
|
||||
options?: {
|
||||
language?: string;
|
||||
force?: boolean;
|
||||
extra?: string[];
|
||||
}
|
||||
) => {
|
||||
return ipcRenderer.invoke("whisper-transcribe", params, options);
|
||||
},
|
||||
onProgress: (
|
||||
callback: (event: IpcRendererEvent, progress: number) => void
|
||||
) => ipcRenderer.on("whisper-on-progress", callback),
|
||||
abort: () => {
|
||||
return ipcRenderer.invoke("whisper-abort");
|
||||
},
|
||||
removeProgressListeners: () => {
|
||||
ipcRenderer.removeAllListeners("whisper-on-progress");
|
||||
},
|
||||
},
|
||||
ffmpeg: {
|
||||
check: () => {
|
||||
return ipcRenderer.invoke("ffmpeg-check-command");
|
||||
|
||||
@@ -79,6 +79,7 @@ export const MediaTranscriptionGenerateButton = (props: {
|
||||
generateTranscription({
|
||||
originalText: data.text,
|
||||
language: data.language,
|
||||
model: data.model,
|
||||
service: data.service as SttEngineOptionEnum | "upload",
|
||||
isolate: data.isolate,
|
||||
})
|
||||
|
||||
@@ -69,6 +69,7 @@ const LoadingContent = () => {
|
||||
generateTranscription({
|
||||
originalText: data.text,
|
||||
language: data.language,
|
||||
model: data.model,
|
||||
service: data.service as SttEngineOptionEnum | "upload",
|
||||
isolate: data.isolate,
|
||||
});
|
||||
|
||||
@@ -28,7 +28,6 @@ export * from "./theme-settings";
|
||||
|
||||
export * from "./proxy-settings";
|
||||
|
||||
export * from "./whisper-model-options";
|
||||
export * from "./network-state";
|
||||
|
||||
export * from "./recorder-settings";
|
||||
|
||||
@@ -6,7 +6,6 @@ import {
|
||||
DialogContent,
|
||||
DialogHeader,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
toast,
|
||||
Select,
|
||||
SelectTrigger,
|
||||
@@ -14,29 +13,26 @@ import {
|
||||
SelectItem,
|
||||
SelectValue,
|
||||
} from "@renderer/components/ui";
|
||||
import { WhisperModelOptions } from "@renderer/components";
|
||||
import {
|
||||
AppSettingsProviderContext,
|
||||
AISettingsProviderContext,
|
||||
} from "@renderer/context";
|
||||
import { useContext, useEffect, useState } from "react";
|
||||
import { InfoIcon, AlertCircleIcon } from "lucide-react";
|
||||
import { useContext, useState } from "react";
|
||||
import { AlertCircleIcon } from "lucide-react";
|
||||
import { SttEngineOptionEnum } from "@/types/enums";
|
||||
import { WHISPER_MODELS } from "@/constants";
|
||||
|
||||
export const WhisperSettings = () => {
|
||||
const { sttEngine, whisperConfig, refreshWhisperConfig, setSttEngine } =
|
||||
useContext(AISettingsProviderContext);
|
||||
const { sttEngine, whisperModel, setWhisperModel, setSttEngine } = useContext(
|
||||
AISettingsProviderContext
|
||||
);
|
||||
const { EnjoyApp } = useContext(AppSettingsProviderContext);
|
||||
const [stderr, setStderr] = useState("");
|
||||
|
||||
useEffect(() => {
|
||||
refreshWhisperConfig();
|
||||
}, []);
|
||||
|
||||
const handleCheck = async () => {
|
||||
toast.promise(
|
||||
async () => {
|
||||
const { success, log } = await EnjoyApp.whisper.check();
|
||||
const { success, log } = await EnjoyApp.echogarden.check();
|
||||
if (success) {
|
||||
setStderr("");
|
||||
return Promise.resolve();
|
||||
@@ -98,46 +94,28 @@ export const WhisperSettings = () => {
|
||||
</SelectContent>
|
||||
</Select>
|
||||
|
||||
{sttEngine === "local" && (
|
||||
{sttEngine === SttEngineOptionEnum.LOCAL && (
|
||||
<>
|
||||
<Select
|
||||
value={whisperModel}
|
||||
onValueChange={(value) => {
|
||||
setWhisperModel(value);
|
||||
}}
|
||||
>
|
||||
<SelectTrigger className="min-w-fit">
|
||||
<SelectValue placeholder="service"></SelectValue>
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{WHISPER_MODELS.map((model) => (
|
||||
<SelectItem key={model} value={model}>
|
||||
{model}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<Button onClick={handleCheck} variant="secondary" size="sm">
|
||||
{t("check")}
|
||||
</Button>
|
||||
<Dialog>
|
||||
<DialogTrigger asChild>
|
||||
<Button variant="secondary" size="sm">
|
||||
{t("model")}
|
||||
</Button>
|
||||
</DialogTrigger>
|
||||
<DialogContent>
|
||||
<DialogHeader>{t("sttAiService")}</DialogHeader>
|
||||
<DialogDescription>
|
||||
{t("chooseAIModelDependingOnYourHardware")}
|
||||
</DialogDescription>
|
||||
|
||||
<WhisperModelOptions />
|
||||
|
||||
<DialogFooter>
|
||||
<div className="text-xs flex items-start space-x-2">
|
||||
<InfoIcon className="mr-1.5 w-4 h-4" />
|
||||
<span className="flex-1 opacity-70">
|
||||
{t("yourModelsWillBeDownloadedTo", {
|
||||
path: whisperConfig.modelsPath,
|
||||
})}
|
||||
</span>
|
||||
<Button
|
||||
onClick={() => {
|
||||
EnjoyApp.shell.openPath(whisperConfig?.modelsPath);
|
||||
}}
|
||||
variant="outline"
|
||||
size="sm"
|
||||
>
|
||||
{t("open")}
|
||||
</Button>
|
||||
</div>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -1,196 +0,0 @@
|
||||
import {
|
||||
AlertDialog,
|
||||
AlertDialogTitle,
|
||||
AlertDialogHeader,
|
||||
AlertDialogFooter,
|
||||
AlertDialogDescription,
|
||||
AlertDialogContent,
|
||||
AlertDialogAction,
|
||||
AlertDialogCancel,
|
||||
Button,
|
||||
ScrollArea,
|
||||
toast,
|
||||
Progress,
|
||||
} from "@renderer/components/ui";
|
||||
import { t } from "i18next";
|
||||
import { CheckCircle, DownloadIcon, XCircleIcon } from "lucide-react";
|
||||
import { WHISPER_MODELS_OPTIONS } from "@/constants";
|
||||
import { useState, useContext, useEffect } from "react";
|
||||
import {
|
||||
AppSettingsProviderContext,
|
||||
AISettingsProviderContext,
|
||||
} from "@renderer/context";
|
||||
|
||||
type ModelType = {
|
||||
type: string;
|
||||
name: string;
|
||||
size: string;
|
||||
url: string;
|
||||
downloaded?: boolean;
|
||||
downloadState?: DownloadStateType;
|
||||
};
|
||||
|
||||
export const WhisperModelOptions = () => {
|
||||
const [selectingModel, setSelectingModel] = useState<ModelType | null>(null);
|
||||
const [availableModels, setAvailableModels] = useState<ModelType[]>([]);
|
||||
const { EnjoyApp } = useContext(AppSettingsProviderContext);
|
||||
const { whisperConfig, setWhisperModel } = useContext(
|
||||
AISettingsProviderContext
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
updateAvailableModels();
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
listenToDownloadState();
|
||||
|
||||
return () => {
|
||||
EnjoyApp.download.removeAllListeners();
|
||||
};
|
||||
}, [selectingModel]);
|
||||
|
||||
const updateAvailableModels = async () => {
|
||||
const models = whisperConfig.availableModels;
|
||||
const options: ModelType[] = WHISPER_MODELS_OPTIONS;
|
||||
|
||||
options.forEach((o) => {
|
||||
o.downloaded = models.findIndex((m) => m.name === o.name) > -1;
|
||||
});
|
||||
setAvailableModels(options);
|
||||
};
|
||||
|
||||
const downloadModel = async () => {
|
||||
if (!selectingModel) return;
|
||||
|
||||
const model = WHISPER_MODELS_OPTIONS.find(
|
||||
(m) => m.name === selectingModel.name
|
||||
);
|
||||
|
||||
EnjoyApp.download.start(model.url, whisperConfig.modelsPath);
|
||||
|
||||
setSelectingModel(null);
|
||||
};
|
||||
|
||||
const listenToDownloadState = () => {
|
||||
EnjoyApp.download.onState((_event, state) => {
|
||||
const model = availableModels.find((m) => m.name === state.name);
|
||||
if (!model) return;
|
||||
|
||||
if (model) {
|
||||
model.downloadState = state;
|
||||
}
|
||||
if (state.state === "completed") {
|
||||
model.downloaded = true;
|
||||
setWhisperModel(model.name);
|
||||
} else if (state.state === "cancelled" || state.state === "interrupted") {
|
||||
model.downloaded = false;
|
||||
model.downloadState = null;
|
||||
}
|
||||
|
||||
setAvailableModels([...availableModels]);
|
||||
});
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<ScrollArea className="max-h-96">
|
||||
{availableModels.map((option) => {
|
||||
return (
|
||||
<div key={option.name}>
|
||||
<div
|
||||
className={`cursor-pointer hover:bg-secondary px-4 py-2 rounded ${
|
||||
whisperConfig.model === option.name ? "bg-secondary" : ""
|
||||
}`}
|
||||
onClick={() => {
|
||||
if (option.downloaded) {
|
||||
toast.promise(setWhisperModel(option.name), {
|
||||
loading: t("checkingWhisperModel"),
|
||||
success: t("whisperModelIsWorkingGood"),
|
||||
error: t("whisperModelIsNotWorking"),
|
||||
});
|
||||
} else if (!option.downloadState) {
|
||||
setSelectingModel(option);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<div className="flex justify-between">
|
||||
<div className="font-semibold">{option.type}</div>
|
||||
{option.downloaded ? (
|
||||
<CheckCircle
|
||||
className={`w-4 ${
|
||||
whisperConfig.model === option.name
|
||||
? "text-green-500"
|
||||
: ""
|
||||
}`}
|
||||
/>
|
||||
) : (
|
||||
<DownloadIcon className="w-4 opacity-70" />
|
||||
)}
|
||||
</div>
|
||||
<div className="text-sm opacity-70 flex justify-between">
|
||||
<span>{option.name}</span>
|
||||
<span>~{option.size}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{!option.downloaded && option.downloadState && (
|
||||
<div className="flex items-center space-x-2 py-2 px-4">
|
||||
<Progress
|
||||
className="h-1"
|
||||
value={
|
||||
(option.downloadState.received /
|
||||
option.downloadState.total) *
|
||||
100
|
||||
}
|
||||
/>
|
||||
<Button
|
||||
onClick={() => {
|
||||
toast.promise(
|
||||
EnjoyApp.download.cancel(option.downloadState.name),
|
||||
{
|
||||
loading: t("cancelling"),
|
||||
success: t("cancelled"),
|
||||
}
|
||||
);
|
||||
}}
|
||||
className=""
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
>
|
||||
<XCircleIcon className="w-4 h-4" />
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</ScrollArea>
|
||||
|
||||
<AlertDialog open={!!selectingModel}>
|
||||
<AlertDialogContent>
|
||||
<AlertDialogHeader>
|
||||
<AlertDialogTitle>{t("download")}</AlertDialogTitle>
|
||||
<AlertDialogDescription>
|
||||
{t("areYouSureToDownload", {
|
||||
name: `${selectingModel?.name}(${selectingModel?.size})`,
|
||||
})}
|
||||
</AlertDialogDescription>
|
||||
</AlertDialogHeader>
|
||||
<AlertDialogFooter>
|
||||
<AlertDialogCancel
|
||||
onClick={() => {
|
||||
setSelectingModel(null);
|
||||
}}
|
||||
>
|
||||
{t("cancel")}
|
||||
</AlertDialogCancel>
|
||||
<AlertDialogAction onClick={() => downloadModel()}>
|
||||
{t("download")}
|
||||
</AlertDialogAction>
|
||||
</AlertDialogFooter>
|
||||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
</>
|
||||
);
|
||||
};
|
||||
@@ -30,7 +30,7 @@ import {
|
||||
toast,
|
||||
} from "@renderer/components/ui";
|
||||
import { t } from "i18next";
|
||||
import { LANGUAGES } from "@/constants";
|
||||
import { LANGUAGES, WHISPER_MODELS } from "@/constants";
|
||||
import { ChevronDownIcon, ChevronUpIcon, LoaderIcon } from "lucide-react";
|
||||
import { parseText } from "media-captions";
|
||||
import { milisecondsToTimestamp } from "@/utils";
|
||||
@@ -39,6 +39,7 @@ import { SttEngineOptionEnum } from "@/types/enums";
|
||||
const transcriptionSchema = z.object({
|
||||
language: z.string(),
|
||||
service: z.union([z.nativeEnum(SttEngineOptionEnum), z.literal("upload")]),
|
||||
model: z.string().optional(),
|
||||
text: z.string().optional(),
|
||||
isolate: z.boolean().optional(),
|
||||
});
|
||||
@@ -60,13 +61,14 @@ export const TranscriptionCreateForm = (props: {
|
||||
originalText,
|
||||
} = props;
|
||||
const { learningLanguage } = useContext(AppSettingsProviderContext);
|
||||
const { sttEngine } = useContext(AISettingsProviderContext);
|
||||
const { sttEngine, whisperModel } = useContext(AISettingsProviderContext);
|
||||
|
||||
const form = useForm<z.infer<typeof transcriptionSchema>>({
|
||||
resolver: zodResolver(transcriptionSchema),
|
||||
values: {
|
||||
language: learningLanguage,
|
||||
service: originalText ? "upload" : sttEngine,
|
||||
model: sttEngine === SttEngineOptionEnum.LOCAL ? whisperModel : "",
|
||||
text: originalText,
|
||||
isolate: false,
|
||||
},
|
||||
@@ -197,6 +199,35 @@ export const TranscriptionCreateForm = (props: {
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
{form.watch("service") === SttEngineOptionEnum.LOCAL && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="model"
|
||||
render={({ field }) => (
|
||||
<FormItem className="grid w-full items-center">
|
||||
<FormLabel>{t("model")}</FormLabel>
|
||||
<Select
|
||||
disabled={transcribing}
|
||||
value={field.value}
|
||||
onValueChange={field.onChange}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{WHISPER_MODELS.map((model) => (
|
||||
<SelectItem key={model} value={model}>
|
||||
{model}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="language"
|
||||
|
||||
@@ -5,13 +5,13 @@ import {
|
||||
} from "@renderer/context";
|
||||
import { SttEngineOptionEnum, UserSettingKeyEnum } from "@/types/enums";
|
||||
import { GPT_PROVIDERS, TTS_PROVIDERS } from "@renderer/components";
|
||||
import { WHISPER_MODELS } from "@/constants";
|
||||
|
||||
type AISettingsProviderState = {
|
||||
setWhisperModel?: (name: string) => Promise<void>;
|
||||
sttEngine?: SttEngineOptionEnum;
|
||||
setSttEngine?: (name: string) => Promise<void>;
|
||||
whisperConfig?: WhisperConfigType;
|
||||
refreshWhisperConfig?: () => void;
|
||||
whisperModel?: string;
|
||||
setWhisperModel?: (name: string) => void;
|
||||
openai?: LlmProviderType;
|
||||
setOpenai?: (config: LlmProviderType) => void;
|
||||
setGptEngine?: (engine: GptEngineSettingType) => void;
|
||||
@@ -38,7 +38,7 @@ export const AISettingsProvider = ({
|
||||
},
|
||||
});
|
||||
const [openai, setOpenai] = useState<LlmProviderType>(null);
|
||||
const [whisperConfig, setWhisperConfig] = useState<WhisperConfigType>(null);
|
||||
const [whisperModel, setWhisperModel] = useState<string>(null);
|
||||
const [sttEngine, setSttEngine] = useState<SttEngineOptionEnum>(
|
||||
SttEngineOptionEnum.ENJOY_AZURE
|
||||
);
|
||||
@@ -87,6 +87,44 @@ export const AISettingsProvider = ({
|
||||
setTtsProviders({ ...providers });
|
||||
};
|
||||
|
||||
const refreshWhisperModel = async () => {
|
||||
const whisperModel = await EnjoyApp.userSettings.get(
|
||||
UserSettingKeyEnum.WHISPER
|
||||
);
|
||||
if (WHISPER_MODELS.includes(whisperModel)) {
|
||||
setWhisperModel(whisperModel);
|
||||
} else {
|
||||
let model = "tiny";
|
||||
if (whisperModel.match(/tiny/)) {
|
||||
model = "tiny";
|
||||
} else if (whisperModel.match(/base/)) {
|
||||
model = "base";
|
||||
} else if (whisperModel.match(/small/)) {
|
||||
model = "small";
|
||||
} else if (whisperModel.match(/medium/)) {
|
||||
model = "medium";
|
||||
} else if (whisperModel.match(/large/)) {
|
||||
model = "large-v3-turbo";
|
||||
}
|
||||
|
||||
if (
|
||||
learningLanguage.match(/en/) &&
|
||||
model.match(/tiny|base|small|medium/)
|
||||
) {
|
||||
model = `${model}.en`;
|
||||
}
|
||||
|
||||
setWhisperModel(model);
|
||||
}
|
||||
};
|
||||
|
||||
const handleSetWhisperModel = async (name: string) => {
|
||||
if (WHISPER_MODELS.includes(name)) {
|
||||
setWhisperModel(name);
|
||||
EnjoyApp.userSettings.set(UserSettingKeyEnum.WHISPER, name);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
refreshGptProviders();
|
||||
refreshTtsProviders();
|
||||
@@ -101,22 +139,8 @@ export const AISettingsProvider = ({
|
||||
useEffect(() => {
|
||||
if (db.state !== "connected") return;
|
||||
if (!libraryPath) return;
|
||||
|
||||
refreshWhisperConfig();
|
||||
}, [db.state, libraryPath]);
|
||||
|
||||
const refreshWhisperConfig = async () => {
|
||||
const config = await EnjoyApp.whisper.config();
|
||||
setWhisperConfig(config);
|
||||
};
|
||||
|
||||
const setWhisperModel = async (name: string) => {
|
||||
return EnjoyApp.whisper.setModel(name).then((config) => {
|
||||
if (!config) return;
|
||||
setWhisperConfig(config);
|
||||
});
|
||||
};
|
||||
|
||||
const handleSetSttEngine = async (name: SttEngineOptionEnum) => {
|
||||
setSttEngine(name);
|
||||
return EnjoyApp.userSettings.set(UserSettingKeyEnum.STT_ENGINE, name);
|
||||
@@ -165,6 +189,8 @@ export const AISettingsProvider = ({
|
||||
setGptEngine(engine);
|
||||
});
|
||||
}
|
||||
|
||||
refreshWhisperModel();
|
||||
};
|
||||
|
||||
const handleSetOpenai = async (config: LlmProviderType) => {
|
||||
@@ -208,9 +234,8 @@ export const AISettingsProvider = ({
|
||||
},
|
||||
openai,
|
||||
setOpenai: (config: LlmProviderType) => handleSetOpenai(config),
|
||||
whisperConfig,
|
||||
refreshWhisperConfig,
|
||||
setWhisperModel,
|
||||
whisperModel,
|
||||
setWhisperModel: handleSetWhisperModel,
|
||||
sttEngine,
|
||||
setSttEngine: (name: SttEngineOptionEnum) => handleSetSttEngine(name),
|
||||
gptProviders,
|
||||
|
||||
@@ -62,6 +62,7 @@ type MediaShadowContextType = {
|
||||
generateTranscription: (params?: {
|
||||
originalText?: string;
|
||||
language?: string;
|
||||
model?: string;
|
||||
service?: SttEngineOptionEnum | "upload";
|
||||
isolate?: boolean;
|
||||
}) => Promise<void>;
|
||||
|
||||
@@ -11,14 +11,14 @@ import axios from "axios";
|
||||
import { useAiCommand } from "./use-ai-command";
|
||||
import { toast } from "@renderer/components/ui";
|
||||
import {
|
||||
Timeline,
|
||||
TimelineEntry,
|
||||
type TimelineEntryType,
|
||||
} from "echogarden/dist/utilities/Timeline";
|
||||
import take from "lodash/take";
|
||||
import sortedUniqBy from "lodash/sortedUniqBy";
|
||||
import { parseText } from "media-captions";
|
||||
import { SttEngineOptionEnum } from "@/types/enums";
|
||||
import { RecognitionResult } from "echogarden/dist/api/API.js";
|
||||
import take from "lodash/take";
|
||||
import sortedUniqBy from "lodash/sortedUniqBy";
|
||||
|
||||
// test a text string has any punctuations or not
|
||||
// some transcribed text may not have any punctuations
|
||||
@@ -26,7 +26,7 @@ const punctuationsPattern = /\w[.,!?](\s|$)/g;
|
||||
|
||||
export const useTranscribe = () => {
|
||||
const { EnjoyApp, user, webApi } = useContext(AppSettingsProviderContext);
|
||||
const { openai } = useContext(AISettingsProviderContext);
|
||||
const { openai, whisperModel } = useContext(AISettingsProviderContext);
|
||||
const { punctuateText } = useAiCommand();
|
||||
const [output, setOutput] = useState<string>("");
|
||||
|
||||
@@ -47,6 +47,7 @@ export const useTranscribe = () => {
|
||||
params?: {
|
||||
targetId?: string;
|
||||
targetType?: string;
|
||||
model?: string;
|
||||
originalText?: string;
|
||||
language: string;
|
||||
service: SttEngineOptionEnum | "upload";
|
||||
@@ -64,6 +65,7 @@ export const useTranscribe = () => {
|
||||
}> => {
|
||||
const url = await transcode(mediaSrc);
|
||||
const {
|
||||
model,
|
||||
targetId,
|
||||
targetType,
|
||||
originalText,
|
||||
@@ -75,34 +77,11 @@ export const useTranscribe = () => {
|
||||
const blob = await (await fetch(url)).blob();
|
||||
|
||||
let result: any;
|
||||
let timeline: Timeline = [];
|
||||
|
||||
if (service === "upload" && originalText) {
|
||||
const caption = await parseText(originalText, { type: "srt" });
|
||||
if (caption.cues.length > 0) {
|
||||
timeline = caption.cues.map((cue) => {
|
||||
return {
|
||||
type: "sentence",
|
||||
text: cue.text,
|
||||
startTime: cue.startTime,
|
||||
endTime: cue.endTime,
|
||||
timeline: [],
|
||||
};
|
||||
});
|
||||
result = {
|
||||
engine: "upload",
|
||||
model: "-",
|
||||
text: timeline.map((entry) => entry.text).join(" "),
|
||||
timeline,
|
||||
};
|
||||
} else {
|
||||
result = {
|
||||
engine: "upload",
|
||||
model: "-",
|
||||
text: originalText,
|
||||
};
|
||||
}
|
||||
result = await alignText(originalText);
|
||||
} else if (service === SttEngineOptionEnum.LOCAL) {
|
||||
result = await transcribeByLocal(url, language);
|
||||
result = await transcribeByLocal(url, { language, model });
|
||||
} else if (service === SttEngineOptionEnum.ENJOY_CLOUDFLARE) {
|
||||
result = await transcribeByCloudflareAi(blob);
|
||||
} else if (service === SttEngineOptionEnum.OPENAI) {
|
||||
@@ -121,67 +100,47 @@ export const useTranscribe = () => {
|
||||
);
|
||||
}
|
||||
|
||||
let transcript = result.text;
|
||||
if (!align) {
|
||||
const { segmentTimeline, transcript } = result;
|
||||
|
||||
if (!align && transcript) {
|
||||
return {
|
||||
...result,
|
||||
transcript,
|
||||
url,
|
||||
timeline: [],
|
||||
};
|
||||
}
|
||||
|
||||
/*
|
||||
* if timeline is available and the transcript contains punctuations
|
||||
* use `alignSegments` to align each sentence with the timeline
|
||||
* otherwise, use `align` to align the whole transcript
|
||||
* if the transcript does not contain any punctuation, use AI command to add punctuation
|
||||
*/
|
||||
if (result.timeline?.length && transcript.match(punctuationsPattern)) {
|
||||
timeline = [...result.timeline];
|
||||
setOutput("Aligning the transcript...");
|
||||
if (segmentTimeline && segmentTimeline.length > 0) {
|
||||
const wordTimeline = await EnjoyApp.echogarden.alignSegments(
|
||||
new Uint8Array(await blob.arrayBuffer()),
|
||||
timeline,
|
||||
segmentTimeline,
|
||||
{
|
||||
language,
|
||||
language: language.split("-")[0],
|
||||
isolate,
|
||||
}
|
||||
);
|
||||
timeline = await EnjoyApp.echogarden.wordToSentenceTimeline(
|
||||
|
||||
const timeline = await EnjoyApp.echogarden.wordToSentenceTimeline(
|
||||
wordTimeline,
|
||||
transcript,
|
||||
language.split("-")[0]
|
||||
);
|
||||
} else {
|
||||
// Remove all content inside `()`, `[]`, `{}` and trim the text
|
||||
// remove all markdown formatting
|
||||
transcript = transcript
|
||||
.replace(/\(.*?\)/g, "")
|
||||
.replace(/\[.*?\]/g, "")
|
||||
.replace(/\{.*?\}/g, "")
|
||||
.replace(/[*_`]/g, "")
|
||||
.trim();
|
||||
|
||||
// if the transcript does not contain any punctuation, use AI command to add punctuation
|
||||
if (!transcript.match(punctuationsPattern)) {
|
||||
try {
|
||||
transcript = await punctuateText(transcript);
|
||||
} catch (err) {
|
||||
toast.error(err.message);
|
||||
console.warn(err.message);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
...result,
|
||||
timeline,
|
||||
};
|
||||
} else if (transcript) {
|
||||
setOutput("Aligning the transcript...");
|
||||
const alignmentResult = await EnjoyApp.echogarden.align(
|
||||
new Uint8Array(await blob.arrayBuffer()),
|
||||
transcript,
|
||||
{
|
||||
language,
|
||||
language: language.split("-")[0],
|
||||
isolate,
|
||||
}
|
||||
);
|
||||
|
||||
const timeline: TimelineEntry[] = [];
|
||||
alignmentResult.timeline.forEach((t: TimelineEntry) => {
|
||||
if (t.type === "sentence") {
|
||||
timeline.push(t);
|
||||
@@ -191,74 +150,120 @@ export const useTranscribe = () => {
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
...result,
|
||||
originalText,
|
||||
transcript,
|
||||
timeline,
|
||||
url,
|
||||
};
|
||||
return {
|
||||
...result,
|
||||
timeline,
|
||||
};
|
||||
} else {
|
||||
throw new Error(t("transcribeFailed"));
|
||||
}
|
||||
};
|
||||
|
||||
const alignText = async (
|
||||
originalText: string
|
||||
): Promise<{
|
||||
engine: string;
|
||||
model: string;
|
||||
transcript: string;
|
||||
segmentTimeline: TimelineEntry[];
|
||||
}> => {
|
||||
const caption = await parseText(originalText, { type: "srt" });
|
||||
|
||||
if (caption.cues.length > 0) {
|
||||
// valid srt file
|
||||
const segmentTimeline = caption.cues.map((cue) => {
|
||||
return {
|
||||
type: "segment",
|
||||
text: cue.text,
|
||||
startTime: cue.startTime,
|
||||
endTime: cue.endTime,
|
||||
timeline: [],
|
||||
} as TimelineEntry;
|
||||
});
|
||||
|
||||
return {
|
||||
engine: "upload",
|
||||
model: "-",
|
||||
transcript: segmentTimeline.map((entry) => entry.text).join(" "),
|
||||
segmentTimeline,
|
||||
};
|
||||
} else {
|
||||
// Remove all content inside `()`, `[]`, `{}` and trim the text
|
||||
// remove all markdown formatting
|
||||
let transcript = originalText
|
||||
.replace(/\(.*?\)/g, "")
|
||||
.replace(/\[.*?\]/g, "")
|
||||
.replace(/\{.*?\}/g, "")
|
||||
.replace(/[*_`]/g, "")
|
||||
.trim();
|
||||
|
||||
// if the transcript does not contain any punctuation, use AI command to add punctuation
|
||||
if (!transcript.match(punctuationsPattern)) {
|
||||
try {
|
||||
const punctuatedText = await punctuateText(transcript);
|
||||
transcript = punctuatedText;
|
||||
} catch (err) {
|
||||
toast.error(err.message);
|
||||
console.warn(err);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
engine: "upload",
|
||||
model: "-",
|
||||
transcript,
|
||||
segmentTimeline: [],
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
const transcribeByLocal = async (
|
||||
url: string,
|
||||
language?: string
|
||||
options: { language: string; model?: string }
|
||||
): Promise<{
|
||||
engine: string;
|
||||
model: string;
|
||||
text: string;
|
||||
timeline: TimelineEntry[];
|
||||
transcript: string;
|
||||
segmentTimeline: TimelineEntry[];
|
||||
}> => {
|
||||
const res = await EnjoyApp.whisper.transcribe(
|
||||
{
|
||||
file: url,
|
||||
},
|
||||
{
|
||||
language,
|
||||
force: true,
|
||||
extra: ["--prompt", `"Hello! Welcome to listen to this audio."`],
|
||||
}
|
||||
);
|
||||
|
||||
if (!res) {
|
||||
throw new Error(t("whisperTranscribeFailed", { error: "" }));
|
||||
let { language, model = whisperModel } = options || {};
|
||||
const languageCode = language.split("-")[0];
|
||||
if (model.match(/en/) && languageCode !== "en") {
|
||||
model = model.replace(".en", "");
|
||||
}
|
||||
|
||||
const timeline: TimelineEntry[] = res.transcription
|
||||
.map((segment) => {
|
||||
// ignore the word if it is empty or in the format of `[xxx]` or `(xxx)`
|
||||
if (
|
||||
!segment.text.trim() ||
|
||||
segment.text.trim().match(/^[\[\(].+[\]\)]$/)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
let res: RecognitionResult;
|
||||
try {
|
||||
res = await EnjoyApp.echogarden.recognize(url, {
|
||||
engine: "whisper",
|
||||
language: languageCode,
|
||||
whisper: {
|
||||
model,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
throw new Error(t("whisperTranscribeFailed", { error: err.message }));
|
||||
}
|
||||
|
||||
return {
|
||||
type: "segment" as TimelineEntryType,
|
||||
text: segment.text.trim(),
|
||||
startTime: segment.offsets.from / 1000.0,
|
||||
endTime: segment.offsets.to / 1000.0,
|
||||
};
|
||||
})
|
||||
.filter((s) => Boolean(s?.text));
|
||||
|
||||
const transcript = timeline
|
||||
.map((segment) => segment.text)
|
||||
.join(" ")
|
||||
.trim();
|
||||
const { transcript, timeline } = res;
|
||||
|
||||
return {
|
||||
engine: "whisper",
|
||||
model: res.model.type,
|
||||
text: transcript,
|
||||
timeline,
|
||||
model,
|
||||
transcript,
|
||||
segmentTimeline: timeline,
|
||||
};
|
||||
};
|
||||
|
||||
const transcribeByOpenAi = async (file: File) => {
|
||||
const transcribeByOpenAi = async (
|
||||
file: File
|
||||
): Promise<{
|
||||
engine: string;
|
||||
model: string;
|
||||
transcript: string;
|
||||
segmentTimeline: TimelineEntry[];
|
||||
}> => {
|
||||
if (!openai?.key) {
|
||||
throw new Error(t("openaiKeyRequired"));
|
||||
}
|
||||
@@ -270,38 +275,39 @@ export const useTranscribe = () => {
|
||||
maxRetries: 0,
|
||||
});
|
||||
|
||||
const res: {
|
||||
text: string;
|
||||
words?: { word: string; start: number; end: number }[];
|
||||
segments?: { text: string; start: number; end: number }[];
|
||||
} = (await client.audio.transcriptions.create({
|
||||
file,
|
||||
model: "whisper-1",
|
||||
response_format: "verbose_json",
|
||||
timestamp_granularities: ["segment"],
|
||||
})) as any;
|
||||
setOutput("Transcribing from OpenAI...");
|
||||
try {
|
||||
const res: {
|
||||
text: string;
|
||||
words?: { word: string; start: number; end: number }[];
|
||||
segments?: { text: string; start: number; end: number }[];
|
||||
} = (await client.audio.transcriptions.create({
|
||||
file,
|
||||
model: "whisper-1",
|
||||
response_format: "verbose_json",
|
||||
timestamp_granularities: ["word", "segment"],
|
||||
})) as any;
|
||||
|
||||
let timeline: TimelineEntry[] = [];
|
||||
if (res.segments) {
|
||||
res.segments.forEach((segment) => {
|
||||
const segmentTimeline = {
|
||||
setOutput("Aligning the transcript...");
|
||||
const segmentTimeline = (res.segments || []).map((segment) => {
|
||||
return {
|
||||
type: "segment" as TimelineEntryType,
|
||||
text: segment.text,
|
||||
startTime: segment.start,
|
||||
endTime: segment.end,
|
||||
timeline: [] as Timeline,
|
||||
timeline: [] as TimelineEntry[],
|
||||
};
|
||||
|
||||
timeline.push(segmentTimeline);
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
engine: "openai",
|
||||
model: "whisper-1",
|
||||
text: res.text,
|
||||
timeline,
|
||||
};
|
||||
return {
|
||||
engine: "openai",
|
||||
model: "whisper-1",
|
||||
transcript: res.text,
|
||||
segmentTimeline,
|
||||
};
|
||||
} catch (err) {
|
||||
throw new Error(t("openaiTranscribeFailed", { error: err.message }));
|
||||
}
|
||||
};
|
||||
|
||||
const transcribeByCloudflareAi = async (
|
||||
@@ -309,35 +315,47 @@ export const useTranscribe = () => {
|
||||
): Promise<{
|
||||
engine: string;
|
||||
model: string;
|
||||
text: string;
|
||||
timeline?: TimelineEntry[];
|
||||
transcript: string;
|
||||
segmentTimeline: TimelineEntry[];
|
||||
}> => {
|
||||
const res: CfWhipserOutputType = (
|
||||
await axios.postForm(`${AI_WORKER_ENDPOINT}/audio/transcriptions`, blob, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${user.accessToken}`,
|
||||
},
|
||||
timeout: 1000 * 60 * 5,
|
||||
})
|
||||
).data;
|
||||
setOutput("Transcribing from Cloudflare...");
|
||||
try {
|
||||
const res: CfWhipserOutputType = (
|
||||
await axios.postForm(
|
||||
`${AI_WORKER_ENDPOINT}/audio/transcriptions`,
|
||||
blob,
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${user.accessToken}`,
|
||||
},
|
||||
timeout: 1000 * 60 * 5,
|
||||
}
|
||||
)
|
||||
).data;
|
||||
|
||||
const segmentTimeline: TimelineEntry[] = [];
|
||||
if (res.vtt) {
|
||||
const caption = await parseText(res.vtt, { type: "vtt" });
|
||||
for (const cue of caption.cues) {
|
||||
segmentTimeline.push({
|
||||
type: "segment",
|
||||
text: cue.text,
|
||||
startTime: cue.startTime,
|
||||
endTime: cue.endTime,
|
||||
timeline: [],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const caption = await parseText(res.vtt, { type: "vtt" });
|
||||
const timeline: Timeline = caption.cues.map((cue) => {
|
||||
return {
|
||||
type: "segment",
|
||||
text: cue.text,
|
||||
startTime: cue.startTime,
|
||||
endTime: cue.endTime,
|
||||
timeline: [],
|
||||
engine: "cloudflare",
|
||||
model: "@cf/openai/whisper",
|
||||
transcript: res.text,
|
||||
segmentTimeline,
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
engine: "cloudflare",
|
||||
model: "@cf/openai/whisper",
|
||||
text: res.text,
|
||||
timeline,
|
||||
};
|
||||
} catch (err) {
|
||||
throw new Error(t("cloudflareTranscribeFailed", { error: err.message }));
|
||||
}
|
||||
};
|
||||
|
||||
const transcribeByAzureAi = async (
|
||||
@@ -350,9 +368,9 @@ export const useTranscribe = () => {
|
||||
): Promise<{
|
||||
engine: string;
|
||||
model: string;
|
||||
text: string;
|
||||
transcript: string;
|
||||
segmentTimeline: TimelineEntry[];
|
||||
tokenId: number;
|
||||
timeline?: TimelineEntry[];
|
||||
}> => {
|
||||
const { id, token, region } = await webApi.generateSpeechToken({
|
||||
...params,
|
||||
@@ -364,90 +382,92 @@ export const useTranscribe = () => {
|
||||
config.speechRecognitionLanguage = language;
|
||||
config.requestWordLevelTimestamps();
|
||||
config.outputFormat = sdk.OutputFormat.Detailed;
|
||||
config.setProfanity(sdk.ProfanityOption.Raw);
|
||||
|
||||
// create the speech recognizer.
|
||||
const reco = new sdk.SpeechRecognizer(config, audioConfig);
|
||||
|
||||
setOutput("Transcribing from Azure...");
|
||||
let results: SpeechRecognitionResultType[] = [];
|
||||
|
||||
const res: {
|
||||
engine: string;
|
||||
model: string;
|
||||
text: string;
|
||||
tokenId: number;
|
||||
timeline?: TimelineEntry[];
|
||||
} = await new Promise((resolve, reject) => {
|
||||
reco.recognizing = (_s, e) => {
|
||||
setOutput(e.result.text);
|
||||
};
|
||||
const { transcript, segmentTimeline }: any = await new Promise(
|
||||
(resolve, reject) => {
|
||||
reco.recognizing = (_s, e) => {
|
||||
setOutput((prev) => prev + e.result.text);
|
||||
};
|
||||
|
||||
reco.recognized = (_s, e) => {
|
||||
const json = e.result.properties.getProperty(
|
||||
sdk.PropertyId.SpeechServiceResponse_JsonResult
|
||||
);
|
||||
const result = JSON.parse(json);
|
||||
results = results.concat(result);
|
||||
};
|
||||
reco.recognized = (_s, e) => {
|
||||
const json = e.result.properties.getProperty(
|
||||
sdk.PropertyId.SpeechServiceResponse_JsonResult
|
||||
);
|
||||
const result = JSON.parse(json);
|
||||
results = results.concat(result);
|
||||
};
|
||||
|
||||
reco.canceled = (_s, e) => {
|
||||
if (e.reason === sdk.CancellationReason.Error) {
|
||||
return reject(new Error(e.errorDetails));
|
||||
}
|
||||
reco.canceled = (_s, e) => {
|
||||
if (e.reason === sdk.CancellationReason.Error) {
|
||||
return reject(new Error(e.errorDetails));
|
||||
}
|
||||
|
||||
reco.stopContinuousRecognitionAsync();
|
||||
console.log("CANCELED: Reason=" + e.reason);
|
||||
};
|
||||
reco.stopContinuousRecognitionAsync();
|
||||
console.log("CANCELED: Reason=" + e.reason);
|
||||
};
|
||||
|
||||
reco.sessionStopped = async (_s, e) => {
|
||||
console.log(
|
||||
"Session stopped. Stop continuous recognition.",
|
||||
e.sessionId,
|
||||
results
|
||||
);
|
||||
reco.stopContinuousRecognitionAsync();
|
||||
reco.sessionStopped = async (_s, e) => {
|
||||
console.log(
|
||||
"Session stopped. Stop continuous recognition.",
|
||||
e.sessionId,
|
||||
results
|
||||
);
|
||||
reco.stopContinuousRecognitionAsync();
|
||||
|
||||
try {
|
||||
const timeline: Timeline = [];
|
||||
results.forEach((result) => {
|
||||
if (!result.DisplayText) return;
|
||||
if (results.length === 0) {
|
||||
return reject(t("azureTranscribeFailed", { error: "" }));
|
||||
}
|
||||
|
||||
const best = take(sortedUniqBy(result.NBest, "Confidence"), 1)[0];
|
||||
if (!best.Words) return;
|
||||
if (!best.Confidence || best.Confidence < 0.5) return;
|
||||
try {
|
||||
const transcript = results
|
||||
.map((result) => result.DisplayText)
|
||||
.join(" ");
|
||||
const segmentTimeline: TimelineEntry[] = [];
|
||||
results.forEach((result) => {
|
||||
if (!result.DisplayText) return;
|
||||
|
||||
const firstWord = best.Words[0];
|
||||
const lastWord = best.Words[best.Words.length - 1];
|
||||
const best = take(sortedUniqBy(result.NBest, "Confidence"), 1)[0];
|
||||
if (!best.Words) return;
|
||||
if (!best.Confidence || best.Confidence < 0.5) return;
|
||||
|
||||
timeline.push({
|
||||
type: "sentence",
|
||||
text: best.Display,
|
||||
startTime: firstWord.Offset / 10000000.0,
|
||||
endTime: (lastWord.Offset + lastWord.Duration) / 10000000.0,
|
||||
timeline: [],
|
||||
const firstWord = best.Words[0];
|
||||
const lastWord = best.Words[best.Words.length - 1];
|
||||
|
||||
segmentTimeline.push({
|
||||
type: "segment",
|
||||
text: best.Display,
|
||||
startTime: firstWord.Offset / 10000000.0,
|
||||
endTime: (lastWord.Offset + lastWord.Duration) / 10000000.0,
|
||||
timeline: [],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
const transcript = timeline
|
||||
.map((result) => result.text)
|
||||
.join(" ")
|
||||
.trim();
|
||||
resolve({
|
||||
transcript,
|
||||
segmentTimeline,
|
||||
});
|
||||
} catch (err) {
|
||||
reject(t("azureTranscribeFailed", { error: err.message }));
|
||||
}
|
||||
};
|
||||
reco.startContinuousRecognitionAsync();
|
||||
}
|
||||
);
|
||||
|
||||
resolve({
|
||||
engine: "azure",
|
||||
model: "whisper",
|
||||
text: transcript,
|
||||
timeline,
|
||||
tokenId: id,
|
||||
});
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
};
|
||||
|
||||
reco.startContinuousRecognitionAsync();
|
||||
});
|
||||
|
||||
return res;
|
||||
return {
|
||||
engine: "azure",
|
||||
model: "whisper",
|
||||
transcript,
|
||||
segmentTimeline,
|
||||
tokenId: id,
|
||||
};
|
||||
};
|
||||
|
||||
return {
|
||||
|
||||
@@ -107,6 +107,7 @@ export const useTranscriptions = (media: AudioType | VideoType) => {
|
||||
const generateTranscription = async (params?: {
|
||||
originalText?: string;
|
||||
language?: string;
|
||||
model?: string;
|
||||
service?: SttEngineOptionEnum | "upload";
|
||||
isolate?: boolean;
|
||||
}) => {
|
||||
@@ -114,6 +115,7 @@ export const useTranscriptions = (media: AudioType | VideoType) => {
|
||||
originalText,
|
||||
language = learningLanguage,
|
||||
service = sttEngine,
|
||||
model,
|
||||
isolate = false,
|
||||
} = params || {};
|
||||
setService(service);
|
||||
@@ -279,26 +281,17 @@ export const useTranscriptions = (media: AudioType | VideoType) => {
|
||||
useEffect(() => {
|
||||
if (!transcribing) return;
|
||||
|
||||
if (service === "local") {
|
||||
EnjoyApp.whisper.onProgress((_, p: number) => {
|
||||
if (p > 100) p = 100;
|
||||
setTranscribingProgress(p);
|
||||
});
|
||||
}
|
||||
|
||||
EnjoyApp.app.onCmdOutput((_, output) => {
|
||||
setTranscribingOutput(output);
|
||||
});
|
||||
|
||||
return () => {
|
||||
EnjoyApp.whisper.removeProgressListeners();
|
||||
EnjoyApp.app.removeCmdOutputListeners();
|
||||
setTranscribingOutput(null);
|
||||
};
|
||||
}, [media, service, transcribing]);
|
||||
|
||||
const abortGenerateTranscription = () => {
|
||||
EnjoyApp.whisper.abort();
|
||||
setTranscribing(false);
|
||||
};
|
||||
|
||||
|
||||
25
enjoy/src/types/enjoy-app.d.ts
vendored
25
enjoy/src/types/enjoy-app.d.ts
vendored
@@ -273,6 +273,10 @@ type EnjoyAppType = {
|
||||
) => Promise<SpeechType>;
|
||||
};
|
||||
echogarden: {
|
||||
recognize: (
|
||||
input: string,
|
||||
options: RecognitionOptions
|
||||
) => Promise<RecognitionResult>;
|
||||
align: (
|
||||
input: string | Uint8Array,
|
||||
transcript: string,
|
||||
@@ -289,26 +293,7 @@ type EnjoyAppType = {
|
||||
language: string
|
||||
) => Promise<Timeline>;
|
||||
transcode: (input: string) => Promise<string>;
|
||||
check: () => Promise<boolean>;
|
||||
};
|
||||
whisper: {
|
||||
config: () => Promise<WhisperConfigType>;
|
||||
check: () => Promise<{ success: boolean; log: string }>;
|
||||
setModel: (model: string) => Promise<WhisperConfigType>;
|
||||
transcribe: (
|
||||
params: {
|
||||
file?: string;
|
||||
blob?: { type: string; arrayBuffer: ArrayBuffer };
|
||||
},
|
||||
options?: {
|
||||
language?: string;
|
||||
force?: boolean;
|
||||
extra?: string[];
|
||||
}
|
||||
) => Promise<Partial<WhisperOutputType>>;
|
||||
onProgress: (callback: (event, progress: number) => void) => void;
|
||||
abort: () => Promise<void>;
|
||||
removeProgressListeners: () => Promise<void>;
|
||||
check: (options?: any) => Promise<{ success: boolean; log: string }>;
|
||||
};
|
||||
ffmpeg: {
|
||||
check: () => Promise<boolean>;
|
||||
|
||||
23
enjoy/src/types/index.d.ts
vendored
23
enjoy/src/types/index.d.ts
vendored
@@ -235,3 +235,26 @@ type DictSettingType = {
|
||||
removing: string[];
|
||||
mdicts: MDict[];
|
||||
};
|
||||
|
||||
type TranscribeParamsType = {
|
||||
mediaSrc: string | Blob;
|
||||
params?: {
|
||||
targetId?: string;
|
||||
targetType?: string;
|
||||
originalText?: string;
|
||||
language: string;
|
||||
service: SttEngineOptionEnum | "upload";
|
||||
isolate?: boolean;
|
||||
align?: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
type TranscribeResultType = {
|
||||
engine: string;
|
||||
model: string;
|
||||
transcript: string;
|
||||
timeline: TimelineEntry[];
|
||||
originalText?: string;
|
||||
tokenId?: number;
|
||||
url: string;
|
||||
};
|
||||
|
||||
@@ -44,16 +44,6 @@ export default defineConfig((env) => {
|
||||
pluginHotRestart("restart"),
|
||||
viteStaticCopy({
|
||||
targets: [
|
||||
{
|
||||
src: `lib/whisper.cpp/${
|
||||
process.env.PACKAGE_OS_ARCH || os.arch()
|
||||
}/${os.platform()}/*`,
|
||||
dest: "lib/whisper",
|
||||
},
|
||||
{
|
||||
src: `lib/whisper.cpp/models/*`,
|
||||
dest: "lib/whisper/models",
|
||||
},
|
||||
{
|
||||
src: `lib/youtubedr/${
|
||||
process.env.PACKAGE_OS_ARCH || os.arch()
|
||||
|
||||
222
yarn.lock
222
yarn.lock
@@ -32,10 +32,10 @@ __metadata:
|
||||
"@nuxtjs/seo": "npm:^2.0.0-rc.23"
|
||||
autoprefixer: "npm:^10.4.20"
|
||||
nuxt: "npm:^3.13.2"
|
||||
nuxt-og-image: "npm:^3.0.4"
|
||||
nuxt-og-image: "npm:^3.0.6"
|
||||
postcss: "npm:^8.4.47"
|
||||
sass: "npm:^1.79.5"
|
||||
tailwindcss: "npm:^3.4.13"
|
||||
tailwindcss: "npm:^3.4.14"
|
||||
vue: "npm:^3.5.12"
|
||||
vue-router: "npm:^4.4.5"
|
||||
languageName: unknown
|
||||
@@ -363,7 +363,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@aws-sdk/client-polly@npm:^3.669.0":
|
||||
"@aws-sdk/client-polly@npm:^3.670.0":
|
||||
version: 3.670.0
|
||||
resolution: "@aws-sdk/client-polly@npm:3.670.0"
|
||||
dependencies:
|
||||
@@ -622,9 +622,9 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@aws-sdk/client-transcribe-streaming@npm:^3.669.0":
|
||||
version: 3.670.0
|
||||
resolution: "@aws-sdk/client-transcribe-streaming@npm:3.670.0"
|
||||
"@aws-sdk/client-transcribe-streaming@npm:^3.672.0":
|
||||
version: 3.672.0
|
||||
resolution: "@aws-sdk/client-transcribe-streaming@npm:3.672.0"
|
||||
dependencies:
|
||||
"@aws-crypto/sha256-browser": "npm:5.2.0"
|
||||
"@aws-crypto/sha256-js": "npm:5.2.0"
|
||||
@@ -674,7 +674,7 @@ __metadata:
|
||||
"@smithy/util-retry": "npm:^3.0.7"
|
||||
"@smithy/util-utf8": "npm:^3.0.0"
|
||||
tslib: "npm:^2.6.2"
|
||||
checksum: 10c0/24952fb07dddce9c82cb8082fd7382681e980d01a072b64278aa9c4d3e20b36b345ad813d4f890d902b3f34fcfc9c252c37f7af165e0e2b40c520cb797ad1852
|
||||
checksum: 10c0/8470177bcf36036a76db9567e01d747c7db5e4977cacee4577a1864da64cda069fd96971829e61a28fb98da34e07de55c0cb6e202443edb7d1504e72dcc53782
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
@@ -2380,17 +2380,6 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@es-joy/jsdoccomment@npm:^0.46.0":
|
||||
version: 0.46.0
|
||||
resolution: "@es-joy/jsdoccomment@npm:0.46.0"
|
||||
dependencies:
|
||||
comment-parser: "npm:1.4.1"
|
||||
esquery: "npm:^1.6.0"
|
||||
jsdoc-type-pratt-parser: "npm:~4.0.0"
|
||||
checksum: 10c0/a7a67936ebf6d9aaf74af018c3ac744769af3552b05ad9b88fca96b2ffdca16e724b0ff497f53634ec4cca81e98d8c471b6b6bde0fa5b725af4222ad9a0707f0
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@esbuild/aix-ppc64@npm:0.20.2":
|
||||
version: 0.20.2
|
||||
resolution: "@esbuild/aix-ppc64@npm:0.20.2"
|
||||
@@ -3839,7 +3828,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@nuxt/devtools-kit@npm:1.6.0, @nuxt/devtools-kit@npm:^1.4.1, @nuxt/devtools-kit@npm:^1.4.2, @nuxt/devtools-kit@npm:^1.5.1":
|
||||
"@nuxt/devtools-kit@npm:1.6.0, @nuxt/devtools-kit@npm:^1.4.1, @nuxt/devtools-kit@npm:^1.4.2, @nuxt/devtools-kit@npm:^1.5.1, @nuxt/devtools-kit@npm:^1.6.0":
|
||||
version: 1.6.0
|
||||
resolution: "@nuxt/devtools-kit@npm:1.6.0"
|
||||
dependencies:
|
||||
@@ -4670,14 +4659,14 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@playwright/test@npm:^1.48.0":
|
||||
version: 1.48.0
|
||||
resolution: "@playwright/test@npm:1.48.0"
|
||||
"@playwright/test@npm:^1.48.1":
|
||||
version: 1.48.1
|
||||
resolution: "@playwright/test@npm:1.48.1"
|
||||
dependencies:
|
||||
playwright: "npm:1.48.0"
|
||||
playwright: "npm:1.48.1"
|
||||
bin:
|
||||
playwright: cli.js
|
||||
checksum: 10c0/595a5db99f75d1ffce11df2980ca0dd20523a0a20dd962e4256e536dddccca0138883ae2e9c82aaa26094fb50def73062badc14c001d20fb62a3a0b2df18ebca
|
||||
checksum: 10c0/32cedc3b2d375cb8f4a830bc820d7726b0235be7a6202e1d6ee46e739b83666271c47c100c11311cf5a916468c18e6a4dc526accf9ef090786e7614c2633b2b8
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
@@ -6056,10 +6045,10 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@rails/actioncable@npm:7.2.100":
|
||||
version: 7.2.100
|
||||
resolution: "@rails/actioncable@npm:7.2.100"
|
||||
checksum: 10c0/dc24fcd1a9f153e5550c691ddff5ea7d3d7589a3be39cf14991d3fb6e77d1d952dbf9ab5f00c9781f85e750d94f7a80ade3231ab284b008d6de311ed33b21cb9
|
||||
"@rails/actioncable@npm:7.2.101":
|
||||
version: 7.2.101
|
||||
resolution: "@rails/actioncable@npm:7.2.101"
|
||||
checksum: 10c0/5917b0d5c51c8b38bb33532fd6483937d6bf030add46e2bca0272aed5b5b43ceec881f8c89a9784b416cb86a2655198f0faac4929a67caa4812ed622346cdc90
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
@@ -8048,7 +8037,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@unocss/preset-wind@npm:^0.63.1":
|
||||
"@unocss/preset-wind@npm:^0.63.1, @unocss/preset-wind@npm:^0.63.4":
|
||||
version: 0.63.4
|
||||
resolution: "@unocss/preset-wind@npm:0.63.4"
|
||||
dependencies:
|
||||
@@ -9748,12 +9737,12 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"chart.js@npm:^4.4.4":
|
||||
version: 4.4.4
|
||||
resolution: "chart.js@npm:4.4.4"
|
||||
"chart.js@npm:^4.4.5":
|
||||
version: 4.4.5
|
||||
resolution: "chart.js@npm:4.4.5"
|
||||
dependencies:
|
||||
"@kurkle/color": "npm:^0.3.0"
|
||||
checksum: 10c0/9fa3206403a6103916f7762c2665d322c42b0cc07fba91526b1d033ddb887c1ba74b3ebc0bd0748a9e55abd1017f25fdb2292cdd6579d8c2d3bcb1c58f71281c
|
||||
checksum: 10c0/042393725d1dc6a7fc55757abc5db88cbce0e84ad115c08d552af56d72197e806eb78e04253d12a5f0cf331a57a059dc30fc2dab9324e4f695806b3a087d1b0c
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
@@ -10251,13 +10240,6 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"comment-parser@npm:1.4.1":
|
||||
version: 1.4.1
|
||||
resolution: "comment-parser@npm:1.4.1"
|
||||
checksum: 10c0/d6c4be3f5be058f98b24f2d557f745d8fe1cc9eb75bebbdccabd404a0e1ed41563171b16285f593011f8b6a5ec81f564fb1f2121418ac5cbf0f49255bf0840dd
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"commondir@npm:^1.0.1":
|
||||
version: 1.0.1
|
||||
resolution: "commondir@npm:1.0.1"
|
||||
@@ -11725,12 +11707,12 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"echogarden@npm:^1.8.1":
|
||||
version: 1.8.1
|
||||
resolution: "echogarden@npm:1.8.1"
|
||||
"echogarden@npm:^1.8.4":
|
||||
version: 1.8.4
|
||||
resolution: "echogarden@npm:1.8.4"
|
||||
dependencies:
|
||||
"@aws-sdk/client-polly": "npm:^3.669.0"
|
||||
"@aws-sdk/client-transcribe-streaming": "npm:^3.669.0"
|
||||
"@aws-sdk/client-polly": "npm:^3.670.0"
|
||||
"@aws-sdk/client-transcribe-streaming": "npm:^3.672.0"
|
||||
"@echogarden/espeak-ng-emscripten": "npm:^0.1.2"
|
||||
"@echogarden/fasttext-wasm": "npm:^0.1.0"
|
||||
"@echogarden/flite-wasi": "npm:^0.1.1"
|
||||
@@ -11760,7 +11742,7 @@ __metadata:
|
||||
jsdom: "npm:^25.0.1"
|
||||
json5: "npm:^2.2.3"
|
||||
kuromoji: "npm:^0.1.2"
|
||||
microsoft-cognitiveservices-speech-sdk: "npm:^1.40.0"
|
||||
microsoft-cognitiveservices-speech-sdk: "npm:^1.41.0"
|
||||
moving-median: "npm:^1.0.0"
|
||||
msgpack-lite: "npm:^0.1.26"
|
||||
onnxruntime-node: "npm:^1.19.2"
|
||||
@@ -11975,16 +11957,16 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"electron@npm:^32.2.0":
|
||||
version: 32.2.0
|
||||
resolution: "electron@npm:32.2.0"
|
||||
"electron@npm:^33.0.1":
|
||||
version: 33.0.1
|
||||
resolution: "electron@npm:33.0.1"
|
||||
dependencies:
|
||||
"@electron/get": "npm:^2.0.0"
|
||||
"@types/node": "npm:^20.9.0"
|
||||
extract-zip: "npm:^2.0.1"
|
||||
bin:
|
||||
electron: cli.js
|
||||
checksum: 10c0/28d988a9d05c89e93d70cc790bd53ecc97135cc3fa9efe3617f10b87cdf85ada468d383afd7858bcf8f064aa189ea7f8987e32c7ebaac70bda64e8f9f85621e8
|
||||
checksum: 10c0/cd5e5cef21df2d5e1ffc95cfe90397c984152db54ac1032deb97f1716f778d41f5fa7c83e42bc86ae3c509a6d45f578515a540b6453ebf818310cc303e95f083
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
@@ -12123,7 +12105,7 @@ __metadata:
|
||||
"@langchain/core": "npm:^0.3.11"
|
||||
"@langchain/ollama": "npm:^0.1.0"
|
||||
"@mozilla/readability": "npm:^0.5.0"
|
||||
"@playwright/test": "npm:^1.48.0"
|
||||
"@playwright/test": "npm:^1.48.1"
|
||||
"@radix-ui/react-accordion": "npm:^1.2.1"
|
||||
"@radix-ui/react-alert-dialog": "npm:^1.1.2"
|
||||
"@radix-ui/react-aspect-ratio": "npm:^1.1.0"
|
||||
@@ -12148,7 +12130,7 @@ __metadata:
|
||||
"@radix-ui/react-toast": "npm:^1.2.2"
|
||||
"@radix-ui/react-toggle": "npm:^1.1.0"
|
||||
"@radix-ui/react-tooltip": "npm:^1.1.3"
|
||||
"@rails/actioncable": "npm:7.2.100"
|
||||
"@rails/actioncable": "npm:7.2.101"
|
||||
"@tailwindcss/typography": "npm:^0.5.15"
|
||||
"@types/ahoy.js": "npm:^0.4.2"
|
||||
"@types/autosize": "npm:^4.0.3"
|
||||
@@ -12180,7 +12162,7 @@ __metadata:
|
||||
axios: "npm:^1.7.7"
|
||||
camelcase: "npm:^8.0.0"
|
||||
camelcase-keys: "npm:^9.1.3"
|
||||
chart.js: "npm:^4.4.4"
|
||||
chart.js: "npm:^4.4.5"
|
||||
cheerio: "npm:^1.0.0"
|
||||
class-variance-authority: "npm:^0.7.0"
|
||||
clsx: "npm:^2.1.1"
|
||||
@@ -12192,8 +12174,8 @@ __metadata:
|
||||
dayjs: "npm:^1.11.13"
|
||||
decamelize: "npm:^6.0.0"
|
||||
decamelize-keys: "npm:^2.0.1"
|
||||
echogarden: "npm:^1.8.1"
|
||||
electron: "npm:^32.2.0"
|
||||
echogarden: "npm:^1.8.4"
|
||||
electron: "npm:^33.0.1"
|
||||
electron-context-menu: "npm:^4.0.4"
|
||||
electron-devtools-installer: "npm:^3.2.0"
|
||||
electron-log: "npm:^5.2.0"
|
||||
@@ -12215,9 +12197,9 @@ __metadata:
|
||||
langchain: "npm:^0.3.2"
|
||||
lodash: "npm:^4.17.21"
|
||||
lru-cache: "npm:^11.0.1"
|
||||
lucide-react: "npm:^0.452.0"
|
||||
lucide-react: "npm:^0.453.0"
|
||||
mark.js: "npm:^8.11.1"
|
||||
microsoft-cognitiveservices-speech-sdk: "npm:^1.40.0"
|
||||
microsoft-cognitiveservices-speech-sdk: "npm:^1.41.0"
|
||||
mustache: "npm:^4.2.0"
|
||||
next-themes: "npm:^0.3.0"
|
||||
octokit: "npm:^4.0.2"
|
||||
@@ -12251,10 +12233,10 @@ __metadata:
|
||||
tailwind-merge: "npm:^2.5.4"
|
||||
tailwind-scrollbar: "npm:^3.1.0"
|
||||
tailwind-scrollbar-hide: "npm:^1.1.7"
|
||||
tailwindcss: "npm:^3.4.13"
|
||||
tailwindcss: "npm:^3.4.14"
|
||||
tailwindcss-animate: "npm:^1.0.7"
|
||||
ts-node: "npm:^10.9.2"
|
||||
tslib: "npm:^2.7.0"
|
||||
tslib: "npm:^2.8.0"
|
||||
typescript: "npm:^5.6.3"
|
||||
umzug: "npm:^3.8.2"
|
||||
unzipper: "npm:^0.12.3"
|
||||
@@ -12925,7 +12907,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"esquery@npm:^1.5.0, esquery@npm:^1.6.0":
|
||||
"esquery@npm:^1.5.0":
|
||||
version: 1.6.0
|
||||
resolution: "esquery@npm:1.6.0"
|
||||
dependencies:
|
||||
@@ -15652,13 +15634,6 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"jsdoc-type-pratt-parser@npm:~4.0.0":
|
||||
version: 4.0.0
|
||||
resolution: "jsdoc-type-pratt-parser@npm:4.0.0"
|
||||
checksum: 10c0/b23ef7bbbe2f56d72630d1c5a233dc9fecaff399063d373c57bef136908c1b05e723dac107177303c03ccf8d75aa51507510b282aa567600477479c5ea0c36d1
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"jsdom@npm:^25.0.1":
|
||||
version: 25.0.1
|
||||
resolution: "jsdom@npm:25.0.1"
|
||||
@@ -16367,12 +16342,12 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"lucide-react@npm:^0.452.0":
|
||||
version: 0.452.0
|
||||
resolution: "lucide-react@npm:0.452.0"
|
||||
"lucide-react@npm:^0.453.0":
|
||||
version: 0.453.0
|
||||
resolution: "lucide-react@npm:0.453.0"
|
||||
peerDependencies:
|
||||
react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0-rc
|
||||
checksum: 10c0/8e8dd351dfcdffc65a3a76eba04f08310fb13c8fce3e2cab74246c72df45d37e467d521b36053298fb33fe1c5c4770be4750eefa224ae5cf6258dffe95c72da1
|
||||
checksum: 10c0/eec257049609eff958329943da6e129c8db1969aa789c6e922f59418d7b6f6913c22e971ac7885efc80eeb3e41e180c48afdee92738a63a31ef5a39836037b7e
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
@@ -16395,7 +16370,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"magic-string@npm:^0.30.0, magic-string@npm:^0.30.10, magic-string@npm:^0.30.11, magic-string@npm:^0.30.3, magic-string@npm:^0.30.4, magic-string@npm:^0.30.8":
|
||||
"magic-string@npm:^0.30.0, magic-string@npm:^0.30.10, magic-string@npm:^0.30.11, magic-string@npm:^0.30.12, magic-string@npm:^0.30.3, magic-string@npm:^0.30.4, magic-string@npm:^0.30.8":
|
||||
version: 0.30.12
|
||||
resolution: "magic-string@npm:0.30.12"
|
||||
dependencies:
|
||||
@@ -17112,18 +17087,17 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"microsoft-cognitiveservices-speech-sdk@npm:^1.40.0":
|
||||
version: 1.40.0
|
||||
resolution: "microsoft-cognitiveservices-speech-sdk@npm:1.40.0"
|
||||
"microsoft-cognitiveservices-speech-sdk@npm:^1.41.0":
|
||||
version: 1.41.0
|
||||
resolution: "microsoft-cognitiveservices-speech-sdk@npm:1.41.0"
|
||||
dependencies:
|
||||
"@es-joy/jsdoccomment": "npm:^0.46.0"
|
||||
"@types/webrtc": "npm:^0.0.37"
|
||||
agent-base: "npm:^6.0.1"
|
||||
bent: "npm:^7.3.12"
|
||||
https-proxy-agent: "npm:^4.0.0"
|
||||
uuid: "npm:^9.0.0"
|
||||
ws: "npm:^7.5.6"
|
||||
checksum: 10c0/a4a1de9a045dd2235dd09ba0df1b1b0d0451fc52ad5cf4f53d2467017d041baef252e00002ff58d9ff8182b4976ce58e4d8b14b17e49c25ebc6de019eefb1be5
|
||||
checksum: 10c0/0335660d8ca809a5b0b86d7664482a169e660ef30d28665201b36d12ef0c9e4dbc18dcd9b4b5d9a473d3f29610b264afb367a1c7a6f78171c968a03f113c64e7
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
@@ -18059,7 +18033,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"nuxt-og-image@npm:^3.0.3, nuxt-og-image@npm:^3.0.4":
|
||||
"nuxt-og-image@npm:^3.0.3":
|
||||
version: 3.0.4
|
||||
resolution: "nuxt-og-image@npm:3.0.4"
|
||||
dependencies:
|
||||
@@ -18096,6 +18070,43 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"nuxt-og-image@npm:^3.0.6":
|
||||
version: 3.0.6
|
||||
resolution: "nuxt-og-image@npm:3.0.6"
|
||||
dependencies:
|
||||
"@nuxt/devtools-kit": "npm:^1.6.0"
|
||||
"@nuxt/kit": "npm:^3.13.2"
|
||||
"@resvg/resvg-js": "npm:^2.6.2"
|
||||
"@resvg/resvg-wasm": "npm:^2.6.2"
|
||||
"@unocss/core": "npm:^0.63.4"
|
||||
"@unocss/preset-wind": "npm:^0.63.4"
|
||||
chrome-launcher: "npm:^1.1.2"
|
||||
defu: "npm:^6.1.4"
|
||||
execa: "npm:^9.4.0"
|
||||
image-size: "npm:^1.1.1"
|
||||
magic-string: "npm:^0.30.12"
|
||||
nuxt-site-config: "npm:^2.2.18"
|
||||
nuxt-site-config-kit: "npm:^2.2.18"
|
||||
nypm: "npm:^0.3.12"
|
||||
ofetch: "npm:^1.4.1"
|
||||
ohash: "npm:^1.1.4"
|
||||
pathe: "npm:^1.1.2"
|
||||
pkg-types: "npm:^1.2.1"
|
||||
playwright-core: "npm:^1.48.0"
|
||||
radix3: "npm:^1.1.2"
|
||||
satori: "npm:^0.11.2"
|
||||
satori-html: "npm:^0.3.2"
|
||||
sirv: "npm:^3.0.0"
|
||||
std-env: "npm:^3.7.0"
|
||||
strip-literal: "npm:^2.1.0"
|
||||
ufo: "npm:^1.5.4"
|
||||
unplugin: "npm:^1.14.1"
|
||||
unwasm: "npm:^0.3.9"
|
||||
yoga-wasm-web: "npm:^0.3.3"
|
||||
checksum: 10c0/d71168e687ab973ab79e3a879520d86155689e5400f1e5241013354a48d3ef5ee2b815412e4fa0cbe129121f22a6d1cf4fb668166d276743f87a9dfc47ee18ca
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"nuxt-schema-org@npm:^3.4.0":
|
||||
version: 3.4.0
|
||||
resolution: "nuxt-schema-org@npm:3.4.0"
|
||||
@@ -18358,7 +18369,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"ofetch@npm:^1.3.4, ofetch@npm:^1.4.0":
|
||||
"ofetch@npm:^1.3.4, ofetch@npm:^1.4.0, ofetch@npm:^1.4.1":
|
||||
version: 1.4.1
|
||||
resolution: "ofetch@npm:1.4.1"
|
||||
dependencies:
|
||||
@@ -19137,7 +19148,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"pkg-types@npm:^1.0.3, pkg-types@npm:^1.1.1, pkg-types@npm:^1.2.0":
|
||||
"pkg-types@npm:^1.0.3, pkg-types@npm:^1.1.1, pkg-types@npm:^1.2.0, pkg-types@npm:^1.2.1":
|
||||
version: 1.2.1
|
||||
resolution: "pkg-types@npm:1.2.1"
|
||||
dependencies:
|
||||
@@ -19148,7 +19159,16 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"playwright-core@npm:1.48.0, playwright-core@npm:^1.47.2":
|
||||
"playwright-core@npm:1.48.1, playwright-core@npm:^1.48.0":
|
||||
version: 1.48.1
|
||||
resolution: "playwright-core@npm:1.48.1"
|
||||
bin:
|
||||
playwright-core: cli.js
|
||||
checksum: 10c0/2f75532b9b7dfa0e586f5660ac1d8ea729bbdbd28dd2c0711e7cfc1adfe5cf7448d7f15a018ec9851a8f50c0743c3990cb9df23064bed603627baeac4dce3915
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"playwright-core@npm:^1.47.2":
|
||||
version: 1.48.0
|
||||
resolution: "playwright-core@npm:1.48.0"
|
||||
bin:
|
||||
@@ -19157,18 +19177,18 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"playwright@npm:1.48.0":
|
||||
version: 1.48.0
|
||||
resolution: "playwright@npm:1.48.0"
|
||||
"playwright@npm:1.48.1":
|
||||
version: 1.48.1
|
||||
resolution: "playwright@npm:1.48.1"
|
||||
dependencies:
|
||||
fsevents: "npm:2.3.2"
|
||||
playwright-core: "npm:1.48.0"
|
||||
playwright-core: "npm:1.48.1"
|
||||
dependenciesMeta:
|
||||
fsevents:
|
||||
optional: true
|
||||
bin:
|
||||
playwright: cli.js
|
||||
checksum: 10c0/4c18b06f7de87442c697c5df5b1e1e4027c5d9f05c07679aca0135dcf18e69a4424ec7a0c91a323487ea4a92fb5146fc80c04329479426722ea742fd5eff1afa
|
||||
checksum: 10c0/96280ae656226e52015c0c69c4c19e9f594c19353a79012a19bd7b7175d7b409c1aed289a629df49ef897a57ccd24668ad15b86c283db10f76212a4db90a94ac
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
@@ -20886,7 +20906,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"satori@npm:^0.11.1":
|
||||
"satori@npm:^0.11.1, satori@npm:^0.11.2":
|
||||
version: 0.11.2
|
||||
resolution: "satori@npm:0.11.2"
|
||||
dependencies:
|
||||
@@ -21274,6 +21294,17 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"sirv@npm:^3.0.0":
|
||||
version: 3.0.0
|
||||
resolution: "sirv@npm:3.0.0"
|
||||
dependencies:
|
||||
"@polka/url": "npm:^1.0.0-next.24"
|
||||
mrmime: "npm:^2.0.0"
|
||||
totalist: "npm:^3.0.0"
|
||||
checksum: 10c0/282c52ee5a93cafa297096ad31aa6c3004a21d4c93abe728b701e51e4329acb887f6e92f07696225414fd6bb4a7782fd64a42d0b6b6467ae0f66bd3fde90b865
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"sisteransi@npm:^1.0.5":
|
||||
version: 1.0.5
|
||||
resolution: "sisteransi@npm:1.0.5"
|
||||
@@ -22014,9 +22045,9 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"tailwindcss@npm:^3.4.13":
|
||||
version: 3.4.13
|
||||
resolution: "tailwindcss@npm:3.4.13"
|
||||
"tailwindcss@npm:^3.4.14":
|
||||
version: 3.4.14
|
||||
resolution: "tailwindcss@npm:3.4.14"
|
||||
dependencies:
|
||||
"@alloc/quick-lru": "npm:^5.2.0"
|
||||
arg: "npm:^5.0.2"
|
||||
@@ -22043,7 +22074,7 @@ __metadata:
|
||||
bin:
|
||||
tailwind: lib/cli.js
|
||||
tailwindcss: lib/cli.js
|
||||
checksum: 10c0/c6525be3dd26febc4ec5e45e80596bff8b48ade7de258c1ec8704297bf47c1ec7b2b186b13662ebaa6ab4795ad8879fb64064f796756bfc8b46558b542b01a6c
|
||||
checksum: 10c0/f6c23f8a3293ce3b2511bca1e50008ac94bd8562cb09fec32fe4f8e8a4f54d9e9fc10e567b7f974abdd4b33e550564a2616d4e793c736955432f28448141ce45
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
@@ -22457,13 +22488,20 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"tslib@npm:^2.0.0, tslib@npm:^2.0.1, tslib@npm:^2.1.0, tslib@npm:^2.2.0, tslib@npm:^2.6.2, tslib@npm:^2.7.0":
|
||||
"tslib@npm:^2.0.0, tslib@npm:^2.0.1, tslib@npm:^2.1.0, tslib@npm:^2.2.0, tslib@npm:^2.6.2":
|
||||
version: 2.7.0
|
||||
resolution: "tslib@npm:2.7.0"
|
||||
checksum: 10c0/469e1d5bf1af585742128827000711efa61010b699cb040ab1800bcd3ccdd37f63ec30642c9e07c4439c1db6e46345582614275daca3e0f4abae29b0083f04a6
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"tslib@npm:^2.8.0":
|
||||
version: 2.8.0
|
||||
resolution: "tslib@npm:2.8.0"
|
||||
checksum: 10c0/31e4d14dc1355e9b89e4d3c893a18abb7f90b6886b089c2da91224d0a7752c79f3ddc41bc1aa0a588ac895bd97bb99c5bc2bfdb2f86de849f31caeb3ba79bbe5
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"tunnel-agent@npm:^0.6.0":
|
||||
version: 0.6.0
|
||||
resolution: "tunnel-agent@npm:0.6.0"
|
||||
|
||||
Reference in New Issue
Block a user