Refactor whisper (#1131)

* refactor transcribe hook

* Fix stt setting

* remove deprecated files

* fix transcribe api

* may select model when transcribe using local

* refactor

* upgrade deps

* update echogarden check try to fix CI

* update echogarden check

* update CI
This commit is contained in:
an-lee
2024-10-17 09:13:04 +08:00
committed by GitHub
parent 9e974e977e
commit 62d6d84fc7
35 changed files with 630 additions and 14942 deletions

View File

@@ -20,7 +20,7 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [macos-12, macos-14, windows-latest]
os: [macos-12, macos-14, windows-latest, ubuntu-latest]
steps:
- uses: actions/checkout@v4
@@ -47,17 +47,24 @@ jobs:
run: yarn enjoy:package
- name: Run main tests with xvfb-run on Ubuntu
continue-on-error: true
if: contains(matrix.os, 'ubuntu')
env:
DEBUG: pw:browser*
run: |
xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn enjoy:test:main
- name: Run main tests on macOS or Windows
if: contains(matrix.os, 'macos') || contains(matrix.os, 'windows')
- name: Run main tests on Windows
continue-on-error: true
if: contains(matrix.os, 'windows')
run: yarn enjoy:test:main
- name: Run main tests on macOS
if: contains(matrix.os, 'macos')
run: yarn enjoy:test:main
- name: Run renderer tests with xvfb-run on Ubuntu
continue-on-error: true
if: contains(matrix.os, 'ubuntu')
run: |
xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn enjoy:test:renderer

View File

@@ -12,7 +12,7 @@
"dependencies": {
"@nuxtjs/seo": "^2.0.0-rc.23",
"nuxt": "^3.13.2",
"nuxt-og-image": "^3.0.4",
"nuxt-og-image": "^3.0.6",
"vue": "^3.5.12",
"vue-router": "^4.4.5"
},
@@ -20,6 +20,6 @@
"autoprefixer": "^10.4.20",
"postcss": "^8.4.47",
"sass": "^1.79.5",
"tailwindcss": "^3.4.13"
"tailwindcss": "^3.4.14"
}
}

View File

@@ -60,9 +60,17 @@ test.afterAll(async () => {
await electronApp.close();
});
test("validate whisper command", async () => {
test("validate echogarden command", async () => {
const res = await page.evaluate(() => {
return window.__ENJOY_APP__.whisper.check();
return window.__ENJOY_APP__.echogarden.check({
engine: "whisper",
whisper: {
model: "tiny.en",
language: "en",
encoderProvider: "cpu",
decoderProvider: "cpu",
},
});
});
console.info(res.log);
expect(res.success).toBeTruthy();
@@ -75,13 +83,6 @@ test("valid ffmpeg command", async () => {
expect(res).toBeTruthy();
});
test("validate echogarden align command", async () => {
const res = await page.evaluate(() => {
return window.__ENJOY_APP__.echogarden.check();
});
expect(res).toBeTruthy();
});
test("should setup default library path", async () => {
const settings = fs.readJsonSync(path.join(resultDir, "settings.json"));
expect(settings.library).not.toBeNull();

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

Binary file not shown.

View File

@@ -19,9 +19,8 @@
"test:main": "yarn run playwright test e2e/main.spec.ts",
"test:renderer": "yarn run playwright test e2e/renderer.spec.ts",
"create-migration": "zx ./src/main/db/create-migration.mjs",
"download-whisper-model": "zx ./scripts/download-whisper-model.mjs",
"download-dictionaries": "zx ./scripts/download-dictionaries.mjs",
"download": "yarn run download-whisper-model && yarn run download-dictionaries"
"download": "yarn run download-dictionaries"
},
"keywords": [],
"author": {
@@ -41,7 +40,7 @@
"@electron-forge/plugin-vite": "<7.5.0",
"@electron-forge/publisher-github": "<7.5.0",
"@electron/fuses": "^1.8.0",
"@playwright/test": "^1.48.0",
"@playwright/test": "^1.48.1",
"@tailwindcss/typography": "^0.5.15",
"@types/ahoy.js": "^0.4.2",
"@types/autosize": "^4.0.3",
@@ -66,7 +65,7 @@
"@typescript-eslint/parser": "^8.9.0",
"@vitejs/plugin-react": "^4.3.2",
"autoprefixer": "^10.4.20",
"electron": "^32.2.0",
"electron": "^33.0.1",
"electron-devtools-installer": "^3.2.0",
"electron-playwright-helpers": "^1.7.1",
"eslint": "^9.12.0",
@@ -78,10 +77,10 @@
"prop-types": "^15.8.1",
"tailwind-merge": "^2.5.4",
"tailwind-scrollbar": "^3.1.0",
"tailwindcss": "^3.4.13",
"tailwindcss": "^3.4.14",
"tailwindcss-animate": "^1.0.7",
"ts-node": "^10.9.2",
"tslib": "^2.7.0",
"tslib": "^2.8.0",
"typescript": "^5.6.3",
"vite": "^5.4.9",
"vite-plugin-static-copy": "^2.0.0",
@@ -120,7 +119,7 @@
"@radix-ui/react-toast": "^1.2.2",
"@radix-ui/react-toggle": "^1.1.0",
"@radix-ui/react-tooltip": "^1.1.3",
"@rails/actioncable": "7.2.100",
"@rails/actioncable": "7.2.101",
"@uidotdev/usehooks": "^2.4.1",
"@vidstack/react": "^1.12.11",
"ahoy.js": "^0.4.4",
@@ -128,7 +127,7 @@
"axios": "^1.7.7",
"camelcase": "^8.0.0",
"camelcase-keys": "^9.1.3",
"chart.js": "^4.4.4",
"chart.js": "^4.4.5",
"cheerio": "^1.0.0",
"class-variance-authority": "^0.7.0",
"clsx": "^2.1.1",
@@ -140,7 +139,7 @@
"dayjs": "^1.11.13",
"decamelize": "^6.0.0",
"decamelize-keys": "^2.0.1",
"echogarden": "^1.8.1",
"echogarden": "^1.8.4",
"electron-context-menu": "^4.0.4",
"electron-log": "^5.2.0",
"electron-settings": "^4.0.4",
@@ -156,9 +155,9 @@
"langchain": "^0.3.2",
"lodash": "^4.17.21",
"lru-cache": "^11.0.1",
"lucide-react": "^0.452.0",
"lucide-react": "^0.453.0",
"mark.js": "^8.11.1",
"microsoft-cognitiveservices-speech-sdk": "^1.40.0",
"microsoft-cognitiveservices-speech-sdk": "^1.41.0",
"mustache": "^4.2.0",
"next-themes": "^0.3.0",
"openai": "^4.67.3",

View File

@@ -1,113 +0,0 @@
#!/usr/bin/env zx
import axios from "axios";
import progress from "progress";
import { createHash } from "crypto";
const model = "ggml-tiny.en.bin";
const sha = "c78c86eb1a8faa21b369bcd33207cc90d64ae9df";
const dir = path.join(process.cwd(), "lib/whisper.cpp/models");
console.info(chalk.blue(`=> Download whisper model ${model}`));
fs.ensureDirSync(dir);
try {
if (fs.statSync(path.join(dir, model)).isFile()) {
console.info(chalk.green(`✅ Model ${model} already exists`));
const hash = await hashFile(path.join(dir, model), { algo: "sha1" });
if (hash === sha) {
console.info(chalk.green(`✅ Model ${model} valid`));
process.exit(0);
} else {
console.error(
chalk.red(`❌ Model ${model} not valid, start to redownload`)
);
fs.removeSync(path.join(dir, model));
}
}
} catch (err) {
if (err && err.code !== "ENOENT") {
console.error(chalk.red(`❌ Error: ${err}`));
process.exit(1);
} else {
console.info(chalk.blue(`=> Start to download model ${model}`));
}
}
const proxyUrl =
process.env.HTTPS_PROXY ||
process.env.https_proxy ||
process.env.HTTP_PROXY ||
process.env.http_proxy;
if (proxyUrl) {
const { hostname, port, protocol } = new URL(proxyUrl);
axios.defaults.proxy = {
host: hostname,
port: port,
protocol: protocol,
};
}
// const modelUrlPrefix =
// "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main";
const modelUrlPrefix = "https://enjoy-storage.baizhiheizi.com";
function hashFile(path, options) {
const algo = options.algo || "sha1";
return new Promise((resolve, reject) => {
const hash = createHash(algo);
const stream = fs.createReadStream(path);
stream.on("error", reject);
stream.on("data", (chunk) => hash.update(chunk));
stream.on("end", () => resolve(hash.digest("hex")));
});
}
const download = async (url, dest) => {
console.info(chalk.blue(`=> Start to download from ${url} to ${dest}`));
return axios
.get(url, { responseType: "stream" })
.then((response) => {
const totalLength = response.headers["content-length"];
const progressBar = new progress(`-> downloading [:bar] :percent :etas`, {
width: 40,
complete: "=",
incomplete: " ",
renderThrottle: 1,
total: parseInt(totalLength),
});
response.data.on("data", (chunk) => {
progressBar.tick(chunk.length);
});
response.data.pipe(fs.createWriteStream(dest)).on("close", async () => {
console.info(chalk.green(`✅ Model ${model} downloaded successfully`));
const hash = await hashFile(path.join(dir, model), { algo: "sha1" });
if (hash === sha) {
console.info(chalk.green(`✅ Model ${model} valid`));
process.exit(0);
} else {
console.error(
chalk.red(
`❌ Model ${model} not valid, please try again using command \`yarn workspace enjoy download-whisper-model\``
)
);
process.exit(1);
}
});
})
.catch((err) => {
console.error(
chalk.red(
`❌ Failed to download ${url}: ${err}.\nPlease try again using command \`yarn workspace enjoy download-whisper-model\``
)
);
process.exit(1);
});
};
await download(`${modelUrlPrefix}/${model}`, path.join(dir, model));

View File

@@ -2,9 +2,20 @@ export * from "./chat-agent-templates";
export * from "./gpt-presets";
export * from "./ipa";
// https://hf-mirror.com/ggerganov/whisper.cpp/tree/main
import whisperModels from "./whisper-models.json";
export const WHISPER_MODELS_OPTIONS = whisperModels;
export const WHISPER_MODELS = [
"tiny",
"tiny.en",
"base",
"base.en",
"small",
"small.en",
"medium",
"medium.en",
"large-v1",
"large-v2",
"large-v3",
"large-v3-turbo",
];
import languages from "./languages.json";
export const LANGUAGES = languages;

View File

@@ -1,79 +0,0 @@
[
{
"type": "tiny",
"name": "ggml-tiny.bin",
"size": "75 MB",
"sha": "bd577a113a864445d4c299885e0cb97d4ba92b5f",
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-tiny.bin"
},
{
"type": "tiny.en",
"name": "ggml-tiny.en.bin",
"size": "75 MB",
"sha": "c78c86eb1a8faa21b369bcd33207cc90d64ae9df",
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin"
},
{
"type": "base",
"name": "ggml-base.bin",
"size": "142 MB",
"sha": "465707469ff3a37a2b9b8d8f89f2f99de7299dac",
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-base.bin"
},
{
"type": "base.en",
"name": "ggml-base.en.bin",
"size": "142 MB",
"sha": "137c40403d78fd54d454da0f9bd998f78703390c",
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin"
},
{
"type": "small",
"name": "ggml-small.bin",
"size": "466 MB",
"sha": "55356645c2b361a969dfd0ef2c5a50d530afd8d5",
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-small.bin"
},
{
"type": "small.en",
"name": "ggml-small.en.bin",
"size": "466 MB",
"sha": "db8a495a91d927739e50b3fc1cc4c6b8f6c2d022",
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-small.en.bin"
},
{
"type": "medium",
"name": "ggml-medium.bin",
"size": "1.5 GB",
"sha": "fd9727b6e1217c2f614f9b698455c4ffd82463b4",
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-medium.bin"
},
{
"type": "medium.en",
"name": "ggml-medium.en.bin",
"size": "1.5 GB",
"sha": "8c30f0e44ce9560643ebd10bbe50cd20eafd3723",
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-medium.en.bin"
},
{
"type": "large-v1",
"name": "ggml-large-v1.bin",
"size": "2.9 GB",
"sha": "b1caaf735c4cc1429223d5a74f0f4d0b9b59a299",
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-large-v1.bin"
},
{
"type": "large-v2",
"name": "ggml-large-v2.bin",
"size": "2.9 GB",
"sha": "0f4c8e34f21cf1a914c59d8b3ce882345ad349d6",
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-large-v2.bin"
},
{
"type": "large",
"name": "ggml-large-v3.bin",
"size": "2.9 GB",
"sha": "ad82bf6a9043ceed055076d0fd39f5f186ff8062",
"url": "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-large-v3.bin"
}
]

View File

@@ -450,6 +450,10 @@
"whisperIsWorkingGood": "Whisper is working good",
"whisperIsNotWorking": "Whisper is not working",
"whisperTranscribeFailed": "Whisper transcribe failed. Maybe your machine is not compatbile with it. Please use other STT service. {{error}}",
"openaiTranscribeFailed": "OpenAI transcribe failed. {{error}}",
"cloudflareTranscribeFailed": "Cloudflare transcribe failed. {{error}}",
"azureTranscribeFailed": "Azure transcribe failed. {{error}}",
"transcribeFailed": "Transcribe failed.",
"checkingWhisperModel": "Checking whisper model",
"whisperModelIsWorkingGood": "Whisper model is working good",
"whisperModelIsNotWorking": "Whisper model is not working",

View File

@@ -450,6 +450,10 @@
"whisperIsWorkingGood": "Whisper 正常工作",
"whisperIsNotWorking": "Whisper 无法正常工作,请尝试更换模型后重试,或联系开发者",
"whisperTranscribeFailed": "Whisper 语音转文本失败。您的设备可能不兼容该功能,请选择其他语音转文本服务。{{error}}",
"openaiTranscribeFailed": "OpenAI 语音转文本失败。{{error}}",
"cloudflareTranscribeFailed": "Cloudflare 语音转文本失败。{{error}}",
"azureTranscribeFailed": "Azure 语音转文本失败。{{error}}",
"transcribeFailed": "语音转文本失败。",
"checkingWhisperModel": "正在检查 Whisper 模型",
"whisperModelIsWorkingGood": "Whisper 模型正常工作",
"whisperModelIsNotWorking": "Whisper 模型无法正常工作,请尝试更换模型后重试,或联系开发者",

View File

@@ -1,6 +1,6 @@
import { ipcMain } from "electron";
import * as Echogarden from "echogarden/dist/api/API.js";
import { AlignmentOptions } from "echogarden/dist/api/API";
import { AlignmentOptions, RecognitionOptions } from "echogarden/dist/api/API";
import {
encodeRawAudioToWave,
decodeWaveToRawAudio,
@@ -22,11 +22,19 @@ import settings from "@main/settings";
import fs from "fs-extra";
import ffmpegPath from "ffmpeg-static";
import { enjoyUrlToPath, pathToEnjoyUrl } from "./utils";
import { UserSetting } from "./db/models";
import { UserSettingKeyEnum } from "@/types/enums";
import { WHISPER_MODELS } from "@/constants";
import { WhisperOptions } from "echogarden/dist/recognition/WhisperSTT.js";
Echogarden.setGlobalOption(
"ffmpegPath",
ffmpegPath.replace("app.asar", "app.asar.unpacked")
);
Echogarden.setGlobalOption(
"packageBaseURL",
"https://hf-mirror.com/echogarden/echogarden-packages/resolve/main/"
);
const __filename = url.fileURLToPath(import.meta.url);
/*
@@ -38,6 +46,7 @@ const __dirname = path
const logger = log.scope("echogarden");
class EchogardenWrapper {
public recognize: typeof Echogarden.recognize;
public align: typeof Echogarden.align;
public alignSegments: typeof Echogarden.alignSegments;
public denoise: typeof Echogarden.denoise;
@@ -50,6 +59,7 @@ class EchogardenWrapper {
public wordTimelineToSegmentSentenceTimeline: typeof wordTimelineToSegmentSentenceTimeline;
constructor() {
this.recognize = Echogarden.recognize;
this.align = Echogarden.align;
this.alignSegments = Echogarden.alignSegments;
this.denoise = Echogarden.denoise;
@@ -63,14 +73,27 @@ class EchogardenWrapper {
wordTimelineToSegmentSentenceTimeline;
}
async check() {
async check(
options: RecognitionOptions = {
engine: "whisper",
whisper: {
model: "tiny.en",
language: "en",
} as WhisperOptions,
}
) {
const sampleFile = path.join(__dirname, "samples", "jfk.wav");
try {
const result = await this.align(
sampleFile,
"And so my fellow Americans ask not what your country can do for you",
{}
);
const whisperModel = await UserSetting.get(UserSettingKeyEnum.WHISPER);
if (WHISPER_MODELS.includes(whisperModel)) {
options.whisper.model = whisperModel;
}
} catch (e) {
logger.error(e);
}
try {
const result = await this.recognize(sampleFile, options);
logger.info(result);
fs.writeJsonSync(
path.join(settings.cachePath(), "echogarden-check.json"),
@@ -78,10 +101,10 @@ class EchogardenWrapper {
{ spaces: 2 }
);
return true;
return { success: true, log: "" };
} catch (e) {
logger.error(e);
return false;
return { success: false, log: e.message };
}
}
@@ -102,6 +125,20 @@ class EchogardenWrapper {
}
registerIpcHandlers() {
ipcMain.handle(
"echogarden-recognize",
async (_event, url: string, options: RecognitionOptions) => {
logger.debug("echogarden-recognize:", options);
try {
const input = enjoyUrlToPath(url);
return await this.recognize(input, options);
} catch (err) {
logger.error(err);
throw err;
}
}
);
ipcMain.handle(
"echogarden-align",
async (
@@ -129,6 +166,9 @@ class EchogardenWrapper {
options: AlignmentOptions
) => {
logger.debug("echogarden-align-segments:", timeline, options);
if (typeof input === "string") {
input = enjoyUrlToPath(input);
}
try {
const rawAudio = await this.ensureRawAudio(input, 16000);
return await this.alignSegments(rawAudio, timeline, options);
@@ -182,8 +222,8 @@ class EchogardenWrapper {
}
);
ipcMain.handle("echogarden-check", async (_event) => {
return this.check();
ipcMain.handle("echogarden-check", async (_event, options: any) => {
return this.check(options);
});
}
}

View File

@@ -1,330 +0,0 @@
import { ipcMain } from "electron";
import settings from "@main/settings";
import path from "path";
import { WHISPER_MODELS_OPTIONS, PROCESS_TIMEOUT } from "@/constants";
import { exec, spawn } from "child_process";
import fs from "fs-extra";
import log from "@main/logger";
import url from "url";
import { enjoyUrlToPath } from "./utils";
import { t } from "i18next";
import { UserSetting } from "@main/db/models";
import db from "@main/db";
import { UserSettingKeyEnum } from "@/types/enums";
const __filename = url.fileURLToPath(import.meta.url);
/*
* whipser bin file will be in /app.asar.unpacked instead of /app.asar
*/
const __dirname = path
.dirname(__filename)
.replace("app.asar", "app.asar.unpacked");
const logger = log.scope("whisper");
class Whipser {
private binMain: string;
private bundledModelsDir: string;
public config: WhisperConfigType;
private abortController: AbortController;
constructor() {
const customWhisperPath = path.join(
settings.libraryPath(),
"whisper",
"main"
);
this.bundledModelsDir = path.join(__dirname, "lib", "whisper", "models");
if (fs.existsSync(customWhisperPath)) {
this.binMain = customWhisperPath;
} else {
this.binMain = path.join(__dirname, "lib", "whisper", "main");
}
this.initialize();
}
async initialize() {
const models = [];
const bundledModels = fs.readdirSync(this.bundledModelsDir);
for (const file of bundledModels) {
const model = WHISPER_MODELS_OPTIONS.find((m) => m.name == file);
if (!model) continue;
models.push({
...model,
savePath: path.join(this.bundledModelsDir, file),
});
}
const dir = path.join(settings.libraryPath(), "whisper", "models");
fs.ensureDirSync(dir);
const files = fs.readdirSync(dir);
for (const file of files) {
const model = WHISPER_MODELS_OPTIONS.find((m) => m.name == file);
if (!model) continue;
models.push({
...model,
savePath: path.join(dir, file),
});
}
if (db.connection) {
const whisperConfig = (await UserSetting.get(
UserSettingKeyEnum.WHISPER
)) as string;
this.config = {
model: whisperConfig || models[0].name,
availableModels: models,
modelsPath: dir,
};
} else {
this.config = {
model: models[0].name,
availableModels: models,
modelsPath: dir,
};
}
}
currentModel() {
if (!this.config.availableModels) return;
let model: WhisperConfigType["availableModels"][0];
if (this.config.model) {
model = (this.config.availableModels || []).find(
(m) => m.name === this.config.model
);
}
if (!model) {
model = this.config.availableModels[0];
this.config = Object.assign({}, this.config, { model: model.name });
UserSetting.set(UserSettingKeyEnum.WHISPER, model.name);
}
return model;
}
async check() {
this.abortController?.abort();
this.abortController = new AbortController();
const model = this.currentModel();
logger.debug(`Checking whisper model: ${model.savePath}`);
const sampleFile = path.join(__dirname, "samples", "jfk.wav");
const tmpDir = settings.cachePath();
const outputFile = path.join(tmpDir, "jfk.json");
fs.rmSync(outputFile, { force: true });
return new Promise((resolve, _reject) => {
const commands = [
`"${this.binMain}"`,
`--file "${sampleFile}"`,
`--model "${model.savePath}"`,
"--output-json",
`--output-file "${path.join(tmpDir, "jfk")}"`,
];
logger.debug(`Checking whisper command: ${commands.join(" ")}`);
exec(
commands.join(" "),
{
timeout: PROCESS_TIMEOUT,
signal: this.abortController.signal,
},
(error, stdout, stderr) => {
if (error) {
logger.error("error", error);
}
if (stderr) {
logger.info("stderr", stderr);
}
if (stdout) {
logger.debug(stdout);
}
resolve({
success: fs.existsSync(outputFile),
log: `${error?.message || ""}\n${stderr}\n${stdout}`,
});
}
);
});
}
/* Ensure the file is in wav format
* and 16kHz sample rate
*/
async transcribe(
params: {
file?: string;
blob?: {
type: string;
arrayBuffer: ArrayBuffer;
};
},
options?: {
language?: string;
force?: boolean;
extra?: string[];
onProgress?: (progress: number) => void;
}
): Promise<Partial<WhisperOutputType>> {
logger.debug("transcribing from local");
this.abortController?.abort();
this.abortController = new AbortController();
const { blob } = params;
let { file } = params;
if (file) {
file = enjoyUrlToPath(file);
} else if (blob) {
const format = blob.type.split("/")[1];
if (format !== "wav") {
throw new Error("Only wav format is supported");
}
file = path.join(settings.cachePath(), `${Date.now()}.${format}`);
await fs.outputFile(file, Buffer.from(blob.arrayBuffer));
} else {
throw new Error("No file or blob provided");
}
const { force = false, extra = [], language, onProgress } = options || {};
const model = this.currentModel();
if (language && !language.startsWith("en") && model.name.includes("en")) {
throw new Error(`Model ${model.name} does not support ${language}`);
}
const filename = path.basename(file, path.extname(file));
const tmpDir = settings.cachePath();
const outputFile = path.join(tmpDir, filename + ".json");
logger.info(`Trying to transcribe ${file} to ${outputFile}`);
if (fs.pathExistsSync(outputFile) && !force) {
logger.info(`File ${outputFile} already exists`);
return fs.readJson(outputFile);
}
const commandArguments = [
"--file",
file,
"--model",
model.savePath,
"--output-json",
"--output-file",
path.join(tmpDir, filename),
"--print-progress",
"--language",
model.name.includes("en") ? "en" : language?.split("-")?.[0] || "auto",
// `--split-on-word`,
// `--max-len`,
// "1",
...extra,
];
logger.info(
`Running command: ${this.binMain} ${commandArguments.join(" ")}`
);
const command = spawn(this.binMain, commandArguments, {
timeout: PROCESS_TIMEOUT,
signal: this.abortController.signal,
});
return new Promise((resolve, reject) => {
command.stdout.on("data", (data) => {
logger.debug(`stdout: ${data}`);
});
command.stderr.on("data", (data) => {
const output = data.toString();
logger.info(`stderr: ${output}`);
if (output.startsWith("whisper_print_progress_callback")) {
const progress = parseInt(output.match(/\d+%/)?.[0] || "0");
if (typeof progress === "number" && onProgress) onProgress(progress);
}
});
command.on("exit", (code) => {
logger.info(`transcribe process exited with code ${code}`);
});
command.on("error", (err) => {
logger.error("transcribe error", err.message);
reject(err);
});
command.on("close", (code) => {
if (code === 0 && fs.pathExistsSync(outputFile)) {
resolve(fs.readJson(outputFile));
} else {
reject(new Error("Transcription failed"));
}
});
});
}
abort() {
this.abortController?.abort();
}
registerIpcHandlers() {
ipcMain.handle("whisper-config", async () => {
await this.initialize();
return this.config;
});
ipcMain.handle("whisper-set-model", async (_event, model) => {
const originalModel = this.config.model;
this.config.model = model;
return this.check()
.then(({ success, log }) => {
if (success) {
return Object.assign({}, this.config, { ready: true });
} else {
throw new Error(log);
}
})
.catch((err) => {
this.config.model = originalModel;
throw err;
})
.finally(() => {
UserSetting.set(UserSettingKeyEnum.WHISPER, this.config.model);
});
});
ipcMain.handle("whisper-check", async (_event) => {
return await this.check();
});
ipcMain.handle("whisper-transcribe", async (event, params, options) => {
return this.transcribe(params, {
...options,
onProgress: (progress) => {
event.sender.send("whisper-on-progress", progress);
},
})
.then((result) => {
return result;
})
.catch((err) => {
logger.error(err);
throw t("whisperTranscribeFailed", { error: err.message });
});
});
ipcMain.handle("whisper-abort", async (_event) => {
return await this.abort();
});
}
}
export default new Whipser();

View File

@@ -12,7 +12,6 @@ import path from "path";
import db from "@main/db";
import settings from "@main/settings";
import downloader from "@main/downloader";
import whisper from "@main/whisper";
import fs from "fs-extra";
import log from "@main/logger";
import { REPO_URL, WS_URL } from "@/constants";
@@ -62,9 +61,6 @@ main.init = async () => {
// echogarden
echogarden.registerIpcHandlers();
// Whisper
whisper.registerIpcHandlers();
// Waveform
waveform.registerIpcHandlers();
@@ -411,7 +407,6 @@ ${log}
segments: path.join(settings.userDataPath(), "segments"),
speeches: path.join(settings.userDataPath(), "speeches"),
recordings: path.join(settings.userDataPath(), "recordings"),
whisper: path.join(settings.libraryPath(), "whisper"),
waveforms: path.join(settings.libraryPath(), "waveforms"),
logs: path.join(settings.libraryPath(), "logs"),
cache: settings.cachePath(),

View File

@@ -3,6 +3,7 @@
import { contextBridge, ipcRenderer, IpcRendererEvent } from "electron";
import { version } from "../package.json";
import { Timeline } from "echogarden/dist/utilities/Timeline";
import { RecognitionOptions } from "echogarden/dist/api/API";
contextBridge.exposeInMainWorld("__ENJOY_APP__", {
app: {
@@ -472,6 +473,9 @@ contextBridge.exposeInMainWorld("__ENJOY_APP__", {
},
},
echogarden: {
recognize: (input: string, options: RecognitionOptions) => {
return ipcRenderer.invoke("echogarden-recognize", input, options);
},
align: (input: string, transcript: string, options: any) => {
return ipcRenderer.invoke("echogarden-align", input, transcript, options);
},
@@ -502,42 +506,6 @@ contextBridge.exposeInMainWorld("__ENJOY_APP__", {
return ipcRenderer.invoke("echogarden-check");
},
},
whisper: {
config: () => {
return ipcRenderer.invoke("whisper-config");
},
setModel: (model: string) => {
return ipcRenderer.invoke("whisper-set-model", model);
},
check: () => {
return ipcRenderer.invoke("whisper-check");
},
transcribe: (
params: {
file?: string;
blob?: {
type: string;
arrayBuffer: ArrayBuffer;
};
},
options?: {
language?: string;
force?: boolean;
extra?: string[];
}
) => {
return ipcRenderer.invoke("whisper-transcribe", params, options);
},
onProgress: (
callback: (event: IpcRendererEvent, progress: number) => void
) => ipcRenderer.on("whisper-on-progress", callback),
abort: () => {
return ipcRenderer.invoke("whisper-abort");
},
removeProgressListeners: () => {
ipcRenderer.removeAllListeners("whisper-on-progress");
},
},
ffmpeg: {
check: () => {
return ipcRenderer.invoke("ffmpeg-check-command");

View File

@@ -79,6 +79,7 @@ export const MediaTranscriptionGenerateButton = (props: {
generateTranscription({
originalText: data.text,
language: data.language,
model: data.model,
service: data.service as SttEngineOptionEnum | "upload",
isolate: data.isolate,
})

View File

@@ -69,6 +69,7 @@ const LoadingContent = () => {
generateTranscription({
originalText: data.text,
language: data.language,
model: data.model,
service: data.service as SttEngineOptionEnum | "upload",
isolate: data.isolate,
});

View File

@@ -28,7 +28,6 @@ export * from "./theme-settings";
export * from "./proxy-settings";
export * from "./whisper-model-options";
export * from "./network-state";
export * from "./recorder-settings";

View File

@@ -6,7 +6,6 @@ import {
DialogContent,
DialogHeader,
DialogDescription,
DialogFooter,
toast,
Select,
SelectTrigger,
@@ -14,29 +13,26 @@ import {
SelectItem,
SelectValue,
} from "@renderer/components/ui";
import { WhisperModelOptions } from "@renderer/components";
import {
AppSettingsProviderContext,
AISettingsProviderContext,
} from "@renderer/context";
import { useContext, useEffect, useState } from "react";
import { InfoIcon, AlertCircleIcon } from "lucide-react";
import { useContext, useState } from "react";
import { AlertCircleIcon } from "lucide-react";
import { SttEngineOptionEnum } from "@/types/enums";
import { WHISPER_MODELS } from "@/constants";
export const WhisperSettings = () => {
const { sttEngine, whisperConfig, refreshWhisperConfig, setSttEngine } =
useContext(AISettingsProviderContext);
const { sttEngine, whisperModel, setWhisperModel, setSttEngine } = useContext(
AISettingsProviderContext
);
const { EnjoyApp } = useContext(AppSettingsProviderContext);
const [stderr, setStderr] = useState("");
useEffect(() => {
refreshWhisperConfig();
}, []);
const handleCheck = async () => {
toast.promise(
async () => {
const { success, log } = await EnjoyApp.whisper.check();
const { success, log } = await EnjoyApp.echogarden.check();
if (success) {
setStderr("");
return Promise.resolve();
@@ -98,46 +94,28 @@ export const WhisperSettings = () => {
</SelectContent>
</Select>
{sttEngine === "local" && (
{sttEngine === SttEngineOptionEnum.LOCAL && (
<>
<Select
value={whisperModel}
onValueChange={(value) => {
setWhisperModel(value);
}}
>
<SelectTrigger className="min-w-fit">
<SelectValue placeholder="service"></SelectValue>
</SelectTrigger>
<SelectContent>
{WHISPER_MODELS.map((model) => (
<SelectItem key={model} value={model}>
{model}
</SelectItem>
))}
</SelectContent>
</Select>
<Button onClick={handleCheck} variant="secondary" size="sm">
{t("check")}
</Button>
<Dialog>
<DialogTrigger asChild>
<Button variant="secondary" size="sm">
{t("model")}
</Button>
</DialogTrigger>
<DialogContent>
<DialogHeader>{t("sttAiService")}</DialogHeader>
<DialogDescription>
{t("chooseAIModelDependingOnYourHardware")}
</DialogDescription>
<WhisperModelOptions />
<DialogFooter>
<div className="text-xs flex items-start space-x-2">
<InfoIcon className="mr-1.5 w-4 h-4" />
<span className="flex-1 opacity-70">
{t("yourModelsWillBeDownloadedTo", {
path: whisperConfig.modelsPath,
})}
</span>
<Button
onClick={() => {
EnjoyApp.shell.openPath(whisperConfig?.modelsPath);
}}
variant="outline"
size="sm"
>
{t("open")}
</Button>
</div>
</DialogFooter>
</DialogContent>
</Dialog>
</>
)}
</div>

View File

@@ -1,196 +0,0 @@
import {
AlertDialog,
AlertDialogTitle,
AlertDialogHeader,
AlertDialogFooter,
AlertDialogDescription,
AlertDialogContent,
AlertDialogAction,
AlertDialogCancel,
Button,
ScrollArea,
toast,
Progress,
} from "@renderer/components/ui";
import { t } from "i18next";
import { CheckCircle, DownloadIcon, XCircleIcon } from "lucide-react";
import { WHISPER_MODELS_OPTIONS } from "@/constants";
import { useState, useContext, useEffect } from "react";
import {
AppSettingsProviderContext,
AISettingsProviderContext,
} from "@renderer/context";
type ModelType = {
type: string;
name: string;
size: string;
url: string;
downloaded?: boolean;
downloadState?: DownloadStateType;
};
export const WhisperModelOptions = () => {
const [selectingModel, setSelectingModel] = useState<ModelType | null>(null);
const [availableModels, setAvailableModels] = useState<ModelType[]>([]);
const { EnjoyApp } = useContext(AppSettingsProviderContext);
const { whisperConfig, setWhisperModel } = useContext(
AISettingsProviderContext
);
useEffect(() => {
updateAvailableModels();
}, []);
useEffect(() => {
listenToDownloadState();
return () => {
EnjoyApp.download.removeAllListeners();
};
}, [selectingModel]);
const updateAvailableModels = async () => {
const models = whisperConfig.availableModels;
const options: ModelType[] = WHISPER_MODELS_OPTIONS;
options.forEach((o) => {
o.downloaded = models.findIndex((m) => m.name === o.name) > -1;
});
setAvailableModels(options);
};
const downloadModel = async () => {
if (!selectingModel) return;
const model = WHISPER_MODELS_OPTIONS.find(
(m) => m.name === selectingModel.name
);
EnjoyApp.download.start(model.url, whisperConfig.modelsPath);
setSelectingModel(null);
};
const listenToDownloadState = () => {
EnjoyApp.download.onState((_event, state) => {
const model = availableModels.find((m) => m.name === state.name);
if (!model) return;
if (model) {
model.downloadState = state;
}
if (state.state === "completed") {
model.downloaded = true;
setWhisperModel(model.name);
} else if (state.state === "cancelled" || state.state === "interrupted") {
model.downloaded = false;
model.downloadState = null;
}
setAvailableModels([...availableModels]);
});
};
return (
<>
<ScrollArea className="max-h-96">
{availableModels.map((option) => {
return (
<div key={option.name}>
<div
className={`cursor-pointer hover:bg-secondary px-4 py-2 rounded ${
whisperConfig.model === option.name ? "bg-secondary" : ""
}`}
onClick={() => {
if (option.downloaded) {
toast.promise(setWhisperModel(option.name), {
loading: t("checkingWhisperModel"),
success: t("whisperModelIsWorkingGood"),
error: t("whisperModelIsNotWorking"),
});
} else if (!option.downloadState) {
setSelectingModel(option);
}
}}
>
<div className="flex justify-between">
<div className="font-semibold">{option.type}</div>
{option.downloaded ? (
<CheckCircle
className={`w-4 ${
whisperConfig.model === option.name
? "text-green-500"
: ""
}`}
/>
) : (
<DownloadIcon className="w-4 opacity-70" />
)}
</div>
<div className="text-sm opacity-70 flex justify-between">
<span>{option.name}</span>
<span>~{option.size}</span>
</div>
</div>
{!option.downloaded && option.downloadState && (
<div className="flex items-center space-x-2 py-2 px-4">
<Progress
className="h-1"
value={
(option.downloadState.received /
option.downloadState.total) *
100
}
/>
<Button
onClick={() => {
toast.promise(
EnjoyApp.download.cancel(option.downloadState.name),
{
loading: t("cancelling"),
success: t("cancelled"),
}
);
}}
className=""
variant="ghost"
size="icon"
>
<XCircleIcon className="w-4 h-4" />
</Button>
</div>
)}
</div>
);
})}
</ScrollArea>
<AlertDialog open={!!selectingModel}>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>{t("download")}</AlertDialogTitle>
<AlertDialogDescription>
{t("areYouSureToDownload", {
name: `${selectingModel?.name}(${selectingModel?.size})`,
})}
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel
onClick={() => {
setSelectingModel(null);
}}
>
{t("cancel")}
</AlertDialogCancel>
<AlertDialogAction onClick={() => downloadModel()}>
{t("download")}
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</>
);
};

View File

@@ -30,7 +30,7 @@ import {
toast,
} from "@renderer/components/ui";
import { t } from "i18next";
import { LANGUAGES } from "@/constants";
import { LANGUAGES, WHISPER_MODELS } from "@/constants";
import { ChevronDownIcon, ChevronUpIcon, LoaderIcon } from "lucide-react";
import { parseText } from "media-captions";
import { milisecondsToTimestamp } from "@/utils";
@@ -39,6 +39,7 @@ import { SttEngineOptionEnum } from "@/types/enums";
const transcriptionSchema = z.object({
language: z.string(),
service: z.union([z.nativeEnum(SttEngineOptionEnum), z.literal("upload")]),
model: z.string().optional(),
text: z.string().optional(),
isolate: z.boolean().optional(),
});
@@ -60,13 +61,14 @@ export const TranscriptionCreateForm = (props: {
originalText,
} = props;
const { learningLanguage } = useContext(AppSettingsProviderContext);
const { sttEngine } = useContext(AISettingsProviderContext);
const { sttEngine, whisperModel } = useContext(AISettingsProviderContext);
const form = useForm<z.infer<typeof transcriptionSchema>>({
resolver: zodResolver(transcriptionSchema),
values: {
language: learningLanguage,
service: originalText ? "upload" : sttEngine,
model: sttEngine === SttEngineOptionEnum.LOCAL ? whisperModel : "",
text: originalText,
isolate: false,
},
@@ -197,6 +199,35 @@ export const TranscriptionCreateForm = (props: {
</FormItem>
)}
/>
{form.watch("service") === SttEngineOptionEnum.LOCAL && (
<FormField
control={form.control}
name="model"
render={({ field }) => (
<FormItem className="grid w-full items-center">
<FormLabel>{t("model")}</FormLabel>
<Select
disabled={transcribing}
value={field.value}
onValueChange={field.onChange}
>
<SelectTrigger>
<SelectValue />
</SelectTrigger>
<SelectContent>
{WHISPER_MODELS.map((model) => (
<SelectItem key={model} value={model}>
{model}
</SelectItem>
))}
</SelectContent>
</Select>
</FormItem>
)}
/>
)}
<FormField
control={form.control}
name="language"

View File

@@ -5,13 +5,13 @@ import {
} from "@renderer/context";
import { SttEngineOptionEnum, UserSettingKeyEnum } from "@/types/enums";
import { GPT_PROVIDERS, TTS_PROVIDERS } from "@renderer/components";
import { WHISPER_MODELS } from "@/constants";
type AISettingsProviderState = {
setWhisperModel?: (name: string) => Promise<void>;
sttEngine?: SttEngineOptionEnum;
setSttEngine?: (name: string) => Promise<void>;
whisperConfig?: WhisperConfigType;
refreshWhisperConfig?: () => void;
whisperModel?: string;
setWhisperModel?: (name: string) => void;
openai?: LlmProviderType;
setOpenai?: (config: LlmProviderType) => void;
setGptEngine?: (engine: GptEngineSettingType) => void;
@@ -38,7 +38,7 @@ export const AISettingsProvider = ({
},
});
const [openai, setOpenai] = useState<LlmProviderType>(null);
const [whisperConfig, setWhisperConfig] = useState<WhisperConfigType>(null);
const [whisperModel, setWhisperModel] = useState<string>(null);
const [sttEngine, setSttEngine] = useState<SttEngineOptionEnum>(
SttEngineOptionEnum.ENJOY_AZURE
);
@@ -87,6 +87,44 @@ export const AISettingsProvider = ({
setTtsProviders({ ...providers });
};
const refreshWhisperModel = async () => {
const whisperModel = await EnjoyApp.userSettings.get(
UserSettingKeyEnum.WHISPER
);
if (WHISPER_MODELS.includes(whisperModel)) {
setWhisperModel(whisperModel);
} else {
let model = "tiny";
if (whisperModel.match(/tiny/)) {
model = "tiny";
} else if (whisperModel.match(/base/)) {
model = "base";
} else if (whisperModel.match(/small/)) {
model = "small";
} else if (whisperModel.match(/medium/)) {
model = "medium";
} else if (whisperModel.match(/large/)) {
model = "large-v3-turbo";
}
if (
learningLanguage.match(/en/) &&
model.match(/tiny|base|small|medium/)
) {
model = `${model}.en`;
}
setWhisperModel(model);
}
};
const handleSetWhisperModel = async (name: string) => {
if (WHISPER_MODELS.includes(name)) {
setWhisperModel(name);
EnjoyApp.userSettings.set(UserSettingKeyEnum.WHISPER, name);
}
};
useEffect(() => {
refreshGptProviders();
refreshTtsProviders();
@@ -101,22 +139,8 @@ export const AISettingsProvider = ({
useEffect(() => {
if (db.state !== "connected") return;
if (!libraryPath) return;
refreshWhisperConfig();
}, [db.state, libraryPath]);
const refreshWhisperConfig = async () => {
const config = await EnjoyApp.whisper.config();
setWhisperConfig(config);
};
const setWhisperModel = async (name: string) => {
return EnjoyApp.whisper.setModel(name).then((config) => {
if (!config) return;
setWhisperConfig(config);
});
};
const handleSetSttEngine = async (name: SttEngineOptionEnum) => {
setSttEngine(name);
return EnjoyApp.userSettings.set(UserSettingKeyEnum.STT_ENGINE, name);
@@ -165,6 +189,8 @@ export const AISettingsProvider = ({
setGptEngine(engine);
});
}
refreshWhisperModel();
};
const handleSetOpenai = async (config: LlmProviderType) => {
@@ -208,9 +234,8 @@ export const AISettingsProvider = ({
},
openai,
setOpenai: (config: LlmProviderType) => handleSetOpenai(config),
whisperConfig,
refreshWhisperConfig,
setWhisperModel,
whisperModel,
setWhisperModel: handleSetWhisperModel,
sttEngine,
setSttEngine: (name: SttEngineOptionEnum) => handleSetSttEngine(name),
gptProviders,

View File

@@ -62,6 +62,7 @@ type MediaShadowContextType = {
generateTranscription: (params?: {
originalText?: string;
language?: string;
model?: string;
service?: SttEngineOptionEnum | "upload";
isolate?: boolean;
}) => Promise<void>;

View File

@@ -11,14 +11,14 @@ import axios from "axios";
import { useAiCommand } from "./use-ai-command";
import { toast } from "@renderer/components/ui";
import {
Timeline,
TimelineEntry,
type TimelineEntryType,
} from "echogarden/dist/utilities/Timeline";
import take from "lodash/take";
import sortedUniqBy from "lodash/sortedUniqBy";
import { parseText } from "media-captions";
import { SttEngineOptionEnum } from "@/types/enums";
import { RecognitionResult } from "echogarden/dist/api/API.js";
import take from "lodash/take";
import sortedUniqBy from "lodash/sortedUniqBy";
// test a text string has any punctuations or not
// some transcribed text may not have any punctuations
@@ -26,7 +26,7 @@ const punctuationsPattern = /\w[.,!?](\s|$)/g;
export const useTranscribe = () => {
const { EnjoyApp, user, webApi } = useContext(AppSettingsProviderContext);
const { openai } = useContext(AISettingsProviderContext);
const { openai, whisperModel } = useContext(AISettingsProviderContext);
const { punctuateText } = useAiCommand();
const [output, setOutput] = useState<string>("");
@@ -47,6 +47,7 @@ export const useTranscribe = () => {
params?: {
targetId?: string;
targetType?: string;
model?: string;
originalText?: string;
language: string;
service: SttEngineOptionEnum | "upload";
@@ -64,6 +65,7 @@ export const useTranscribe = () => {
}> => {
const url = await transcode(mediaSrc);
const {
model,
targetId,
targetType,
originalText,
@@ -75,34 +77,11 @@ export const useTranscribe = () => {
const blob = await (await fetch(url)).blob();
let result: any;
let timeline: Timeline = [];
if (service === "upload" && originalText) {
const caption = await parseText(originalText, { type: "srt" });
if (caption.cues.length > 0) {
timeline = caption.cues.map((cue) => {
return {
type: "sentence",
text: cue.text,
startTime: cue.startTime,
endTime: cue.endTime,
timeline: [],
};
});
result = {
engine: "upload",
model: "-",
text: timeline.map((entry) => entry.text).join(" "),
timeline,
};
} else {
result = {
engine: "upload",
model: "-",
text: originalText,
};
}
result = await alignText(originalText);
} else if (service === SttEngineOptionEnum.LOCAL) {
result = await transcribeByLocal(url, language);
result = await transcribeByLocal(url, { language, model });
} else if (service === SttEngineOptionEnum.ENJOY_CLOUDFLARE) {
result = await transcribeByCloudflareAi(blob);
} else if (service === SttEngineOptionEnum.OPENAI) {
@@ -121,67 +100,47 @@ export const useTranscribe = () => {
);
}
let transcript = result.text;
if (!align) {
const { segmentTimeline, transcript } = result;
if (!align && transcript) {
return {
...result,
transcript,
url,
timeline: [],
};
}
/*
* if timeline is available and the transcript contains punctuations
* use `alignSegments` to align each sentence with the timeline
* otherwise, use `align` to align the whole transcript
* if the transcript does not contain any punctuation, use AI command to add punctuation
*/
if (result.timeline?.length && transcript.match(punctuationsPattern)) {
timeline = [...result.timeline];
setOutput("Aligning the transcript...");
if (segmentTimeline && segmentTimeline.length > 0) {
const wordTimeline = await EnjoyApp.echogarden.alignSegments(
new Uint8Array(await blob.arrayBuffer()),
timeline,
segmentTimeline,
{
language,
language: language.split("-")[0],
isolate,
}
);
timeline = await EnjoyApp.echogarden.wordToSentenceTimeline(
const timeline = await EnjoyApp.echogarden.wordToSentenceTimeline(
wordTimeline,
transcript,
language.split("-")[0]
);
} else {
// Remove all content inside `()`, `[]`, `{}` and trim the text
// remove all markdown formatting
transcript = transcript
.replace(/\(.*?\)/g, "")
.replace(/\[.*?\]/g, "")
.replace(/\{.*?\}/g, "")
.replace(/[*_`]/g, "")
.trim();
// if the transcript does not contain any punctuation, use AI command to add punctuation
if (!transcript.match(punctuationsPattern)) {
try {
transcript = await punctuateText(transcript);
} catch (err) {
toast.error(err.message);
console.warn(err.message);
}
}
return {
...result,
timeline,
};
} else if (transcript) {
setOutput("Aligning the transcript...");
const alignmentResult = await EnjoyApp.echogarden.align(
new Uint8Array(await blob.arrayBuffer()),
transcript,
{
language,
language: language.split("-")[0],
isolate,
}
);
const timeline: TimelineEntry[] = [];
alignmentResult.timeline.forEach((t: TimelineEntry) => {
if (t.type === "sentence") {
timeline.push(t);
@@ -191,74 +150,120 @@ export const useTranscribe = () => {
});
}
});
}
return {
...result,
originalText,
transcript,
timeline,
url,
};
return {
...result,
timeline,
};
} else {
throw new Error(t("transcribeFailed"));
}
};
const alignText = async (
originalText: string
): Promise<{
engine: string;
model: string;
transcript: string;
segmentTimeline: TimelineEntry[];
}> => {
const caption = await parseText(originalText, { type: "srt" });
if (caption.cues.length > 0) {
// valid srt file
const segmentTimeline = caption.cues.map((cue) => {
return {
type: "segment",
text: cue.text,
startTime: cue.startTime,
endTime: cue.endTime,
timeline: [],
} as TimelineEntry;
});
return {
engine: "upload",
model: "-",
transcript: segmentTimeline.map((entry) => entry.text).join(" "),
segmentTimeline,
};
} else {
// Remove all content inside `()`, `[]`, `{}` and trim the text
// remove all markdown formatting
let transcript = originalText
.replace(/\(.*?\)/g, "")
.replace(/\[.*?\]/g, "")
.replace(/\{.*?\}/g, "")
.replace(/[*_`]/g, "")
.trim();
// if the transcript does not contain any punctuation, use AI command to add punctuation
if (!transcript.match(punctuationsPattern)) {
try {
const punctuatedText = await punctuateText(transcript);
transcript = punctuatedText;
} catch (err) {
toast.error(err.message);
console.warn(err);
}
}
return {
engine: "upload",
model: "-",
transcript,
segmentTimeline: [],
};
}
};
const transcribeByLocal = async (
url: string,
language?: string
options: { language: string; model?: string }
): Promise<{
engine: string;
model: string;
text: string;
timeline: TimelineEntry[];
transcript: string;
segmentTimeline: TimelineEntry[];
}> => {
const res = await EnjoyApp.whisper.transcribe(
{
file: url,
},
{
language,
force: true,
extra: ["--prompt", `"Hello! Welcome to listen to this audio."`],
}
);
if (!res) {
throw new Error(t("whisperTranscribeFailed", { error: "" }));
let { language, model = whisperModel } = options || {};
const languageCode = language.split("-")[0];
if (model.match(/en/) && languageCode !== "en") {
model = model.replace(".en", "");
}
const timeline: TimelineEntry[] = res.transcription
.map((segment) => {
// ignore the word if it is empty or in the format of `[xxx]` or `(xxx)`
if (
!segment.text.trim() ||
segment.text.trim().match(/^[\[\(].+[\]\)]$/)
) {
return null;
}
let res: RecognitionResult;
try {
res = await EnjoyApp.echogarden.recognize(url, {
engine: "whisper",
language: languageCode,
whisper: {
model,
},
});
} catch (err) {
throw new Error(t("whisperTranscribeFailed", { error: err.message }));
}
return {
type: "segment" as TimelineEntryType,
text: segment.text.trim(),
startTime: segment.offsets.from / 1000.0,
endTime: segment.offsets.to / 1000.0,
};
})
.filter((s) => Boolean(s?.text));
const transcript = timeline
.map((segment) => segment.text)
.join(" ")
.trim();
const { transcript, timeline } = res;
return {
engine: "whisper",
model: res.model.type,
text: transcript,
timeline,
model,
transcript,
segmentTimeline: timeline,
};
};
const transcribeByOpenAi = async (file: File) => {
const transcribeByOpenAi = async (
file: File
): Promise<{
engine: string;
model: string;
transcript: string;
segmentTimeline: TimelineEntry[];
}> => {
if (!openai?.key) {
throw new Error(t("openaiKeyRequired"));
}
@@ -270,38 +275,39 @@ export const useTranscribe = () => {
maxRetries: 0,
});
const res: {
text: string;
words?: { word: string; start: number; end: number }[];
segments?: { text: string; start: number; end: number }[];
} = (await client.audio.transcriptions.create({
file,
model: "whisper-1",
response_format: "verbose_json",
timestamp_granularities: ["segment"],
})) as any;
setOutput("Transcribing from OpenAI...");
try {
const res: {
text: string;
words?: { word: string; start: number; end: number }[];
segments?: { text: string; start: number; end: number }[];
} = (await client.audio.transcriptions.create({
file,
model: "whisper-1",
response_format: "verbose_json",
timestamp_granularities: ["word", "segment"],
})) as any;
let timeline: TimelineEntry[] = [];
if (res.segments) {
res.segments.forEach((segment) => {
const segmentTimeline = {
setOutput("Aligning the transcript...");
const segmentTimeline = (res.segments || []).map((segment) => {
return {
type: "segment" as TimelineEntryType,
text: segment.text,
startTime: segment.start,
endTime: segment.end,
timeline: [] as Timeline,
timeline: [] as TimelineEntry[],
};
timeline.push(segmentTimeline);
});
}
return {
engine: "openai",
model: "whisper-1",
text: res.text,
timeline,
};
return {
engine: "openai",
model: "whisper-1",
transcript: res.text,
segmentTimeline,
};
} catch (err) {
throw new Error(t("openaiTranscribeFailed", { error: err.message }));
}
};
const transcribeByCloudflareAi = async (
@@ -309,35 +315,47 @@ export const useTranscribe = () => {
): Promise<{
engine: string;
model: string;
text: string;
timeline?: TimelineEntry[];
transcript: string;
segmentTimeline: TimelineEntry[];
}> => {
const res: CfWhipserOutputType = (
await axios.postForm(`${AI_WORKER_ENDPOINT}/audio/transcriptions`, blob, {
headers: {
Authorization: `Bearer ${user.accessToken}`,
},
timeout: 1000 * 60 * 5,
})
).data;
setOutput("Transcribing from Cloudflare...");
try {
const res: CfWhipserOutputType = (
await axios.postForm(
`${AI_WORKER_ENDPOINT}/audio/transcriptions`,
blob,
{
headers: {
Authorization: `Bearer ${user.accessToken}`,
},
timeout: 1000 * 60 * 5,
}
)
).data;
const segmentTimeline: TimelineEntry[] = [];
if (res.vtt) {
const caption = await parseText(res.vtt, { type: "vtt" });
for (const cue of caption.cues) {
segmentTimeline.push({
type: "segment",
text: cue.text,
startTime: cue.startTime,
endTime: cue.endTime,
timeline: [],
});
}
}
const caption = await parseText(res.vtt, { type: "vtt" });
const timeline: Timeline = caption.cues.map((cue) => {
return {
type: "segment",
text: cue.text,
startTime: cue.startTime,
endTime: cue.endTime,
timeline: [],
engine: "cloudflare",
model: "@cf/openai/whisper",
transcript: res.text,
segmentTimeline,
};
});
return {
engine: "cloudflare",
model: "@cf/openai/whisper",
text: res.text,
timeline,
};
} catch (err) {
throw new Error(t("cloudflareTranscribeFailed", { error: err.message }));
}
};
const transcribeByAzureAi = async (
@@ -350,9 +368,9 @@ export const useTranscribe = () => {
): Promise<{
engine: string;
model: string;
text: string;
transcript: string;
segmentTimeline: TimelineEntry[];
tokenId: number;
timeline?: TimelineEntry[];
}> => {
const { id, token, region } = await webApi.generateSpeechToken({
...params,
@@ -364,90 +382,92 @@ export const useTranscribe = () => {
config.speechRecognitionLanguage = language;
config.requestWordLevelTimestamps();
config.outputFormat = sdk.OutputFormat.Detailed;
config.setProfanity(sdk.ProfanityOption.Raw);
// create the speech recognizer.
const reco = new sdk.SpeechRecognizer(config, audioConfig);
setOutput("Transcribing from Azure...");
let results: SpeechRecognitionResultType[] = [];
const res: {
engine: string;
model: string;
text: string;
tokenId: number;
timeline?: TimelineEntry[];
} = await new Promise((resolve, reject) => {
reco.recognizing = (_s, e) => {
setOutput(e.result.text);
};
const { transcript, segmentTimeline }: any = await new Promise(
(resolve, reject) => {
reco.recognizing = (_s, e) => {
setOutput((prev) => prev + e.result.text);
};
reco.recognized = (_s, e) => {
const json = e.result.properties.getProperty(
sdk.PropertyId.SpeechServiceResponse_JsonResult
);
const result = JSON.parse(json);
results = results.concat(result);
};
reco.recognized = (_s, e) => {
const json = e.result.properties.getProperty(
sdk.PropertyId.SpeechServiceResponse_JsonResult
);
const result = JSON.parse(json);
results = results.concat(result);
};
reco.canceled = (_s, e) => {
if (e.reason === sdk.CancellationReason.Error) {
return reject(new Error(e.errorDetails));
}
reco.canceled = (_s, e) => {
if (e.reason === sdk.CancellationReason.Error) {
return reject(new Error(e.errorDetails));
}
reco.stopContinuousRecognitionAsync();
console.log("CANCELED: Reason=" + e.reason);
};
reco.stopContinuousRecognitionAsync();
console.log("CANCELED: Reason=" + e.reason);
};
reco.sessionStopped = async (_s, e) => {
console.log(
"Session stopped. Stop continuous recognition.",
e.sessionId,
results
);
reco.stopContinuousRecognitionAsync();
reco.sessionStopped = async (_s, e) => {
console.log(
"Session stopped. Stop continuous recognition.",
e.sessionId,
results
);
reco.stopContinuousRecognitionAsync();
try {
const timeline: Timeline = [];
results.forEach((result) => {
if (!result.DisplayText) return;
if (results.length === 0) {
return reject(t("azureTranscribeFailed", { error: "" }));
}
const best = take(sortedUniqBy(result.NBest, "Confidence"), 1)[0];
if (!best.Words) return;
if (!best.Confidence || best.Confidence < 0.5) return;
try {
const transcript = results
.map((result) => result.DisplayText)
.join(" ");
const segmentTimeline: TimelineEntry[] = [];
results.forEach((result) => {
if (!result.DisplayText) return;
const firstWord = best.Words[0];
const lastWord = best.Words[best.Words.length - 1];
const best = take(sortedUniqBy(result.NBest, "Confidence"), 1)[0];
if (!best.Words) return;
if (!best.Confidence || best.Confidence < 0.5) return;
timeline.push({
type: "sentence",
text: best.Display,
startTime: firstWord.Offset / 10000000.0,
endTime: (lastWord.Offset + lastWord.Duration) / 10000000.0,
timeline: [],
const firstWord = best.Words[0];
const lastWord = best.Words[best.Words.length - 1];
segmentTimeline.push({
type: "segment",
text: best.Display,
startTime: firstWord.Offset / 10000000.0,
endTime: (lastWord.Offset + lastWord.Duration) / 10000000.0,
timeline: [],
});
});
});
const transcript = timeline
.map((result) => result.text)
.join(" ")
.trim();
resolve({
transcript,
segmentTimeline,
});
} catch (err) {
reject(t("azureTranscribeFailed", { error: err.message }));
}
};
reco.startContinuousRecognitionAsync();
}
);
resolve({
engine: "azure",
model: "whisper",
text: transcript,
timeline,
tokenId: id,
});
} catch (err) {
reject(err);
}
};
reco.startContinuousRecognitionAsync();
});
return res;
return {
engine: "azure",
model: "whisper",
transcript,
segmentTimeline,
tokenId: id,
};
};
return {

View File

@@ -107,6 +107,7 @@ export const useTranscriptions = (media: AudioType | VideoType) => {
const generateTranscription = async (params?: {
originalText?: string;
language?: string;
model?: string;
service?: SttEngineOptionEnum | "upload";
isolate?: boolean;
}) => {
@@ -114,6 +115,7 @@ export const useTranscriptions = (media: AudioType | VideoType) => {
originalText,
language = learningLanguage,
service = sttEngine,
model,
isolate = false,
} = params || {};
setService(service);
@@ -279,26 +281,17 @@ export const useTranscriptions = (media: AudioType | VideoType) => {
useEffect(() => {
if (!transcribing) return;
if (service === "local") {
EnjoyApp.whisper.onProgress((_, p: number) => {
if (p > 100) p = 100;
setTranscribingProgress(p);
});
}
EnjoyApp.app.onCmdOutput((_, output) => {
setTranscribingOutput(output);
});
return () => {
EnjoyApp.whisper.removeProgressListeners();
EnjoyApp.app.removeCmdOutputListeners();
setTranscribingOutput(null);
};
}, [media, service, transcribing]);
const abortGenerateTranscription = () => {
EnjoyApp.whisper.abort();
setTranscribing(false);
};

View File

@@ -273,6 +273,10 @@ type EnjoyAppType = {
) => Promise<SpeechType>;
};
echogarden: {
recognize: (
input: string,
options: RecognitionOptions
) => Promise<RecognitionResult>;
align: (
input: string | Uint8Array,
transcript: string,
@@ -289,26 +293,7 @@ type EnjoyAppType = {
language: string
) => Promise<Timeline>;
transcode: (input: string) => Promise<string>;
check: () => Promise<boolean>;
};
whisper: {
config: () => Promise<WhisperConfigType>;
check: () => Promise<{ success: boolean; log: string }>;
setModel: (model: string) => Promise<WhisperConfigType>;
transcribe: (
params: {
file?: string;
blob?: { type: string; arrayBuffer: ArrayBuffer };
},
options?: {
language?: string;
force?: boolean;
extra?: string[];
}
) => Promise<Partial<WhisperOutputType>>;
onProgress: (callback: (event, progress: number) => void) => void;
abort: () => Promise<void>;
removeProgressListeners: () => Promise<void>;
check: (options?: any) => Promise<{ success: boolean; log: string }>;
};
ffmpeg: {
check: () => Promise<boolean>;

View File

@@ -235,3 +235,26 @@ type DictSettingType = {
removing: string[];
mdicts: MDict[];
};
type TranscribeParamsType = {
mediaSrc: string | Blob;
params?: {
targetId?: string;
targetType?: string;
originalText?: string;
language: string;
service: SttEngineOptionEnum | "upload";
isolate?: boolean;
align?: boolean;
};
};
type TranscribeResultType = {
engine: string;
model: string;
transcript: string;
timeline: TimelineEntry[];
originalText?: string;
tokenId?: number;
url: string;
};

View File

@@ -44,16 +44,6 @@ export default defineConfig((env) => {
pluginHotRestart("restart"),
viteStaticCopy({
targets: [
{
src: `lib/whisper.cpp/${
process.env.PACKAGE_OS_ARCH || os.arch()
}/${os.platform()}/*`,
dest: "lib/whisper",
},
{
src: `lib/whisper.cpp/models/*`,
dest: "lib/whisper/models",
},
{
src: `lib/youtubedr/${
process.env.PACKAGE_OS_ARCH || os.arch()

222
yarn.lock
View File

@@ -32,10 +32,10 @@ __metadata:
"@nuxtjs/seo": "npm:^2.0.0-rc.23"
autoprefixer: "npm:^10.4.20"
nuxt: "npm:^3.13.2"
nuxt-og-image: "npm:^3.0.4"
nuxt-og-image: "npm:^3.0.6"
postcss: "npm:^8.4.47"
sass: "npm:^1.79.5"
tailwindcss: "npm:^3.4.13"
tailwindcss: "npm:^3.4.14"
vue: "npm:^3.5.12"
vue-router: "npm:^4.4.5"
languageName: unknown
@@ -363,7 +363,7 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/client-polly@npm:^3.669.0":
"@aws-sdk/client-polly@npm:^3.670.0":
version: 3.670.0
resolution: "@aws-sdk/client-polly@npm:3.670.0"
dependencies:
@@ -622,9 +622,9 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/client-transcribe-streaming@npm:^3.669.0":
version: 3.670.0
resolution: "@aws-sdk/client-transcribe-streaming@npm:3.670.0"
"@aws-sdk/client-transcribe-streaming@npm:^3.672.0":
version: 3.672.0
resolution: "@aws-sdk/client-transcribe-streaming@npm:3.672.0"
dependencies:
"@aws-crypto/sha256-browser": "npm:5.2.0"
"@aws-crypto/sha256-js": "npm:5.2.0"
@@ -674,7 +674,7 @@ __metadata:
"@smithy/util-retry": "npm:^3.0.7"
"@smithy/util-utf8": "npm:^3.0.0"
tslib: "npm:^2.6.2"
checksum: 10c0/24952fb07dddce9c82cb8082fd7382681e980d01a072b64278aa9c4d3e20b36b345ad813d4f890d902b3f34fcfc9c252c37f7af165e0e2b40c520cb797ad1852
checksum: 10c0/8470177bcf36036a76db9567e01d747c7db5e4977cacee4577a1864da64cda069fd96971829e61a28fb98da34e07de55c0cb6e202443edb7d1504e72dcc53782
languageName: node
linkType: hard
@@ -2380,17 +2380,6 @@ __metadata:
languageName: node
linkType: hard
"@es-joy/jsdoccomment@npm:^0.46.0":
version: 0.46.0
resolution: "@es-joy/jsdoccomment@npm:0.46.0"
dependencies:
comment-parser: "npm:1.4.1"
esquery: "npm:^1.6.0"
jsdoc-type-pratt-parser: "npm:~4.0.0"
checksum: 10c0/a7a67936ebf6d9aaf74af018c3ac744769af3552b05ad9b88fca96b2ffdca16e724b0ff497f53634ec4cca81e98d8c471b6b6bde0fa5b725af4222ad9a0707f0
languageName: node
linkType: hard
"@esbuild/aix-ppc64@npm:0.20.2":
version: 0.20.2
resolution: "@esbuild/aix-ppc64@npm:0.20.2"
@@ -3839,7 +3828,7 @@ __metadata:
languageName: node
linkType: hard
"@nuxt/devtools-kit@npm:1.6.0, @nuxt/devtools-kit@npm:^1.4.1, @nuxt/devtools-kit@npm:^1.4.2, @nuxt/devtools-kit@npm:^1.5.1":
"@nuxt/devtools-kit@npm:1.6.0, @nuxt/devtools-kit@npm:^1.4.1, @nuxt/devtools-kit@npm:^1.4.2, @nuxt/devtools-kit@npm:^1.5.1, @nuxt/devtools-kit@npm:^1.6.0":
version: 1.6.0
resolution: "@nuxt/devtools-kit@npm:1.6.0"
dependencies:
@@ -4670,14 +4659,14 @@ __metadata:
languageName: node
linkType: hard
"@playwright/test@npm:^1.48.0":
version: 1.48.0
resolution: "@playwright/test@npm:1.48.0"
"@playwright/test@npm:^1.48.1":
version: 1.48.1
resolution: "@playwright/test@npm:1.48.1"
dependencies:
playwright: "npm:1.48.0"
playwright: "npm:1.48.1"
bin:
playwright: cli.js
checksum: 10c0/595a5db99f75d1ffce11df2980ca0dd20523a0a20dd962e4256e536dddccca0138883ae2e9c82aaa26094fb50def73062badc14c001d20fb62a3a0b2df18ebca
checksum: 10c0/32cedc3b2d375cb8f4a830bc820d7726b0235be7a6202e1d6ee46e739b83666271c47c100c11311cf5a916468c18e6a4dc526accf9ef090786e7614c2633b2b8
languageName: node
linkType: hard
@@ -6056,10 +6045,10 @@ __metadata:
languageName: node
linkType: hard
"@rails/actioncable@npm:7.2.100":
version: 7.2.100
resolution: "@rails/actioncable@npm:7.2.100"
checksum: 10c0/dc24fcd1a9f153e5550c691ddff5ea7d3d7589a3be39cf14991d3fb6e77d1d952dbf9ab5f00c9781f85e750d94f7a80ade3231ab284b008d6de311ed33b21cb9
"@rails/actioncable@npm:7.2.101":
version: 7.2.101
resolution: "@rails/actioncable@npm:7.2.101"
checksum: 10c0/5917b0d5c51c8b38bb33532fd6483937d6bf030add46e2bca0272aed5b5b43ceec881f8c89a9784b416cb86a2655198f0faac4929a67caa4812ed622346cdc90
languageName: node
linkType: hard
@@ -8048,7 +8037,7 @@ __metadata:
languageName: node
linkType: hard
"@unocss/preset-wind@npm:^0.63.1":
"@unocss/preset-wind@npm:^0.63.1, @unocss/preset-wind@npm:^0.63.4":
version: 0.63.4
resolution: "@unocss/preset-wind@npm:0.63.4"
dependencies:
@@ -9748,12 +9737,12 @@ __metadata:
languageName: node
linkType: hard
"chart.js@npm:^4.4.4":
version: 4.4.4
resolution: "chart.js@npm:4.4.4"
"chart.js@npm:^4.4.5":
version: 4.4.5
resolution: "chart.js@npm:4.4.5"
dependencies:
"@kurkle/color": "npm:^0.3.0"
checksum: 10c0/9fa3206403a6103916f7762c2665d322c42b0cc07fba91526b1d033ddb887c1ba74b3ebc0bd0748a9e55abd1017f25fdb2292cdd6579d8c2d3bcb1c58f71281c
checksum: 10c0/042393725d1dc6a7fc55757abc5db88cbce0e84ad115c08d552af56d72197e806eb78e04253d12a5f0cf331a57a059dc30fc2dab9324e4f695806b3a087d1b0c
languageName: node
linkType: hard
@@ -10251,13 +10240,6 @@ __metadata:
languageName: node
linkType: hard
"comment-parser@npm:1.4.1":
version: 1.4.1
resolution: "comment-parser@npm:1.4.1"
checksum: 10c0/d6c4be3f5be058f98b24f2d557f745d8fe1cc9eb75bebbdccabd404a0e1ed41563171b16285f593011f8b6a5ec81f564fb1f2121418ac5cbf0f49255bf0840dd
languageName: node
linkType: hard
"commondir@npm:^1.0.1":
version: 1.0.1
resolution: "commondir@npm:1.0.1"
@@ -11725,12 +11707,12 @@ __metadata:
languageName: node
linkType: hard
"echogarden@npm:^1.8.1":
version: 1.8.1
resolution: "echogarden@npm:1.8.1"
"echogarden@npm:^1.8.4":
version: 1.8.4
resolution: "echogarden@npm:1.8.4"
dependencies:
"@aws-sdk/client-polly": "npm:^3.669.0"
"@aws-sdk/client-transcribe-streaming": "npm:^3.669.0"
"@aws-sdk/client-polly": "npm:^3.670.0"
"@aws-sdk/client-transcribe-streaming": "npm:^3.672.0"
"@echogarden/espeak-ng-emscripten": "npm:^0.1.2"
"@echogarden/fasttext-wasm": "npm:^0.1.0"
"@echogarden/flite-wasi": "npm:^0.1.1"
@@ -11760,7 +11742,7 @@ __metadata:
jsdom: "npm:^25.0.1"
json5: "npm:^2.2.3"
kuromoji: "npm:^0.1.2"
microsoft-cognitiveservices-speech-sdk: "npm:^1.40.0"
microsoft-cognitiveservices-speech-sdk: "npm:^1.41.0"
moving-median: "npm:^1.0.0"
msgpack-lite: "npm:^0.1.26"
onnxruntime-node: "npm:^1.19.2"
@@ -11975,16 +11957,16 @@ __metadata:
languageName: node
linkType: hard
"electron@npm:^32.2.0":
version: 32.2.0
resolution: "electron@npm:32.2.0"
"electron@npm:^33.0.1":
version: 33.0.1
resolution: "electron@npm:33.0.1"
dependencies:
"@electron/get": "npm:^2.0.0"
"@types/node": "npm:^20.9.0"
extract-zip: "npm:^2.0.1"
bin:
electron: cli.js
checksum: 10c0/28d988a9d05c89e93d70cc790bd53ecc97135cc3fa9efe3617f10b87cdf85ada468d383afd7858bcf8f064aa189ea7f8987e32c7ebaac70bda64e8f9f85621e8
checksum: 10c0/cd5e5cef21df2d5e1ffc95cfe90397c984152db54ac1032deb97f1716f778d41f5fa7c83e42bc86ae3c509a6d45f578515a540b6453ebf818310cc303e95f083
languageName: node
linkType: hard
@@ -12123,7 +12105,7 @@ __metadata:
"@langchain/core": "npm:^0.3.11"
"@langchain/ollama": "npm:^0.1.0"
"@mozilla/readability": "npm:^0.5.0"
"@playwright/test": "npm:^1.48.0"
"@playwright/test": "npm:^1.48.1"
"@radix-ui/react-accordion": "npm:^1.2.1"
"@radix-ui/react-alert-dialog": "npm:^1.1.2"
"@radix-ui/react-aspect-ratio": "npm:^1.1.0"
@@ -12148,7 +12130,7 @@ __metadata:
"@radix-ui/react-toast": "npm:^1.2.2"
"@radix-ui/react-toggle": "npm:^1.1.0"
"@radix-ui/react-tooltip": "npm:^1.1.3"
"@rails/actioncable": "npm:7.2.100"
"@rails/actioncable": "npm:7.2.101"
"@tailwindcss/typography": "npm:^0.5.15"
"@types/ahoy.js": "npm:^0.4.2"
"@types/autosize": "npm:^4.0.3"
@@ -12180,7 +12162,7 @@ __metadata:
axios: "npm:^1.7.7"
camelcase: "npm:^8.0.0"
camelcase-keys: "npm:^9.1.3"
chart.js: "npm:^4.4.4"
chart.js: "npm:^4.4.5"
cheerio: "npm:^1.0.0"
class-variance-authority: "npm:^0.7.0"
clsx: "npm:^2.1.1"
@@ -12192,8 +12174,8 @@ __metadata:
dayjs: "npm:^1.11.13"
decamelize: "npm:^6.0.0"
decamelize-keys: "npm:^2.0.1"
echogarden: "npm:^1.8.1"
electron: "npm:^32.2.0"
echogarden: "npm:^1.8.4"
electron: "npm:^33.0.1"
electron-context-menu: "npm:^4.0.4"
electron-devtools-installer: "npm:^3.2.0"
electron-log: "npm:^5.2.0"
@@ -12215,9 +12197,9 @@ __metadata:
langchain: "npm:^0.3.2"
lodash: "npm:^4.17.21"
lru-cache: "npm:^11.0.1"
lucide-react: "npm:^0.452.0"
lucide-react: "npm:^0.453.0"
mark.js: "npm:^8.11.1"
microsoft-cognitiveservices-speech-sdk: "npm:^1.40.0"
microsoft-cognitiveservices-speech-sdk: "npm:^1.41.0"
mustache: "npm:^4.2.0"
next-themes: "npm:^0.3.0"
octokit: "npm:^4.0.2"
@@ -12251,10 +12233,10 @@ __metadata:
tailwind-merge: "npm:^2.5.4"
tailwind-scrollbar: "npm:^3.1.0"
tailwind-scrollbar-hide: "npm:^1.1.7"
tailwindcss: "npm:^3.4.13"
tailwindcss: "npm:^3.4.14"
tailwindcss-animate: "npm:^1.0.7"
ts-node: "npm:^10.9.2"
tslib: "npm:^2.7.0"
tslib: "npm:^2.8.0"
typescript: "npm:^5.6.3"
umzug: "npm:^3.8.2"
unzipper: "npm:^0.12.3"
@@ -12925,7 +12907,7 @@ __metadata:
languageName: node
linkType: hard
"esquery@npm:^1.5.0, esquery@npm:^1.6.0":
"esquery@npm:^1.5.0":
version: 1.6.0
resolution: "esquery@npm:1.6.0"
dependencies:
@@ -15652,13 +15634,6 @@ __metadata:
languageName: node
linkType: hard
"jsdoc-type-pratt-parser@npm:~4.0.0":
version: 4.0.0
resolution: "jsdoc-type-pratt-parser@npm:4.0.0"
checksum: 10c0/b23ef7bbbe2f56d72630d1c5a233dc9fecaff399063d373c57bef136908c1b05e723dac107177303c03ccf8d75aa51507510b282aa567600477479c5ea0c36d1
languageName: node
linkType: hard
"jsdom@npm:^25.0.1":
version: 25.0.1
resolution: "jsdom@npm:25.0.1"
@@ -16367,12 +16342,12 @@ __metadata:
languageName: node
linkType: hard
"lucide-react@npm:^0.452.0":
version: 0.452.0
resolution: "lucide-react@npm:0.452.0"
"lucide-react@npm:^0.453.0":
version: 0.453.0
resolution: "lucide-react@npm:0.453.0"
peerDependencies:
react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0-rc
checksum: 10c0/8e8dd351dfcdffc65a3a76eba04f08310fb13c8fce3e2cab74246c72df45d37e467d521b36053298fb33fe1c5c4770be4750eefa224ae5cf6258dffe95c72da1
checksum: 10c0/eec257049609eff958329943da6e129c8db1969aa789c6e922f59418d7b6f6913c22e971ac7885efc80eeb3e41e180c48afdee92738a63a31ef5a39836037b7e
languageName: node
linkType: hard
@@ -16395,7 +16370,7 @@ __metadata:
languageName: node
linkType: hard
"magic-string@npm:^0.30.0, magic-string@npm:^0.30.10, magic-string@npm:^0.30.11, magic-string@npm:^0.30.3, magic-string@npm:^0.30.4, magic-string@npm:^0.30.8":
"magic-string@npm:^0.30.0, magic-string@npm:^0.30.10, magic-string@npm:^0.30.11, magic-string@npm:^0.30.12, magic-string@npm:^0.30.3, magic-string@npm:^0.30.4, magic-string@npm:^0.30.8":
version: 0.30.12
resolution: "magic-string@npm:0.30.12"
dependencies:
@@ -17112,18 +17087,17 @@ __metadata:
languageName: node
linkType: hard
"microsoft-cognitiveservices-speech-sdk@npm:^1.40.0":
version: 1.40.0
resolution: "microsoft-cognitiveservices-speech-sdk@npm:1.40.0"
"microsoft-cognitiveservices-speech-sdk@npm:^1.41.0":
version: 1.41.0
resolution: "microsoft-cognitiveservices-speech-sdk@npm:1.41.0"
dependencies:
"@es-joy/jsdoccomment": "npm:^0.46.0"
"@types/webrtc": "npm:^0.0.37"
agent-base: "npm:^6.0.1"
bent: "npm:^7.3.12"
https-proxy-agent: "npm:^4.0.0"
uuid: "npm:^9.0.0"
ws: "npm:^7.5.6"
checksum: 10c0/a4a1de9a045dd2235dd09ba0df1b1b0d0451fc52ad5cf4f53d2467017d041baef252e00002ff58d9ff8182b4976ce58e4d8b14b17e49c25ebc6de019eefb1be5
checksum: 10c0/0335660d8ca809a5b0b86d7664482a169e660ef30d28665201b36d12ef0c9e4dbc18dcd9b4b5d9a473d3f29610b264afb367a1c7a6f78171c968a03f113c64e7
languageName: node
linkType: hard
@@ -18059,7 +18033,7 @@ __metadata:
languageName: node
linkType: hard
"nuxt-og-image@npm:^3.0.3, nuxt-og-image@npm:^3.0.4":
"nuxt-og-image@npm:^3.0.3":
version: 3.0.4
resolution: "nuxt-og-image@npm:3.0.4"
dependencies:
@@ -18096,6 +18070,43 @@ __metadata:
languageName: node
linkType: hard
"nuxt-og-image@npm:^3.0.6":
version: 3.0.6
resolution: "nuxt-og-image@npm:3.0.6"
dependencies:
"@nuxt/devtools-kit": "npm:^1.6.0"
"@nuxt/kit": "npm:^3.13.2"
"@resvg/resvg-js": "npm:^2.6.2"
"@resvg/resvg-wasm": "npm:^2.6.2"
"@unocss/core": "npm:^0.63.4"
"@unocss/preset-wind": "npm:^0.63.4"
chrome-launcher: "npm:^1.1.2"
defu: "npm:^6.1.4"
execa: "npm:^9.4.0"
image-size: "npm:^1.1.1"
magic-string: "npm:^0.30.12"
nuxt-site-config: "npm:^2.2.18"
nuxt-site-config-kit: "npm:^2.2.18"
nypm: "npm:^0.3.12"
ofetch: "npm:^1.4.1"
ohash: "npm:^1.1.4"
pathe: "npm:^1.1.2"
pkg-types: "npm:^1.2.1"
playwright-core: "npm:^1.48.0"
radix3: "npm:^1.1.2"
satori: "npm:^0.11.2"
satori-html: "npm:^0.3.2"
sirv: "npm:^3.0.0"
std-env: "npm:^3.7.0"
strip-literal: "npm:^2.1.0"
ufo: "npm:^1.5.4"
unplugin: "npm:^1.14.1"
unwasm: "npm:^0.3.9"
yoga-wasm-web: "npm:^0.3.3"
checksum: 10c0/d71168e687ab973ab79e3a879520d86155689e5400f1e5241013354a48d3ef5ee2b815412e4fa0cbe129121f22a6d1cf4fb668166d276743f87a9dfc47ee18ca
languageName: node
linkType: hard
"nuxt-schema-org@npm:^3.4.0":
version: 3.4.0
resolution: "nuxt-schema-org@npm:3.4.0"
@@ -18358,7 +18369,7 @@ __metadata:
languageName: node
linkType: hard
"ofetch@npm:^1.3.4, ofetch@npm:^1.4.0":
"ofetch@npm:^1.3.4, ofetch@npm:^1.4.0, ofetch@npm:^1.4.1":
version: 1.4.1
resolution: "ofetch@npm:1.4.1"
dependencies:
@@ -19137,7 +19148,7 @@ __metadata:
languageName: node
linkType: hard
"pkg-types@npm:^1.0.3, pkg-types@npm:^1.1.1, pkg-types@npm:^1.2.0":
"pkg-types@npm:^1.0.3, pkg-types@npm:^1.1.1, pkg-types@npm:^1.2.0, pkg-types@npm:^1.2.1":
version: 1.2.1
resolution: "pkg-types@npm:1.2.1"
dependencies:
@@ -19148,7 +19159,16 @@ __metadata:
languageName: node
linkType: hard
"playwright-core@npm:1.48.0, playwright-core@npm:^1.47.2":
"playwright-core@npm:1.48.1, playwright-core@npm:^1.48.0":
version: 1.48.1
resolution: "playwright-core@npm:1.48.1"
bin:
playwright-core: cli.js
checksum: 10c0/2f75532b9b7dfa0e586f5660ac1d8ea729bbdbd28dd2c0711e7cfc1adfe5cf7448d7f15a018ec9851a8f50c0743c3990cb9df23064bed603627baeac4dce3915
languageName: node
linkType: hard
"playwright-core@npm:^1.47.2":
version: 1.48.0
resolution: "playwright-core@npm:1.48.0"
bin:
@@ -19157,18 +19177,18 @@ __metadata:
languageName: node
linkType: hard
"playwright@npm:1.48.0":
version: 1.48.0
resolution: "playwright@npm:1.48.0"
"playwright@npm:1.48.1":
version: 1.48.1
resolution: "playwright@npm:1.48.1"
dependencies:
fsevents: "npm:2.3.2"
playwright-core: "npm:1.48.0"
playwright-core: "npm:1.48.1"
dependenciesMeta:
fsevents:
optional: true
bin:
playwright: cli.js
checksum: 10c0/4c18b06f7de87442c697c5df5b1e1e4027c5d9f05c07679aca0135dcf18e69a4424ec7a0c91a323487ea4a92fb5146fc80c04329479426722ea742fd5eff1afa
checksum: 10c0/96280ae656226e52015c0c69c4c19e9f594c19353a79012a19bd7b7175d7b409c1aed289a629df49ef897a57ccd24668ad15b86c283db10f76212a4db90a94ac
languageName: node
linkType: hard
@@ -20886,7 +20906,7 @@ __metadata:
languageName: node
linkType: hard
"satori@npm:^0.11.1":
"satori@npm:^0.11.1, satori@npm:^0.11.2":
version: 0.11.2
resolution: "satori@npm:0.11.2"
dependencies:
@@ -21274,6 +21294,17 @@ __metadata:
languageName: node
linkType: hard
"sirv@npm:^3.0.0":
version: 3.0.0
resolution: "sirv@npm:3.0.0"
dependencies:
"@polka/url": "npm:^1.0.0-next.24"
mrmime: "npm:^2.0.0"
totalist: "npm:^3.0.0"
checksum: 10c0/282c52ee5a93cafa297096ad31aa6c3004a21d4c93abe728b701e51e4329acb887f6e92f07696225414fd6bb4a7782fd64a42d0b6b6467ae0f66bd3fde90b865
languageName: node
linkType: hard
"sisteransi@npm:^1.0.5":
version: 1.0.5
resolution: "sisteransi@npm:1.0.5"
@@ -22014,9 +22045,9 @@ __metadata:
languageName: node
linkType: hard
"tailwindcss@npm:^3.4.13":
version: 3.4.13
resolution: "tailwindcss@npm:3.4.13"
"tailwindcss@npm:^3.4.14":
version: 3.4.14
resolution: "tailwindcss@npm:3.4.14"
dependencies:
"@alloc/quick-lru": "npm:^5.2.0"
arg: "npm:^5.0.2"
@@ -22043,7 +22074,7 @@ __metadata:
bin:
tailwind: lib/cli.js
tailwindcss: lib/cli.js
checksum: 10c0/c6525be3dd26febc4ec5e45e80596bff8b48ade7de258c1ec8704297bf47c1ec7b2b186b13662ebaa6ab4795ad8879fb64064f796756bfc8b46558b542b01a6c
checksum: 10c0/f6c23f8a3293ce3b2511bca1e50008ac94bd8562cb09fec32fe4f8e8a4f54d9e9fc10e567b7f974abdd4b33e550564a2616d4e793c736955432f28448141ce45
languageName: node
linkType: hard
@@ -22457,13 +22488,20 @@ __metadata:
languageName: node
linkType: hard
"tslib@npm:^2.0.0, tslib@npm:^2.0.1, tslib@npm:^2.1.0, tslib@npm:^2.2.0, tslib@npm:^2.6.2, tslib@npm:^2.7.0":
"tslib@npm:^2.0.0, tslib@npm:^2.0.1, tslib@npm:^2.1.0, tslib@npm:^2.2.0, tslib@npm:^2.6.2":
version: 2.7.0
resolution: "tslib@npm:2.7.0"
checksum: 10c0/469e1d5bf1af585742128827000711efa61010b699cb040ab1800bcd3ccdd37f63ec30642c9e07c4439c1db6e46345582614275daca3e0f4abae29b0083f04a6
languageName: node
linkType: hard
"tslib@npm:^2.8.0":
version: 2.8.0
resolution: "tslib@npm:2.8.0"
checksum: 10c0/31e4d14dc1355e9b89e4d3c893a18abb7f90b6886b089c2da91224d0a7752c79f3ddc41bc1aa0a588ac895bd97bb99c5bc2bfdb2f86de849f31caeb3ba79bbe5
languageName: node
linkType: hard
"tunnel-agent@npm:^0.6.0":
version: 0.6.0
resolution: "tunnel-agent@npm:0.6.0"