Feat: bundle a default whisper model (#304)

* add scripts to download whisper model & ffmpeg wasm for bundle

* use default whisper model if no downloaded

.github/workflows/playwright.yml

@@ -1,28 +0,0 @@
-name: Playwright Tests
-on: workflow_dispatch
-jobs:
-  test:
-    timeout-minutes: 60
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [macos-11, macos-12, macos-13, macos-latest, windows-latest, ubuntu-latest]
-    steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 20
-    - name: Install dependencies
-      run: npm install -g yarn && yarn
-    # - name: Install Playwright Browsers
-    #   run: yarn playwright install --with-deps
-    - name: Package
-      run: yarn package:enjoy
-    - name: Run Playwright tests
-      run: yarn test:enjoy
-    - uses: actions/upload-artifact@v3
-      if: always()
-      with:
-        name: playwright-report
-        path: playwright-report/
-        retention-days: 30

.github/workflows/test-enjoy-app.yml

@@ -0,0 +1,36 @@
+name: Test Enjoy App
+on: workflow_dispatch
+jobs:
+  test:
+    timeout-minutes: 60
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          [
+            macos-11,
+            macos-12,
+            macos-13,
+            macos-latest,
+            windows-latest,
+            ubuntu-latest,
+          ]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+      - name: Install dependencies
+        run: npm install -g yarn && yarn
+      - name: Install Playwright Browsers
+        run: yarn playwright install --with-deps
+      - name: Package
+        run: yarn package:enjoy
+      - name: Run Playwright tests
+        run: yarn test:enjoy
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30

.gitignore

@@ -115,3 +115,11 @@ package-lock.json
 */playwright-report/
 */blob-report/
 */playwright/.cache/
+
+# whisper models
+ggml-*.bin
+
+# ffmpeg wasm
+ffmpeg-core.wasm
+ffmpeg-core.js
+ffmpeg-core.worker.js

enjoy/assets/libs/ffmpeg-core.worker.js

@@ -1 +0,0 @@
-"use strict";var Module={};var ENVIRONMENT_IS_NODE=typeof process=="object"&&typeof process.versions=="object"&&typeof process.versions.node=="string";if(ENVIRONMENT_IS_NODE){var nodeWorkerThreads=require("worker_threads");var parentPort=nodeWorkerThreads.parentPort;parentPort.on("message",data=>onmessage({data:data}));var fs=require("fs");Object.assign(global,{self:global,require:require,Module:Module,location:{href:__filename},Worker:nodeWorkerThreads.Worker,importScripts:function(f){(0,eval)(fs.readFileSync(f,"utf8")+"//# sourceURL="+f)},postMessage:function(msg){parentPort.postMessage(msg)},performance:global.performance||{now:function(){return Date.now()}}})}var initializedJS=false;function threadPrintErr(){var text=Array.prototype.slice.call(arguments).join(" ");if(ENVIRONMENT_IS_NODE){fs.writeSync(2,text+"\n");return}console.error(text)}function threadAlert(){var text=Array.prototype.slice.call(arguments).join(" ");postMessage({cmd:"alert",text:text,threadId:Module["_pthread_self"]()})}var err=threadPrintErr;self.alert=threadAlert;Module["instantiateWasm"]=(info,receiveInstance)=>{var module=Module["wasmModule"];Module["wasmModule"]=null;var instance=new WebAssembly.Instance(module,info);return receiveInstance(instance)};self.onunhandledrejection=e=>{throw e.reason??e};function handleMessage(e){try{if(e.data.cmd==="load"){let messageQueue=[];self.onmessage=e=>messageQueue.push(e);self.startWorker=instance=>{Module=instance;postMessage({"cmd":"loaded"});for(let msg of messageQueue){handleMessage(msg)}self.onmessage=handleMessage};Module["wasmModule"]=e.data.wasmModule;for(const handler of e.data.handlers){Module[handler]=function(){postMessage({cmd:"callHandler",handler:handler,args:[...arguments]})}}Module["wasmMemory"]=e.data.wasmMemory;Module["buffer"]=Module["wasmMemory"].buffer;Module["ENVIRONMENT_IS_PTHREAD"]=true;(e.data.urlOrBlob?import(e.data.urlOrBlob):import("./ffmpeg-core.js")).then(exports=>exports.default(Module))}else if(e.data.cmd==="run"){Module["__emscripten_thread_init"](e.data.pthread_ptr,0,0,1);Module["__emscripten_thread_mailbox_await"](e.data.pthread_ptr);Module["establishStackSpace"]();Module["PThread"].receiveObjectTransfer(e.data);Module["PThread"].threadInitTLS();if(!initializedJS){initializedJS=true}try{Module["invokeEntryPoint"](e.data.start_routine,e.data.arg)}catch(ex){if(ex!="unwind"){throw ex}}}else if(e.data.cmd==="cancel"){if(Module["_pthread_self"]()){Module["__emscripten_thread_exit"](-1)}}else if(e.data.target==="setimmediate"){}else if(e.data.cmd==="checkMailbox"){if(initializedJS){Module["checkMailbox"]()}}else if(e.data.cmd){err("worker.js received unknown command "+e.data.cmd);err(e.data)}}catch(ex){if(Module["__emscripten_thread_crashed"]){Module["__emscripten_thread_crashed"]()}throw ex}}self.onmessage=handleMessage;

enjoy/package.json

@@ -14,7 +14,10 @@
     "publish": "rimraf .vite && electron-forge publish",
     "lint": "eslint --ext .ts,.tsx .",
     "test": "playwright test",
-    "create-migration": "zx ./src/main/db/create-migration.mjs"
+    "create-migration": "zx ./src/main/db/create-migration.mjs",
+    "download-whisper-model": "zx ./scripts/download-whisper-model.mjs",
+    "download-ffmpeg-wasm": "zx ./scripts/download-ffmpeg-wasm.mjs",
+    "postinstall": "zx ./scripts/download-whisper-model.mjs && zx ./scripts/download-ffmpeg-wasm.mjs"
   },
   "keywords": [],
   "author": {
@@ -56,6 +59,7 @@
     "eslint-plugin-import": "^2.29.1",
     "flora-colossus": "^2.0.0",
     "octokit": "^3.1.2",
+    "progress": "^2.0.3",
     "tailwind-merge": "^2.2.1",
     "tailwindcss": "^3.4.1",
     "tailwindcss-animate": "^1.0.7",

enjoy/scripts/download-ffmpeg-wasm.mjs

@@ -0,0 +1,149 @@
+#!/usr/bin/env zx
+
+import axios from "axios";
+import { createHash } from "crypto";
+import { HttpsProxyAgent } from "https-proxy-agent";
+
+console.log(chalk.blue("=> Download ffmpeg wasm files"));
+
+const files = [
+  {
+    name: "ffmpeg-core.wasm",
+    md5: "ff1676d6a417d1162dba70dbe8dfd354",
+  },
+  {
+    name: "ffmpeg-core.worker.js",
+    md5: "09dc7f1cd71bb52bd9afc22afdf1f6da",
+  },
+  {
+    name: "ffmpeg-core.js",
+    md5: "30296628fd78e4ef1c939f36c1d31527",
+  },
+];
+const pendingFiles = [];
+const dir = path.join(process.cwd(), "assets/libs");
+fs.ensureDirSync(dir);
+
+await Promise.all(
+  files.map(async (file) => {
+    try {
+      if (fs.statSync(path.join(dir, file.name)).isFile()) {
+        console.log(chalk.green(`=> File ${file.name} already exists`));
+
+        const hash = await hashFile(path.join(dir, file.name), { algo: "md5" });
+        if (hash === file.md5) {
+          console.log(chalk.green(`=> File ${file.name} MD5 match`));
+        } else {
+          console.log(
+            chalk.yellow(
+              `=> File ${file.name} MD5 not match, start to redownload`
+            )
+          );
+          fs.removeSync(path.join(dir, file.name));
+          pendingFiles.push(file);
+        }
+      } else {
+        pendingFiles.push(file);
+      }
+    } catch (err) {
+      if (err && err.code !== "ENOENT") {
+        console.log(chalk.red(`=> Error: ${err}`));
+        process.exit(1);
+      }
+      pendingFiles.push(file);
+    }
+  })
+);
+
+if (pendingFiles.length === 0) {
+  console.log(chalk.green("=> All files already exist"));
+  process.exit(0);
+} else {
+  console.log(chalk.blue(`=> Start to download ${pendingFiles.length} files`));
+}
+
+const proxyUrl =
+  process.env.HTTPS_PROXY ||
+  process.env.https_proxy ||
+  process.env.HTTP_PROXY ||
+  process.env.http_proxy;
+
+if (proxyUrl) {
+  const { hostname, port, protocol } = new URL(proxyUrl);
+  const httpsAgent = new HttpsProxyAgent(proxyUrl);
+  axios.defaults.proxy = {
+    host: hostname,
+    port: port,
+    protocol: protocol,
+  };
+  axios.defaults.httpsAgent = httpsAgent;
+  console.log(chalk.blue(`=> Use proxy: ${proxyUrl}`));
+}
+
+const download = async (url, dest, md5) => {
+  return spinner(async () => {
+    console.log(chalk.blue(`=> Start to download file ${url}`));
+    await axios
+      .get(url, {
+        responseType: "arraybuffer",
+      })
+      .then(async (response) => {
+        const data = Buffer.from(response.data, "binary");
+
+        fs.writeFileSync(dest, data);
+        const hash = await hashFile(dest, { algo: "md5" });
+        console.log(chalk.blue(`=> File ${dest}(MD5: ${hash})`));
+        if (hash === md5) {
+          console.log(chalk.green(`=> ${dest} downloaded successfully`));
+        } else {
+          console.log(
+            chalk.red(
+              `=> Error: ${dest} MD5 not match, ${hash} should be ${md5}`
+            )
+          );
+          process.exit(1);
+        }
+      })
+      .catch((err) => {
+        console.log(chalk.red(`=> Error: ${err}`));
+        process.exit(1);
+      });
+  });
+};
+
+function hashFile(file, options) {
+  const algo = options.algo || "md5";
+  return new Promise((resolve, reject) => {
+    const hash = createHash(algo);
+    const stream = fs.createReadStream(file);
+    stream.on("error", reject);
+    stream.on("data", (chunk) => hash.update(chunk));
+    stream.on("end", () => resolve(hash.digest("hex")));
+  });
+}
+
+const cleanup = () => {
+  files.forEach((file) => {
+    try {
+      fs.removeSync(path.join(dir, file.name));
+    } catch (err) {
+      console.log(chalk.red(`=> Error: ${err}`));
+    }
+  });
+};
+
+const baseURL = "https://unpkg.com/@ffmpeg/core-mt@0.12.6/dist/esm";
+try {
+  await Promise.all(
+    pendingFiles.map((file) =>
+      download(`${baseURL}/${file.name}`, path.join(dir, file.name), file.md5)
+    )
+  );
+} catch (err) {
+  console.log(chalk.red(`=> Error: ${err}`));
+  cleanup();
+  process.exit(1);
+}
+
+console.log(chalk.green("=> All files downloaded successfully"));
+process.exit(0);

enjoy/scripts/download-whisper-model.mjs

@@ -0,0 +1,96 @@
+#!/usr/bin/env zx
+
+import axios from "axios";
+import progress from "progress";
+import { createHash } from "crypto";
+
+const model = "ggml-base.en-q5_1.bin";
+const md5 = "55309cc6613788f07ac7988985210734";
+
+const dir = path.join(process.cwd(), "lib/whisper.cpp/models");
+
+console.log(chalk.blue(`=> Download whisper model ${model}`));
+
+fs.ensureDirSync(dir);
+try {
+  if (fs.statSync(path.join(dir, model)).isFile()) {
+    console.log(chalk.green(`=> Model ${model} already exists`));
+    const hash = await hashFile(path.join(dir, model), { algo: "md5" });
+    if (hash === md5) {
+      console.log(chalk.green(`=> Model ${model} MD5 match`));
+      process.exit(0);
+    } else {
+      console.log(
+        chalk.red(`=> Model ${model} MD5 not match, start to redownload`)
+      );
+      fs.removeSync(path.join(dir, model));
+    }
+  }
+} catch (err) {
+  if (err && err.code !== "ENOENT") {
+    console.log(chalk.red(`=> Error: ${err}`));
+    process.exit(1);
+  } else {
+    console.log(chalk.blue(`=> Start to download model ${model}`));
+  }
+}
+
+const proxyUrl =
+  process.env.HTTPS_PROXY ||
+  process.env.https_proxy ||
+  process.env.HTTP_PROXY ||
+  process.env.http_proxy;
+
+if (proxyUrl) {
+  const { hostname, port, protocol } = new URL(proxyUrl);
+  axios.defaults.proxy = {
+    host: hostname,
+    port: port,
+    protocol: protocol,
+  };
+}
+
+const modelUrlPrefix =
+  "https://huggingface.co/ggerganov/whisper.cpp/resolve/main";
+
+function hashFile(path, options) {
+  const algo = options.algo || "md5";
+  return new Promise((resolve, reject) => {
+    const hash = createHash(algo);
+    const stream = fs.createReadStream(path);
+    stream.on("error", reject);
+    stream.on("data", (chunk) => hash.update(chunk));
+    stream.on("end", () => resolve(hash.digest("hex")));
+  });
+}
+
+const download = async (url, dest) => {
+  return axios
+    .get(url, { responseType: "stream" })
+    .then((response) => {
+      const totalLength = response.headers["content-length"];
+
+      const progressBar = new progress(`-> downloading [:bar] :percent :etas`, {
+        width: 40,
+        complete: "=",
+        incomplete: " ",
+        renderThrottle: 1,
+        total: parseInt(totalLength),
+      });
+
+      response.data.on("data", (chunk) => {
+        progressBar.tick(chunk.length);
+      });
+
+      response.data.pipe(fs.createWriteStream(dest)).on("close", () => {
+        console.log(chalk.green(`=> Model ${model} downloaded successfully`));
+        process.exit(0);
+      });
+    })
+    .catch((err) => {
+      console.log(chalk.red(`=> Error: ${err}`));
+      process.exit(1);
+    });
+};
+
+await download(`${modelUrlPrefix}/${model}`, path.join(dir, model));

enjoy/src/main/whisper.ts

@@ -1,29 +1,16 @@
 import { ipcMain } from "electron";
 import settings from "@main/settings";
 import path from "path";
-import {
-  WHISPER_MODELS_OPTIONS,
-  PROCESS_TIMEOUT,
-  AI_WORKER_ENDPOINT,
-  WEB_API_URL,
-} from "@/constants";
+import { WHISPER_MODELS_OPTIONS, PROCESS_TIMEOUT } from "@/constants";
 import { exec, spawn } from "child_process";
 import fs from "fs-extra";
 import log from "electron-log/main";
-import { t } from "i18next";
-import axios from "axios";
-import { milisecondsToTimestamp } from "@/utils";
-import { AzureSpeechSdk } from "@main/azure-speech-sdk";
-import { Client } from "@/api";
-import take from "lodash/take";
-import sortedUniqBy from "lodash/sortedUniqBy";
 
 const logger = log.scope("whisper");
 
-const MAGIC_TOKENS = ["Mrs.", "Ms.", "Mr.", "Dr.", "Prof.", "St."];
-const END_OF_WORD_REGEX = /[^\.!,\?][\.!\?]/g;
 class Whipser {
   private binMain: string;
+  private defaultModel: string;
   public config: WhisperConfigType;
 
   constructor(config?: WhisperConfigType) {
@@ -33,6 +20,13 @@ class Whipser {
       "whisper",
       "main"
     );
+    this.defaultModel = path.join(
+      __dirname,
+      "lib",
+      "whisper",
+      "models",
+      "ggml-base.en-q5_1.bin"
+    );
     if (fs.existsSync(customWhisperPath)) {
       this.binMain = customWhisperPath;
     } else {
@@ -108,9 +102,7 @@ class Whipser {
   async check() {
     await this.initialize();
 
-    if (!this.currentModel()) {
-      throw new Error("No model selected");
-    }
+    const model = this.currentModel() || this.defaultModel;
 
     const sampleFile = path.join(__dirname, "samples", "jfk.wav");
     const tmpDir = settings.cachePath();
@@ -120,7 +112,7 @@ class Whipser {
       const commands = [
         `"${this.binMain}"`,
         `--file "${sampleFile}"`,
-        `--model "${this.currentModel()}"`,
+        `--model "${model}"`,
         "--output-json",
         `--output-file "${path.join(tmpDir, "jfk")}"`,
       ];
@@ -177,9 +169,7 @@ class Whipser {
       throw new Error("No file or blob provided");
     }
 
-    if (!this.currentModel()) {
-      throw new Error(t("pleaseDownloadWhisperModelFirst"));
-    }
+    const model = this.currentModel() || this.defaultModel;
 
     if (blob) {
       const format = blob.type.split("/")[1];
@@ -207,7 +197,7 @@ class Whipser {
       "--file",
       file,
       "--model",
-      this.currentModel(),
+      model,
       "--output-json",
       "--output-file",
       path.join(tmpDir, filename),

enjoy/vite.main.config.mts

@@ -38,6 +38,10 @@ export default defineConfig({
           }/${os.platform()}/*`,
           dest: "lib/whisper",
         },
+        {
+          src: `lib/whisper.cpp/models/*`,
+          dest: "lib/whisper/models",
+        },
         {
           src: `lib/youtubedr/${
             process.env.PACKAGE_OS_ARCH || os.arch()

yarn.lock

@@ -6156,6 +6156,7 @@ __metadata:
     openai: "npm:^4.27.0"
     pitchfinder: "npm:^2.3.2"
     postcss: "npm:^8.4.35"
+    progress: "npm:^2.0.3"
     proxy-agent: "npm:^6.3.1"
     react: "npm:^18.2.0"
     react-activity-calendar: "npm:^2.2.7"