From e124609437ecdeb4f6c0f3d3104521214044ea22 Mon Sep 17 00:00:00 2001 From: an-lee Date: Thu, 13 Jun 2024 12:55:11 +0800 Subject: [PATCH] Feat: unscripted pronounciation assessment (#666) * clean code * add pronunciation assessment page * load assessments * recording target constraint * display assessment card * update style * may check assessment detail * fix style * add new assessment page * update pronunciation assessment form * add language column to models * create pronunciation assessment * upload file to assess * locales * add source for assessment * display language --- enjoy/src/i18n/en.json | 13 +- enjoy/src/i18n/zh-CN.json | 13 +- enjoy/src/main/db/handlers/index.ts | 21 +- .../pronunciation-assessments-handler.ts | 113 ++++++ .../main/db/handlers/recordings-handler.ts | 20 +- enjoy/src/main/db/index.ts | 4 +- .../1718164365171-add-language-column.js | 49 +++ enjoy/src/main/db/models/audio.ts | 3 + .../db/models/pronunciation-assessment.ts | 24 ++ enjoy/src/main/db/models/recording.ts | 98 +++-- enjoy/src/main/db/models/transcription.ts | 3 + enjoy/src/main/db/models/video.ts | 3 + enjoy/src/preload.ts | 17 + .../media-transcription-read-button.tsx | 4 +- .../src/renderer/components/misc/sidebar.tsx | 12 +- .../pronunciation-assessments/index.ts | 4 +- .../pronunciation-assessment-card.tsx | 138 +++++++ .../pronunciation-assessment-form.tsx | 343 ++++++++++++++++++ ...onunciation-assessment-fulltext-result.tsx | 2 +- .../pronunciation-assessment-score-result.tsx | 8 +- .../recordings/recording-detail.tsx | 10 +- enjoy/src/renderer/components/ui/badage.tsx | 36 ++ enjoy/src/renderer/components/ui/index.ts | 1 + .../pages/pronunciation-assessments/index.tsx | 164 +++++++++ .../pages/pronunciation-assessments/new.tsx | 29 ++ enjoy/src/renderer/router.tsx | 10 + enjoy/src/types/enjoy-app.d.ts | 7 + enjoy/src/types/pronunciation-assessment.d.ts | 6 +- package.json | 2 +- 29 files changed, 1070 insertions(+), 87 deletions(-) create mode 100644 enjoy/src/main/db/handlers/pronunciation-assessments-handler.ts create mode 100644 enjoy/src/main/db/migrations/1718164365171-add-language-column.js create mode 100644 enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-card.tsx create mode 100644 enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-form.tsx create mode 100644 enjoy/src/renderer/components/ui/badage.tsx create mode 100644 enjoy/src/renderer/pages/pronunciation-assessments/index.tsx create mode 100644 enjoy/src/renderer/pages/pronunciation-assessments/new.tsx diff --git a/enjoy/src/i18n/en.json b/enjoy/src/i18n/en.json index 262f9ca8..4057796a 100644 --- a/enjoy/src/i18n/en.json +++ b/enjoy/src/i18n/en.json @@ -142,8 +142,7 @@ "practice": "Practice", "reading": "Reading", "aiAssistant": "AI Assistant", - "aiCoaches": "AI Coaches", - "translator": "Translator", + "pronunciationAssessment": "Assessment", "mine": "Mine", "preferences": "Preferences", "profile": "My Profile", @@ -597,5 +596,13 @@ "summarize": "Summarize", "noResultsFound": "No results found", "readThrough": "Read through", - "selectCrypto": "Select crypto" + "selectCrypto": "Select crypto", + "newAssessment": "New Assessment", + "record": "Record", + "upload": "Upload", + "noFileOrRecording": "No file uploaded or recording", + "referenceText": "Reference text", + "inputReferenceTextOrLeaveItBlank": "Input the reference text or leave it blank", + "assessing": "Assessing", + "assessedSuccessfully": "Assessed successfully" } diff --git a/enjoy/src/i18n/zh-CN.json b/enjoy/src/i18n/zh-CN.json index 9f0e19b6..13500c34 100644 --- a/enjoy/src/i18n/zh-CN.json +++ b/enjoy/src/i18n/zh-CN.json @@ -142,8 +142,7 @@ "practice": "练习记录", "reading": "阅读", "aiAssistant": "智能助手", - "aiCoaches": "AI 教练", - "translator": "翻译助手", + "pronunciationAssessment": "发音评估", "mine": "我的", "preferences": "软件设置", "profile": "个人主页", @@ -597,5 +596,13 @@ "summarize": "提炼主题", "noResultsFound": "没有找到结果", "readThrough": "朗读全文", - "selectCrypto": "选择加密货币" + "selectCrypto": "选择加密货币", + "newAssessment": "新评估", + "record": "录音", + "upload": "上传", + "noFileOrRecording": "没有上传文件或录音", + "referenceText": "参考文本", + "inputReferenceTextOrLeaveItBlank": "输入参考文本,或者留空", + "assessing": "正在评估", + "assessedSuccessfully": "评估成功" } diff --git a/enjoy/src/main/db/handlers/index.ts b/enjoy/src/main/db/handlers/index.ts index 5ef6e0e0..1d3dd8ae 100644 --- a/enjoy/src/main/db/handlers/index.ts +++ b/enjoy/src/main/db/handlers/index.ts @@ -1,10 +1,11 @@ -export * from './audios-handler'; -export * from './cache-objects-handler'; -export * from './conversations-handler'; -export * from './messages-handler'; -export * from './notes-handler'; -export * from './recordings-handler'; -export * from './speeches-handler'; -export * from './segments-handler'; -export * from './transcriptions-handler'; -export * from './videos-handler'; +export * from "./audios-handler"; +export * from "./cache-objects-handler"; +export * from "./conversations-handler"; +export * from "./messages-handler"; +export * from "./notes-handler"; +export * from "./pronunciation-assessments-handler"; +export * from "./recordings-handler"; +export * from "./speeches-handler"; +export * from "./segments-handler"; +export * from "./transcriptions-handler"; +export * from "./videos-handler"; diff --git a/enjoy/src/main/db/handlers/pronunciation-assessments-handler.ts b/enjoy/src/main/db/handlers/pronunciation-assessments-handler.ts new file mode 100644 index 00000000..00c3ed2e --- /dev/null +++ b/enjoy/src/main/db/handlers/pronunciation-assessments-handler.ts @@ -0,0 +1,113 @@ +import { ipcMain, IpcMainEvent } from "electron"; +import { PronunciationAssessment, Recording } from "@main/db/models"; +import { Attributes, FindOptions, WhereOptions } from "sequelize"; + +class PronunciationAssessmentsHandler { + private async findAll( + _event: IpcMainEvent, + options: FindOptions> + ) { + const assessments = await PronunciationAssessment.findAll({ + include: [ + { + association: "recording", + model: Recording, + required: false, + }, + ], + order: [["createdAt", "DESC"]], + ...options, + }); + + if (!assessments) { + return []; + } + return assessments.map((assessment) => assessment.toJSON()); + } + + private async findOne( + _event: IpcMainEvent, + where: WhereOptions + ) { + const assessment = await PronunciationAssessment.findOne({ + where: { + ...where, + }, + include: [ + { + association: "recording", + model: Recording, + required: false, + }, + ], + }); + + return assessment.toJSON(); + } + + private async create( + _event: IpcMainEvent, + data: Partial> & { + blob: { + type: string; + arrayBuffer: ArrayBuffer; + }; + } + ) { + const recording = await Recording.createFromBlob(data.blob, { + targetId: "00000000-0000-0000-0000-000000000000", + targetType: "None", + referenceText: data.referenceText, + language: data.language, + }); + + try { + const assessment = await recording.assess(data.language); + return assessment.toJSON(); + } catch (error) { + await recording.destroy(); + throw error; + } + } + + private async update( + _event: IpcMainEvent, + id: string, + data: Attributes + ) { + const assessment = await PronunciationAssessment.findOne({ + where: { id: id }, + }); + + if (!assessment) { + throw new Error("Assessment not found"); + } + + await assessment.update(data); + } + + private async destroy(_event: IpcMainEvent, id: string) { + const assessment = await PronunciationAssessment.findOne({ + where: { + id, + }, + }); + + if (!assessment) { + throw new Error("Assessment not found"); + } + + await assessment.destroy(); + } + + register() { + ipcMain.handle("pronunciation-assessments-find-all", this.findAll); + ipcMain.handle("pronunciation-assessments-find-one", this.findOne); + ipcMain.handle("pronunciation-assessments-create", this.create); + ipcMain.handle("pronunciation-assessments-update", this.update); + ipcMain.handle("pronunciation-assessments-destroy", this.destroy); + } +} + +export const pronunciationAssessmentsHandler = + new PronunciationAssessmentsHandler(); diff --git a/enjoy/src/main/db/handlers/recordings-handler.ts b/enjoy/src/main/db/handlers/recordings-handler.ts index d84d6311..e5e792e9 100644 --- a/enjoy/src/main/db/handlers/recordings-handler.ts +++ b/enjoy/src/main/db/handlers/recordings-handler.ts @@ -163,7 +163,7 @@ class RecordingsHandler { return await recording.upload(); } - private async assess(event: IpcMainEvent, id: string, language?: string) { + private async assess(_event: IpcMainEvent, id: string, language?: string) { const recording = await Recording.findOne({ where: { id, @@ -171,23 +171,11 @@ class RecordingsHandler { }); if (!recording) { - event.sender.send("on-notification", { - type: "error", - message: t("models.recording.notFound"), - }); + throw new Error(t("models.recording.notFound")); } - return recording - .assess(language) - .then((res) => { - return res; - }) - .catch((err) => { - event.sender.send("on-notification", { - type: "error", - message: err.message, - }); - }); + const assessment = await recording.assess(language) + return assessment.toJSON(); } private async stats( diff --git a/enjoy/src/main/db/index.ts b/enjoy/src/main/db/index.ts index eacda7a0..83d9312f 100644 --- a/enjoy/src/main/db/index.ts +++ b/enjoy/src/main/db/index.ts @@ -21,6 +21,7 @@ import { conversationsHandler, messagesHandler, notesHandler, + pronunciationAssessmentsHandler, recordingsHandler, segmentsHandler, speechesHandler, @@ -28,7 +29,7 @@ import { videosHandler, } from "./handlers"; import path from "path"; -import url from 'url'; +import url from "url"; const __filename = url.fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -101,6 +102,7 @@ db.connect = async () => { conversationsHandler.register(); messagesHandler.register(); notesHandler.register(); + pronunciationAssessmentsHandler.register(); recordingsHandler.register(); segmentsHandler.register(); speechesHandler.register(); diff --git a/enjoy/src/main/db/migrations/1718164365171-add-language-column.js b/enjoy/src/main/db/migrations/1718164365171-add-language-column.js new file mode 100644 index 00000000..1aaf1061 --- /dev/null +++ b/enjoy/src/main/db/migrations/1718164365171-add-language-column.js @@ -0,0 +1,49 @@ +import { DataTypes } from "sequelize"; + +async function up({ context: queryInterface }) { + queryInterface.addColumn("audios", "language", { + type: DataTypes.STRING, + allowNull: true, + }); + queryInterface.addColumn("videos", "language", { + type: DataTypes.STRING, + allowNull: true, + }); + queryInterface.addColumn("transcriptions", "language", { + type: DataTypes.STRING, + allowNull: true, + }); + queryInterface.addColumn("recordings", "language", { + type: DataTypes.STRING, + allowNull: true, + }); + queryInterface.addColumn("pronunciation_assessments", "language", { + type: DataTypes.STRING, + allowNull: true, + }); +} + +async function down({ context: queryInterface }) { + queryInterface.removeColumn("audios", "language", { + type: DataTypes.STRING, + allowNull: true, + }); + queryInterface.removeColumn("videos", "language", { + type: DataTypes.STRING, + allowNull: true, + }); + queryInterface.removeColumn("transcriptions", "language", { + type: DataTypes.STRING, + allowNull: true, + }); + queryInterface.removeColumn("recordings", "language", { + type: DataTypes.STRING, + allowNull: true, + }); + queryInterface.removeColumn("pronunciation_assessments", "language", { + type: DataTypes.STRING, + allowNull: true, + }); +} + +export { up, down }; diff --git a/enjoy/src/main/db/models/audio.ts b/enjoy/src/main/db/models/audio.ts index d456ee18..1783a514 100644 --- a/enjoy/src/main/db/models/audio.ts +++ b/enjoy/src/main/db/models/audio.ts @@ -46,6 +46,9 @@ export class Audio extends Model