lostecho/everyone-can-use-english
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat: unscripted pronounciation assessment (#666)

Browse Source 
* clean code

* add pronunciation assessment page

* load assessments

* recording target constraint

* display assessment card

* update style

* may check assessment detail

* fix style

* add new assessment page

* update pronunciation assessment form

* add language column to models

* create pronunciation assessment

* upload file to assess

* locales

* add source for assessment

* display language
This commit is contained in:
an-lee
2024-06-13 12:55:11 +08:00
committed by GitHub
parent 7f9e997dad
commit e124609437
29 changed files with 1070 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
enjoy/src/i18n/en.json
View File

@@ -142,8 +142,7 @@
"practice": "Practice",
"reading": "Reading",
"aiAssistant": "AI Assistant",
"aiCoaches": "AI Coaches",
"translator": "Translator",
"pronunciationAssessment": "Assessment",
"mine": "Mine",
"preferences": "Preferences",
"profile": "My Profile",
@@ -597,5 +596,13 @@
"summarize": "Summarize",
"noResultsFound": "No results found",
"readThrough": "Read through",
"selectCrypto": "Select crypto"
"selectCrypto": "Select crypto",
"newAssessment": "New Assessment",
"record": "Record",
"upload": "Upload",
"noFileOrRecording": "No file uploaded or recording",
"referenceText": "Reference text",
"inputReferenceTextOrLeaveItBlank": "Input the reference text or leave it blank",
"assessing": "Assessing",
"assessedSuccessfully": "Assessed successfully"
}

13
enjoy/src/i18n/zh-CN.json
View File

@@ -142,8 +142,7 @@
"practice": "练习记录",
"reading": "阅读",
"aiAssistant": "智能助手",
"aiCoaches": "AI 教练",
"translator": "翻译助手",
"pronunciationAssessment": "发音评估",
"mine": "我的",
"preferences": "软件设置",
"profile": "个人主页",
@@ -597,5 +596,13 @@
"summarize": "提炼主题",
"noResultsFound": "没有找到结果",
"readThrough": "朗读全文",
"selectCrypto": "选择加密货币"
"selectCrypto": "选择加密货币",
"newAssessment": "新评估",
"record": "录音",
"upload": "上传",
"noFileOrRecording": "没有上传文件或录音",
"referenceText": "参考文本",
"inputReferenceTextOrLeaveItBlank": "输入参考文本，或者留空",
"assessing": "正在评估",
"assessedSuccessfully": "评估成功"
}

21
enjoy/src/main/db/handlers/index.ts
View File

@@ -1,10 +1,11 @@
export * from './audios-handler';
export * from './cache-objects-handler';
export * from './conversations-handler';
export * from './messages-handler';
export * from './notes-handler';
export * from './recordings-handler';
export * from './speeches-handler';
export * from './segments-handler';
export * from './transcriptions-handler';
export * from './videos-handler';
export * from "./audios-handler";
export * from "./cache-objects-handler";
export * from "./conversations-handler";
export * from "./messages-handler";
export * from "./notes-handler";
export * from "./pronunciation-assessments-handler";
export * from "./recordings-handler";
export * from "./speeches-handler";
export * from "./segments-handler";
export * from "./transcriptions-handler";
export * from "./videos-handler";

113
enjoy/src/main/db/handlers/pronunciation-assessments-handler.ts Normal file
View File

@@ -0,0 +1,113 @@
import { ipcMain, IpcMainEvent } from "electron";
import { PronunciationAssessment, Recording } from "@main/db/models";
import { Attributes, FindOptions, WhereOptions } from "sequelize";
class PronunciationAssessmentsHandler {
private async findAll(
_event: IpcMainEvent,
options: FindOptions<Attributes<PronunciationAssessment>>
) {
const assessments = await PronunciationAssessment.findAll({
include: [
{
association: "recording",
model: Recording,
required: false,
},
],
order: [["createdAt", "DESC"]],
...options,
});
if (!assessments) {
return [];
}
return assessments.map((assessment) => assessment.toJSON());
}
private async findOne(
_event: IpcMainEvent,
where: WhereOptions<PronunciationAssessment>
) {
const assessment = await PronunciationAssessment.findOne({
where: {
...where,
},
include: [
{
association: "recording",
model: Recording,
required: false,
},
],
});
return assessment.toJSON();
}
private async create(
_event: IpcMainEvent,
data: Partial<Attributes<PronunciationAssessment>> & {
blob: {
type: string;
arrayBuffer: ArrayBuffer;
};
}
) {
const recording = await Recording.createFromBlob(data.blob, {
targetId: "00000000-0000-0000-0000-000000000000",
targetType: "None",
referenceText: data.referenceText,
language: data.language,
});
try {
const assessment = await recording.assess(data.language);
return assessment.toJSON();
} catch (error) {
await recording.destroy();
throw error;
}
}
private async update(
_event: IpcMainEvent,
id: string,
data: Attributes<PronunciationAssessment>
) {
const assessment = await PronunciationAssessment.findOne({
where: { id: id },
});
if (!assessment) {
throw new Error("Assessment not found");
}
await assessment.update(data);
}
private async destroy(_event: IpcMainEvent, id: string) {
const assessment = await PronunciationAssessment.findOne({
where: {
id,
},
});
if (!assessment) {
throw new Error("Assessment not found");
}
await assessment.destroy();
}
register() {
ipcMain.handle("pronunciation-assessments-find-all", this.findAll);
ipcMain.handle("pronunciation-assessments-find-one", this.findOne);
ipcMain.handle("pronunciation-assessments-create", this.create);
ipcMain.handle("pronunciation-assessments-update", this.update);
ipcMain.handle("pronunciation-assessments-destroy", this.destroy);
}
}
export const pronunciationAssessmentsHandler =
new PronunciationAssessmentsHandler();

20
enjoy/src/main/db/handlers/recordings-handler.ts
View File

@@ -163,7 +163,7 @@ class RecordingsHandler {
return await recording.upload();
}
private async assess(event: IpcMainEvent, id: string, language?: string) {
private async assess(_event: IpcMainEvent, id: string, language?: string) {
const recording = await Recording.findOne({
where: {
id,
@@ -171,23 +171,11 @@ class RecordingsHandler {
});
if (!recording) {
event.sender.send("on-notification", {
type: "error",
message: t("models.recording.notFound"),
});
throw new Error(t("models.recording.notFound"));
}
return recording
.assess(language)
.then((res) => {
return res;
})
.catch((err) => {
event.sender.send("on-notification", {
type: "error",
message: err.message,
});
});
const assessment = await recording.assess(language)
return assessment.toJSON();
}
private async stats(

4
enjoy/src/main/db/index.ts
View File

@@ -21,6 +21,7 @@ import {
conversationsHandler,
messagesHandler,
notesHandler,
pronunciationAssessmentsHandler,
recordingsHandler,
segmentsHandler,
speechesHandler,
@@ -28,7 +29,7 @@ import {
videosHandler,
} from "./handlers";
import path from "path";
import url from 'url';
import url from "url";
const __filename = url.fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
@@ -101,6 +102,7 @@ db.connect = async () => {
conversationsHandler.register();
messagesHandler.register();
notesHandler.register();
pronunciationAssessmentsHandler.register();
recordingsHandler.register();
segmentsHandler.register();
speechesHandler.register();

49
enjoy/src/main/db/migrations/1718164365171-add-language-column.js Normal file
View File

@@ -0,0 +1,49 @@
import { DataTypes } from "sequelize";
async function up({ context: queryInterface }) {
queryInterface.addColumn("audios", "language", {
type: DataTypes.STRING,
allowNull: true,
});
queryInterface.addColumn("videos", "language", {
type: DataTypes.STRING,
allowNull: true,
});
queryInterface.addColumn("transcriptions", "language", {
type: DataTypes.STRING,
allowNull: true,
});
queryInterface.addColumn("recordings", "language", {
type: DataTypes.STRING,
allowNull: true,
});
queryInterface.addColumn("pronunciation_assessments", "language", {
type: DataTypes.STRING,
allowNull: true,
});
}
async function down({ context: queryInterface }) {
queryInterface.removeColumn("audios", "language", {
type: DataTypes.STRING,
allowNull: true,
});
queryInterface.removeColumn("videos", "language", {
type: DataTypes.STRING,
allowNull: true,
});
queryInterface.removeColumn("transcriptions", "language", {
type: DataTypes.STRING,
allowNull: true,
});
queryInterface.removeColumn("recordings", "language", {
type: DataTypes.STRING,
allowNull: true,
});
queryInterface.removeColumn("pronunciation_assessments", "language", {
type: DataTypes.STRING,
allowNull: true,
});
}
export { up, down };

3
enjoy/src/main/db/models/audio.ts
View File

@@ -46,6 +46,9 @@ export class Audio extends Model<Audio> {
@Column({ primaryKey: true, type: DataType.UUID })
id: string;
@Column(DataType.STRING)
language: string;
@Column(DataType.STRING)
source: string;

24
enjoy/src/main/db/models/pronunciation-assessment.ts
View File

@@ -11,6 +11,7 @@ import {
Model,
DataType,
AllowNull,
AfterFind,
} from "sequelize-typescript";
import mainWindow from "@main/window";
import { Recording } from "@main/db/models";
@@ -39,6 +40,9 @@ export class PronunciationAssessment extends Model<PronunciationAssessment> {
@Column({ primaryKey: true, type: DataType.UUID })
id: string;
@Column(DataType.STRING)
language: string;
@AllowNull(false)
@Column(DataType.UUID)
targetId: string;
@@ -47,6 +51,9 @@ export class PronunciationAssessment extends Model<PronunciationAssessment> {
@Column(DataType.STRING)
targetType: string;
@Column(DataType.VIRTUAL)
target: Recording;
@BelongsTo(() => Recording, {
foreignKey: "targetId",
constraints: false,
@@ -105,6 +112,23 @@ export class PronunciationAssessment extends Model<PronunciationAssessment> {
});
}
@AfterFind
static async findTarget(
findResult: PronunciationAssessment | PronunciationAssessment[]
) {
if (!findResult) return;
if (!Array.isArray(findResult)) findResult = [findResult];
for (const instance of findResult) {
if (instance.targetType === "Recording" && instance.recording) {
instance.target = instance.recording.toJSON();
}
// To prevent mistakes:
delete instance.recording;
delete instance.dataValues.recording;
}
}
@AfterCreate
static autoSync(pronunciationAssessment: PronunciationAssessment) {
pronunciationAssessment.sync().catch(() => {});

98
enjoy/src/main/db/models/recording.ts
View File

@@ -28,6 +28,8 @@ import { AzureSpeechSdk } from "@main/azure-speech-sdk";
import echogarden from "@main/echogarden";
import camelcaseKeys from "camelcase-keys";
import { t } from "i18next";
import { Attributes } from "sequelize";
import { v5 as uuidv5 } from "uuid";
const logger = log.scope("db/models/recording");
@@ -43,6 +45,9 @@ export class Recording extends Model<Recording> {
@Column({ primaryKey: true, type: DataType.UUID })
id: string;
@Column(DataType.STRING)
language: string;
@BelongsTo(() => Audio, { foreignKey: "targetId", constraints: false })
audio: Audio;
@@ -52,11 +57,13 @@ export class Recording extends Model<Recording> {
@Column(DataType.VIRTUAL)
target: Audio | Video;
@IsUUID("all")
@Default("00000000-0000-0000-0000-000000000000")
@Column(DataType.UUID)
targetId: string;
@Column(DataType.STRING)
targetType: "Audio" | "Video";
targetType: "Audio" | "Video" | "None";
@HasOne(() => PronunciationAssessment, {
foreignKey: "targetId",
@@ -136,6 +143,8 @@ export class Recording extends Model<Recording> {
}
async sync() {
if (this.isSynced) return;
const webApi = new Client({
baseUrl: process.env.WEB_API_URL || WEB_API_URL,
accessToken: settings.getSync("user.accessToken") as string,
@@ -172,7 +181,7 @@ export class Recording extends Model<Recording> {
const result = await sdk.pronunciationAssessment({
filePath: this.filePath,
reference: this.referenceText,
language,
language: language || this.language,
});
const resultJson = camelcaseKeys(
@@ -196,16 +205,19 @@ export class Recording extends Model<Recording> {
vocabularyScore: result.contentAssessmentResult?.vocabularyScore,
topicScore: result.contentAssessmentResult?.topicScore,
result: resultJson,
language: language || this.language,
},
{
include: Recording,
}
);
return _pronunciationAssessment.toJSON();
return _pronunciationAssessment;
}
@AfterFind
static async findTarget(findResult: Recording | Recording[]) {
if (!findResult) return;
if (!Array.isArray(findResult)) findResult = [findResult];
for (const instance of findResult) {
@@ -231,14 +243,21 @@ export class Recording extends Model<Recording> {
@AfterCreate
static increaseResourceCache(recording: Recording) {
if (recording.targetType !== "Audio") return;
Audio.findByPk(recording.targetId).then((audio) => {
audio.increment("recordingsCount");
audio.increment("recordingsDuration", {
by: recording.duration,
if (recording.targetType === "Audio") {
Audio.findByPk(recording.targetId).then((audio) => {
audio.increment("recordingsCount");
audio.increment("recordingsDuration", {
by: recording.duration,
});
});
});
} else if (recording.targetType === "Video") {
Video.findByPk(recording.targetId).then((video) => {
video.increment("recordingsCount");
video.increment("recordingsDuration", {
by: recording.duration,
});
});
}
}
@AfterCreate
@@ -283,9 +302,7 @@ export class Recording extends Model<Recording> {
accessToken: settings.getSync("user.accessToken") as string,
logger: log.scope("recording/cleanupFile"),
});
webApi.deleteRecording(recording.id).catch((err) => {
logger.error("deleteRecording failed:", err.message);
});
webApi.deleteRecording(recording.id);
}
static async createFromBlob(
@@ -293,15 +310,10 @@ export class Recording extends Model<Recording> {
type: string;
arrayBuffer: ArrayBuffer;
},
params: {
targetId: string;
targetType: "Audio" | "Video";
duration: number;
referenceId?: number;
referenceText?: string;
}
params: Partial<Attributes<Recording>>
) {
const { targetId, targetType, referenceId, referenceText } = params;
const { targetId, targetType, referenceId, referenceText, language } =
params;
let { duration } = params;
if (blob.arrayBuffer.byteLength === 0) {
@@ -328,34 +340,42 @@ export class Recording extends Model<Recording> {
}
// save recording to file
const file = path.join(
settings.userDataPath(),
"recordings",
`${Date.now()}.wav`
);
const file = path.join(settings.cachePath(), `${Date.now()}.wav`);
await fs.outputFile(file, echogarden.encodeRawAudioToWave(rawAudio));
// hash file
const md5 = await hashFile(file, { algo: "md5" });
const existed = await Recording.findOne({ where: { md5 } });
if (existed) {
fs.remove(file);
return existed;
}
// rename file
const filename = `${md5}.wav`;
fs.renameSync(file, path.join(path.dirname(file), filename));
return this.create(
fs.moveSync(
file,
path.join(settings.userDataPath(), "recordings", filename),
{
targetId,
targetType,
filename,
duration,
md5,
referenceId,
referenceText,
},
{
include: [Audio],
overwrite: true,
}
);
const userId = settings.getSync("user.id");
const id = uuidv5(`${userId}/${md5}`, uuidv5.URL);
return this.create({
id,
targetId,
targetType,
filename,
duration,
md5,
referenceId,
referenceText,
language,
});
}
static notify(recording: Recording, action: "create" | "update" | "destroy") {

3
enjoy/src/main/db/models/transcription.ts
View File

@@ -32,6 +32,9 @@ export class Transcription extends Model<Transcription> {
@Column({ primaryKey: true, type: DataType.UUID })
id: string;
@Column(DataType.STRING)
language: string;
@Column(DataType.UUID)
targetId: string;

3
enjoy/src/main/db/models/video.ts
View File

@@ -46,6 +46,9 @@ export class Video extends Model<Video> {
@Column({ primaryKey: true, type: DataType.UUID })
id: string;
@Column(DataType.STRING)
language: string;
@Column(DataType.STRING)
source: string;

17
enjoy/src/preload.ts
View File

@@ -356,6 +356,23 @@ contextBridge.exposeInMainWorld("__ENJOY_APP__", {
return ipcRenderer.invoke("conversations-destroy", id);
},
},
pronunciationAssessments: {
findAll: (params: { where?: any; offset?: number; limit?: number }) => {
return ipcRenderer.invoke("pronunciation-assessments-find-all", params);
},
findOne: (params: any) => {
return ipcRenderer.invoke("pronunciation-assessments-find-one", params);
},
create: (params: any) => {
return ipcRenderer.invoke("pronunciation-assessments-create", params);
},
update: (id: string, params: any) => {
return ipcRenderer.invoke("pronunciation-assessments-update", id, params);
},
destroy: (id: string) => {
return ipcRenderer.invoke("pronunciation-assessments-destroy", id);
},
},
messages: {
findAll: (params: { where?: any; offset?: number; limit?: number }) => {
return ipcRenderer.invoke("messages-find-all", params);

4
enjoy/src/renderer/components/medias/media-transcription-read-button.tsx
View File

@@ -41,8 +41,8 @@ import {
Trash2Icon,
} from "lucide-react";
import RecordPlugin from "wavesurfer.js/dist/plugins/record";
import { useRecordings } from "@/renderer/hooks";
import { formatDateTime } from "@/renderer/lib/utils";
import { useRecordings } from "@renderer/hooks";
import { formatDateTime } from "@renderer/lib/utils";
import { MediaPlayer, MediaProvider } from "@vidstack/react";
import {
DefaultAudioLayout,

12
enjoy/src/renderer/components/misc/sidebar.tsx
View File

@@ -14,7 +14,6 @@ import {
DropdownMenuSubTrigger,
DropdownMenuItem,
Separator,
toast,
} from "@renderer/components/ui";
import {
SettingsIcon,
@@ -30,6 +29,7 @@ import {
HelpCircleIcon,
ExternalLinkIcon,
NotebookPenIcon,
SpeechIcon,
} from "lucide-react";
import { useLocation, Link } from "react-router-dom";
import { t } from "i18next";
@@ -44,7 +44,6 @@ export const Sidebar = () => {
const { EnjoyApp, cable } = useContext(AppSettingsProviderContext);
useEffect(() => {
console.log("Subscrbing ->");
const channel = new NoticiationsChannel(cable);
channel.subscribe();
}, []);
@@ -108,6 +107,15 @@ export const Sidebar = () => {
testid="sidebar-conversations"
/>
<SidebarItem
href="/pronunciation_assessments"
label={t("sidebar.pronunciationAssessment")}
tooltip={t("sidebar.pronunciationAssessment")}
active={activeTab.startsWith("/pronunciation_assessments")}
Icon={SpeechIcon}
testid="sidebar-pronunciation-assessments"
/>
<SidebarItem
href="/notes"
label={t("sidebar.notes")}

4
enjoy/src/renderer/components/pronunciation-assessments/index.ts
View File

@@ -1,4 +1,6 @@
export * from "./pronunciation-assessment-word-result";
export * from "./pronunciation-assessment-card";
export * from "./pronunciation-assessment-form";
export * from "./pronunciation-assessment-fulltext-result";
export * from "./pronunciation-assessment-score-result";
export * from "./pronunciation-assessment-score-icon";
export * from "./pronunciation-assessment-word-result";

138
enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-card.tsx Normal file
View File

@@ -0,0 +1,138 @@
import {
Button,
DropdownMenu,
DropdownMenuContent,
DropdownMenuTrigger,
DropdownMenuItem,
RadialProgress,
Badge,
} from "@renderer/components/ui";
import { scoreColor } from "./pronunciation-assessment-score-result";
import { t } from "i18next";
import { formatDateTime } from "@/renderer/lib/utils";
import { MoreHorizontalIcon, Trash2Icon } from "lucide-react";
import { Link } from "react-router-dom";
export const PronunciationAssessmentCard = (props: {
pronunciationAssessment: PronunciationAssessmentType;
onSelect: (assessment: PronunciationAssessmentType) => void;
onDelete: (assessment: PronunciationAssessmentType) => void;
}) => {
const { pronunciationAssessment: assessment, onSelect, onDelete } = props;
return (
<div
key={assessment.id}
className="bg-background p-4 rounded-lg border hover:shadow"
>
<div className="flex items-start space-x-4">
<div className="flex-1 flex flex-col min-h-32">
<div className="select-text line-clamp-2 text-muted-foreground font-serif pl-3 border-l-4 mb-4">
{assessment.referenceText || assessment.target.referenceText}
</div>
<div className="flex items-center gap-2 flex-wrap mb-4">
{[
{
label: t("models.pronunciationAssessment.pronunciationScore"),
value: assessment.pronunciationScore,
},
{
label: t("models.pronunciationAssessment.accuracyScore"),
value: assessment.accuracyScore,
},
{
label: t("models.pronunciationAssessment.fluencyScore"),
value: assessment.fluencyScore,
},
{
label: t("models.pronunciationAssessment.completenessScore"),
value: assessment.completenessScore,
},
{
label: t("models.pronunciationAssessment.prosodyScore"),
value: assessment.prosodyScore,
},
{
label: t("models.pronunciationAssessment.grammarScore"),
value: assessment.grammarScore,
},
{
label: t("models.pronunciationAssessment.vocabularyScore"),
value: assessment.vocabularyScore,
},
{
label: t("models.pronunciationAssessment.topicScore"),
value: assessment.topicScore,
},
].map(({ label, value }) => {
if (typeof value === "number") {
return (
<div className="flex items-center space-x-2 mb-2">
<span className="text-muted-foreground text-sm">
{label}:
</span>
<span
className={`text-sm font-bold ${scoreColor(value || 0)}`}
>
{value}
</span>
</div>
);
}
})}
</div>
{["Audio", "Video"].includes(assessment.target?.targetType) && (
<div className="flex items-center gap-2 mb-4">
<span className="text-sm">{t("source")}:</span>
<Link
to={`/${assessment.target.targetType.toLowerCase()}s/${
assessment.target.targetId
}?segmentIndex=${assessment.target.referenceId}`}
className="text-sm"
>
{t(assessment.target?.targetType?.toLowerCase())}
</Link>
</div>
)}
<div className="mt-auto flex items-center gap-4">
{assessment.language && <Badge variant="secondary">{assessment.language}</Badge>}
<div className="text-xs text-muted-foreground">
{formatDateTime(assessment.createdAt)}
</div>
<DropdownMenu>
<DropdownMenuTrigger>
<MoreHorizontalIcon className="w-4 h-4" />
</DropdownMenuTrigger>
<DropdownMenuContent>
<DropdownMenuItem
className="text-destructive cursor-pointer"
onClick={() => onDelete(assessment)}
>
<Trash2Icon className="w-4 h-4 mr-2" />
<span>{t("delete")}</span>
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</div>
</div>
<div className="h-32">
<RadialProgress
className="w-20 h-20 mx-auto mb-2"
ringClassName={`${scoreColor(assessment.pronunciationScore || 0)}`}
progress={assessment.pronunciationScore || 0}
fontSize={24}
/>
<div className="flex justify-center">
<Button
onClick={() => onSelect(assessment)}
variant="secondary"
size="sm"
>
{t("detail")}
</Button>
</div>
</div>
</div>
</div>
);
};

343
enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-form.tsx Normal file
View File

@@ -0,0 +1,343 @@
import {
Button,
Input,
SelectContent,
SelectTrigger,
SelectValue,
Select,
SelectItem,
Form,
FormField,
FormItem,
FormLabel,
FormMessage,
Textarea,
toast,
Tabs,
TabsList,
TabsTrigger,
TabsContent,
} from "@renderer/components/ui";
import { t } from "i18next";
import { useNavigate } from "react-router-dom";
import { useContext, useEffect, useRef, useState } from "react";
import { AppSettingsProviderContext } from "@/renderer/context";
import { LANGUAGES } from "@/constants";
import { z } from "zod";
import { useForm } from "react-hook-form";
import { zodResolver } from "@hookform/resolvers/zod";
import { LoaderIcon, MicIcon, SquareIcon } from "lucide-react";
import WaveSurfer from "wavesurfer.js";
import RecordPlugin from "wavesurfer.js/dist/plugins/record";
const pronunciationAssessmentSchema = z.object({
file: z.instanceof(FileList).optional(),
recording: z.instanceof(Blob).optional(),
language: z.string().min(2),
referenceText: z.string().optional(),
});
export const PronunciationAssessmentForm = () => {
const navigate = useNavigate();
const { EnjoyApp, learningLanguage } = useContext(AppSettingsProviderContext);
const [submitting, setSubmitting] = useState(false);
const form = useForm<z.infer<typeof pronunciationAssessmentSchema>>({
resolver: zodResolver(pronunciationAssessmentSchema),
values: {
language: learningLanguage,
referenceText: "",
},
});
const fileField = form.register("file");
const onSubmit = async (
data: z.infer<typeof pronunciationAssessmentSchema>
) => {
console.log(data);
if ((!data.file || data.file.length === 0) && !data.recording) {
toast.error(t("noFileOrRecording"));
form.setError("recording", { message: t("noFileOrRecording") });
return;
}
const { language, referenceText, file, recording } = data;
let arrayBuffer: ArrayBuffer;
if (recording) {
arrayBuffer = await recording.arrayBuffer();
} else {
arrayBuffer = await new Blob([file[0]]).arrayBuffer();
}
setSubmitting(true);
toast.promise(
EnjoyApp.pronunciationAssessments
.create({
language,
referenceText,
blob: {
type: recording?.type || file[0].type,
arrayBuffer,
},
})
.then(() => {
navigate("/pronunciation_assessments");
})
.finally(() => setSubmitting(false)),
{
loading: t("assessing"),
success: t("assessedSuccessfully"),
error: (err) => err.message,
}
);
};
return (
<div className="max-w-screen-md mx-auto">
<Form {...form}>
<form
onSubmit={form.handleSubmit(onSubmit)}
className="h-full flex flex-col"
>
<Tabs className="mb-6" defaultValue="record">
<TabsList className="mb-2">
<TabsTrigger value="record">{t("record")}</TabsTrigger>
<TabsTrigger value="upload">{t("upload")}</TabsTrigger>
</TabsList>
<TabsContent value="upload">
<div className="grid gap-4">
<FormField
control={form.control}
name="file"
render={() => (
<FormItem className="grid w-full items-center gap-1.5">
<Input
placeholder={t("upload")}
type="file"
className="cursor-pointer"
accept="audio/*"
{...fileField}
/>
<FormMessage />
</FormItem>
)}
/>
</div>
</TabsContent>
<TabsContent value="record">
<div className="grid gap-4 border p-4 rounded-lg">
<FormField
control={form.control}
name="recording"
render={({ field }) => (
<FormItem className="grid w-full items-center gap-1.5">
<Input
placeholder={t("recording")}
type="file"
className="hidden"
accept="audio/*"
{...fileField}
/>
<RecorderButton
onStart={() => {
form.resetField("recording");
}}
onFinish={(blob) => {
field.onChange(blob);
}}
/>
</FormItem>
)}
/>
{form.watch("recording") && (
<div className="">
<audio controls className="w-full">
<source
src={URL.createObjectURL(form.watch("recording"))}
/>
</audio>
</div>
)}
</div>
</TabsContent>
</Tabs>
<div className="mb-6">
<FormField
control={form.control}
name="language"
render={({ field }) => (
<FormItem className="grid w-full items-center gap-1.5">
<FormLabel>{t("language")}</FormLabel>
<Select value={field.value} onValueChange={field.onChange}>
<SelectTrigger>
<SelectValue />
</SelectTrigger>
<SelectContent>
{LANGUAGES.map((language) => (
<SelectItem key={language.code} value={language.code}>
{language.name}
</SelectItem>
))}
</SelectContent>
</Select>
<FormMessage />
</FormItem>
)}
/>
</div>
<div className="mb-6">
<FormField
control={form.control}
name="referenceText"
render={({ field }) => (
<FormItem className="grid w-full items-center gap-1.5">
<FormLabel>{t("referenceText")}</FormLabel>
<Textarea
placeholder={t("inputReferenceTextOrLeaveItBlank")}
className="h-64"
{...field}
/>
<FormMessage />
</FormItem>
)}
/>
</div>
<div className="mt-6">
<Button
disabled={submitting || !form.formState.isDirty}
className="w-full h-12"
data-testid="conversation-form-submit"
size="lg"
type="submit"
>
{submitting && <LoaderIcon className="mr-2 animate-spin" />}
{t("confirm")}
</Button>
</div>
</form>
</Form>
</div>
);
};
const TEN_MINUTES = 60 * 10;
let interval: NodeJS.Timeout;
const RecorderButton = (props: {
onStart?: () => void;
onFinish: (blob: Blob) => void;
}) => {
const { onStart, onFinish } = props;
const { EnjoyApp } = useContext(AppSettingsProviderContext);
const [isRecording, setIsRecording] = useState(false);
const [recorder, setRecorder] = useState<RecordPlugin>();
const [access, setAccess] = useState<boolean>(false);
const [duration, setDuration] = useState<number>(0);
const ref = useRef(null);
const askForMediaAccess = () => {
EnjoyApp.system.preferences.mediaAccess("microphone").then((access) => {
if (access) {
setAccess(true);
} else {
setAccess(false);
toast.warning(t("noMicrophoneAccess"));
}
});
};
const startRecord = () => {
if (isRecording) return;
if (!recorder) {
toast.warning(t("noMicrophoneAccess"));
return;
}
onStart();
RecordPlugin.getAvailableAudioDevices()
.then((devices) => devices.find((d) => d.kind === "audioinput"))
.then((device) => {
if (device) {
recorder.startRecording({ deviceId: device.deviceId });
setIsRecording(true);
setDuration(0);
interval = setInterval(() => {
setDuration((duration) => {
if (duration >= TEN_MINUTES) {
recorder.stopRecording();
}
return duration + 0.1;
});
}, 100);
} else {
toast.error(t("cannotFindMicrophone"));
}
});
};
useEffect(() => {
if (!access) return;
if (!ref?.current) return;
const ws = WaveSurfer.create({
container: ref.current,
fillParent: true,
height: 40,
autoCenter: false,
normalize: false,
});
const record = ws.registerPlugin(RecordPlugin.create());
setRecorder(record);
record.on("record-end", async (blob: Blob) => {
if (interval) clearInterval(interval);
onFinish(blob);
setIsRecording(false);
});
return () => {
if (interval) clearInterval(interval);
recorder?.stopRecording();
ws?.destroy();
};
}, [access, ref]);
useEffect(() => {
askForMediaAccess();
}, []);
return (
<div className="w-full">
<div className="flex items-center justify-center">
<Button
type="button"
variant="ghost"
className="aspect-square p-0 h-12 rounded-full bg-red-500 hover:bg-red-500/90"
onClick={() => {
if (isRecording) {
recorder?.stopRecording();
} else {
startRecord();
}
}}
>
{isRecording ? (
<SquareIcon fill="white" className="w-6 h-6 text-white" />
) : (
<MicIcon className="w-6 h-6 text-white" />
)}
</Button>
</div>
<div className="w-full flex items-center">
<div
ref={ref}
className={isRecording ? "w-full mr-4" : "h-0 overflow-hidden"}
></div>
{isRecording && (
<div className="text-muted-foreground text-sm w-24">
{duration.toFixed(1)} / {TEN_MINUTES}
</div>
)}
</div>
</div>
);
};

2
enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-fulltext-result.tsx
View File

@@ -55,7 +55,7 @@ export const PronunciationAssessmentFulltextResult = (props: {
}, []);
return (
<ScrollArea className="h-72 py-4 px-8">
<ScrollArea className="min-h-72 py-4 px-8">
<div className="flex items-start justify-between space-x-6">
<div className="flex-1 py-4">
{words.map((result, index: number) => (

8
enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-score-result.tsx
View File

@@ -9,11 +9,11 @@ export const PronunciationAssessmentScoreResult = (props: {
fluencyScore?: number;
completenessScore?: number;
prosodyScore?: number;
assessing: boolean;
onAssess: () => void;
assessing?: boolean;
onAssess?: () => void;
}) => {
const {
assessing,
assessing = false,
onAssess,
pronunciationScore,
accuracyScore,
@@ -142,7 +142,7 @@ const ScoreBarComponent = ({
);
};
const scoreColor = (score: number, type: "text" | "bg" = "text") => {
export const scoreColor = (score: number, type: "text" | "bg" = "text") => {
if (!score) return "gray";
if (score >= 80) return type == "text" ? "text-green-600" : "bg-green-600";

10
enjoy/src/renderer/components/recordings/recording-detail.tsx
View File

@@ -8,11 +8,15 @@ import { useState, useContext } from "react";
import { AppSettingsProviderContext } from "@renderer/context";
import { Tooltip } from "react-tooltip";
export const RecordingDetail = (props: { recording: RecordingType }) => {
export const RecordingDetail = (props: {
recording: RecordingType;
pronunciationAssessment?: PronunciationAssessmentType;
}) => {
const { recording } = props;
if (!recording) return;
const { pronunciationAssessment } = recording;
const pronunciationAssessment =
props.pronunciationAssessment || recording.pronunciationAssessment;
const { result } = pronunciationAssessment || {};
const [currentTime, setCurrentTime] = useState<number>(0);
const [seek, setSeek] = useState<{
@@ -58,7 +62,7 @@ export const RecordingDetail = (props: { recording: RecordingType }) => {
}}
/>
) : (
<ScrollArea className="h-72 py-4 px-8 select-text">
<ScrollArea className="min-h-72 py-4 px-8 select-text">
{(recording?.referenceText || "").split("\n").map((line, index) => (
<div key={index} className="text-xl font-serif tracking-wide mb-2">
{line}

36
enjoy/src/renderer/components/ui/badage.tsx Normal file
View File

@@ -0,0 +1,36 @@
import * as React from "react";
import { cva, type VariantProps } from "class-variance-authority";
import { cn } from "@renderer/lib/utils";
const badgeVariants = cva(
"inline-flex items-center rounded-md border px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2",
{
variants: {
variant: {
default:
"border-transparent bg-primary text-primary-foreground shadow hover:bg-primary/80",
secondary:
"border-transparent bg-secondary text-secondary-foreground hover:bg-secondary/80",
destructive:
"border-transparent bg-destructive text-destructive-foreground shadow hover:bg-destructive/80",
outline: "text-foreground",
},
},
defaultVariants: {
variant: "default",
},
}
);
export interface BadgeProps
extends React.HTMLAttributes<HTMLDivElement>,
VariantProps<typeof badgeVariants> {}
function Badge({ className, variant, ...props }: BadgeProps) {
return (
<div className={cn(badgeVariants({ variant }), className)} {...props} />
);
}
export { Badge, badgeVariants };

1
enjoy/src/renderer/components/ui/index.ts
View File

@@ -1,5 +1,6 @@
export * from "./accordion";
export * from "./alert";
export * from "./badage";
export * from "./button";
export * from "./menubar";
export * from "./progress";

164
enjoy/src/renderer/pages/pronunciation-assessments/index.tsx Normal file
View File

@@ -0,0 +1,164 @@
import { useEffect, useState, useContext } from "react";
import { AppSettingsProviderContext } from "@renderer/context";
import {
Alert,
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
Button,
Sheet,
SheetClose,
SheetContent,
SheetHeader,
toast,
} from "@renderer/components/ui";
import { ChevronDownIcon, ChevronLeftIcon } from "lucide-react";
import { useNavigate } from "react-router-dom";
import { t } from "i18next";
import {
PronunciationAssessmentCard,
RecordingDetail,
} from "@renderer/components";
export default () => {
const navigate = useNavigate();
const { EnjoyApp } = useContext(AppSettingsProviderContext);
const [assessments, setAssessments] = useState<PronunciationAssessmentType[]>(
[]
);
const [hasMore, setHasMore] = useState<boolean>(true);
const [selecting, setSelecting] =
useState<PronunciationAssessmentType | null>(null);
const [deleting, setDeleting] = useState<PronunciationAssessmentType | null>(
null
);
const handleDelete = async (assessment: PronunciationAssessmentType) => {
try {
await EnjoyApp.pronunciationAssessments.destroy(assessment.id);
setAssessments(assessments.filter((a) => a.id !== assessment.id));
setDeleting(null);
} catch (err) {
toast.error(err.message);
}
};
const fetchAssessments = (params?: { offset: number; limit?: number }) => {
const { offset = 0, limit = 10 } = params || {};
if (offset > 0 && !hasMore) return;
EnjoyApp.pronunciationAssessments
.findAll({
limit,
offset,
})
.then((fetchedAssessments) => {
if (offset === 0) {
setAssessments(fetchedAssessments);
} else {
setAssessments([...assessments, ...fetchedAssessments]);
}
setHasMore(fetchedAssessments.length === limit);
})
.catch((err) => {
toast.error(err.message);
});
};
useEffect(() => {
fetchAssessments();
}, []);
return (
<div className="h-full px-4 py-6 lg:px-8">
<div className="max-w-5xl mx-auto">
<div className="flex space-x-1 items-center mb-4">
<Button variant="ghost" size="icon" onClick={() => navigate(-1)}>
<ChevronLeftIcon className="w-5 h-5" />
</Button>
<span>{t("sidebar.pronunciationAssessment")}</span>
</div>
<div className="flex items-center justify-start mb-4">
<Button onClick={() => navigate("/pronunciation_assessments/new")}>
{t("newAssessment")}
</Button>
</div>
<div className="grid grid-cols-1 gap-4 mb-4">
{assessments.map((assessment) => (
<PronunciationAssessmentCard
key={assessment.id}
pronunciationAssessment={assessment}
onSelect={setSelecting}
onDelete={setDeleting}
/>
))}
</div>
{hasMore && (
<div className="flex justify-center">
<Button
variant="secondary"
onClick={() => fetchAssessments({ offset: assessments.length })}
>
{t("loadMore")}
</Button>
</div>
)}
</div>
<Sheet
open={Boolean(selecting)}
onOpenChange={(value) => {
if (!value) setSelecting(null);
}}
>
<SheetContent
side="bottom"
className="rounded-t-2xl shadow-lg max-h-screen overflow-y-scroll"
displayClose={false}
>
<SheetHeader className="flex items-center justify-center -mt-4 mb-2">
<SheetClose>
<ChevronDownIcon />
</SheetClose>
</SheetHeader>
{selecting && (
<RecordingDetail
recording={selecting.target}
pronunciationAssessment={selecting}
/>
)}
</SheetContent>
</Sheet>
<AlertDialog
open={Boolean(deleting)}
onOpenChange={(value) => {
if (!value) setDeleting(null);
}}
>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>{t("delete")}</AlertDialogTitle>
<AlertDialogDescription>
{t("areYouSureToDeleteThisAssessment")}
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>{t("cancel")}</AlertDialogCancel>
<AlertDialogAction onClick={() => handleDelete(deleting)}>
{t("confirm")}
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</div>
);
};

29
enjoy/src/renderer/pages/pronunciation-assessments/new.tsx Normal file
View File

@@ -0,0 +1,29 @@
import { ChevronLeftIcon } from "lucide-react";
import { t } from "i18next";
import { Link, useNavigate } from "react-router-dom";
import { Button } from "@renderer/components/ui";
import { PronunciationAssessmentForm } from "@renderer/components";
export default () => {
const navigate = useNavigate();
return (
<div className="h-full px-4 py-6 lg:px-8">
<div className="max-w-5xl mx-auto">
<div className="flex space-x-1 items-center mb-4">
<Button variant="ghost" size="icon" onClick={() => navigate(-1)}>
<ChevronLeftIcon className="w-5 h-5" />
</Button>
<Link to="/pronunciation_assessments">
{t("sidebar.pronunciationAssessment")}
</Link>
<span>/</span>
<span>{t("newAssessment")}</span>
</div>
<div className="">
<PronunciationAssessmentForm />
</div>
</div>
</div>
);
};

10
enjoy/src/renderer/router.tsx
View File

@@ -18,6 +18,8 @@ import Home from "./pages/home";
import Community from "./pages/community";
import StoryPreview from "./pages/story-preview";
import Notes from "./pages/notes";
import PronunciationAssessmentsIndex from "./pages/pronunciation-assessments/index";
import PronunciationAssessmentsNew from "./pages/pronunciation-assessments/new";
export default createHashRouter([
{
@@ -46,6 +48,14 @@ export default createHashRouter([
path: "/conversations/:id",
element: <Conversation />,
},
{
path: "/pronunciation_assessments",
element: <PronunciationAssessmentsIndex />,
},
{
path: "/pronunciation_assessments/new",
element: <PronunciationAssessmentsNew />,
},
{
path: "/vocabulary",
element: <Vocabulary />,

7
enjoy/src/types/enjoy-app.d.ts vendored
View File

@@ -203,6 +203,13 @@ type EnjoyAppType = {
targetType
) => Promise<SegementRecordingStatsType>;
};
pronunciationAssessments: {
findAll: (params: any) => Promise<PronunciationAssessmentType[]>;
findOne: (params: any) => Promise<PronunciationAssessmentType>;
create: (params: any) => Promise<PronunciationAssessmentType>;
update: (id: string, params: any) => Promise<PronunciationAssessmentType>;
destroy: (id: string) => Promise<void>;
};
conversations: {
findAll: (params: any) => Promise<ConversationType[]>;
findOne: (params: any) => Promise<ConversationType>;

6
enjoy/src/types/pronunciation-assessment.d.ts vendored
View File

@@ -1,6 +1,9 @@
type PronunciationAssessmentType = {
id: string;
recordingId: string;
language?: string;
targetId: string;
targetType: string;
referenceText: string;
accuracyScore: number;
completenessScore: number;
fluencyScore: number;
@@ -28,6 +31,7 @@ type PronunciationAssessmentType = {
updatedAt: Date;
isSynced: boolean;
recording?: RecodingType;
target?: RecodingType;
};
type PronunciationAssessmentWordResultType = {

2
package.json
View File

@@ -16,7 +16,7 @@
"enjoy:make": "yarn workspace enjoy make",
"enjoy:publish": "yarn workspace enjoy publish",
"enjoy:lint": "yarn workspace enjoy eslint --ext .ts,.tsx .",
"enjoy:create-migration": "yarn workspace enjoy zx ./src/main/db/create-migration.mjs",
"enjoy:create-migration": "yarn workspace enjoy create-migration",
"docs:dev": "yarn workspace 1000-hours dev",
"docs:build": "yarn workspace 1000-hours build",
"docs:preview": "yarn workspace 1000-hours preview"