From 0644c3bbd7e38e6b721d01605497bc5adc87f410 Mon Sep 17 00:00:00 2001 From: an-lee Date: Fri, 26 Apr 2024 15:05:36 +0800 Subject: [PATCH] Feat: make notes on caption (#544) * add segment model * add note model * db handle segment & note * add notes & segments handler * refactor media caption components * segment & note create * fix type * update note column & may sync * display selected words for note * refactor selected words * auto select words when editing note * refactor * refactor caption component * display notes * refactor notes components * fix * refactor segment & notes into context * destroy note * update locale * fix caption switch issue * fix layout * refactor caption layout * remove deprecated code * may share note * improve UI * fix notes list auto update after created * remove console.log * add notes page * refactor note parameters * refactor components * mark note on transcription * handle no notes * improve style * improve style * show context menu on selection text * fix utils --- enjoy/src/api/client.ts | 14 +- enjoy/src/i18n/en.json | 18 +- enjoy/src/i18n/zh-CN.json | 22 +- enjoy/src/main.ts | 5 +- enjoy/src/main/db/handlers/index.ts | 8 +- enjoy/src/main/db/handlers/notes-handler.ts | 170 ++++++++ .../src/main/db/handlers/segments-handler.ts | 62 +++ enjoy/src/main/db/index.ts | 10 +- .../1713590784186-create-segment.js | 70 ++++ .../migrations/1713690537982-create-note.js | 45 +++ enjoy/src/main/db/models/index.ts | 20 +- enjoy/src/main/db/models/note.ts | 155 ++++++++ enjoy/src/main/db/models/segment.ts | 246 ++++++++++++ enjoy/src/main/db/models/transcription.ts | 3 +- enjoy/src/main/ffmpeg.ts | 38 ++ enjoy/src/main/utils.ts | 2 +- enjoy/src/preload.ts | 47 +++ .../components/audios/audio-player.tsx | 23 +- .../conversations/conversation-shortcuts.tsx | 2 +- enjoy/src/renderer/components/index.ts | 10 +- enjoy/src/renderer/components/medias/index.ts | 1 - .../components/medias/media-caption.tsx | 346 +++++++++-------- .../components/medias/media-captions/index.ts | 4 + .../media-captions/media-caption-tabs.tsx | 67 ++++ .../media-captions/tab-content-analysis.tsx | 133 +++++++ .../media-captions/tab-content-note.tsx | 93 +++++ .../tab-content-translation.tsx} | 363 ++++-------------- .../medias/media-player-controls.tsx | 20 +- .../components/medias/media-transcription.tsx | 28 +- enjoy/src/renderer/components/notes/index.ts | 4 + .../renderer/components/notes/note-card.tsx | 165 ++++++++ .../renderer/components/notes/note-form.tsx | 110 ++++++ .../components/notes/note-segment-group.tsx | 106 +++++ .../components/notes/note-segment.tsx | 80 ++++ enjoy/src/renderer/components/posts/index.ts | 8 +- .../renderer/components/posts/post-audio.tsx | 149 +------ .../renderer/components/posts/post-card.tsx | 10 + .../renderer/components/posts/post-note.tsx | 20 + enjoy/src/renderer/components/posts/posts.tsx | 11 +- .../components/videos/video-player.tsx | 27 +- .../src/renderer/components/widgets/index.ts | 13 +- .../renderer/components/widgets/sidebar.tsx | 9 + .../components/widgets/wavesurfer-player.tsx | 141 +++++++ .../context/media-player-provider.tsx | 30 +- enjoy/src/renderer/hooks/index.ts | 22 +- enjoy/src/renderer/hooks/use-notes.tsx | 97 +++++ enjoy/src/renderer/hooks/use-segments.tsx | 54 +++ enjoy/src/renderer/lib/utils.ts | 7 +- enjoy/src/renderer/pages/audio.tsx | 6 +- enjoy/src/renderer/pages/conversations.tsx | 2 +- enjoy/src/renderer/pages/notes.tsx | 88 +++++ enjoy/src/renderer/pages/video.tsx | 6 +- enjoy/src/renderer/router.tsx | 5 + enjoy/src/types/enjoy-app.d.ts | 31 ++ enjoy/src/types/note.d.ts | 14 + enjoy/src/types/post.d.ts | 4 +- enjoy/src/types/segment.d.ts | 18 + enjoy/src/types/transcription.d.ts | 1 + 58 files changed, 2586 insertions(+), 677 deletions(-) create mode 100644 enjoy/src/main/db/handlers/notes-handler.ts create mode 100644 enjoy/src/main/db/handlers/segments-handler.ts create mode 100644 enjoy/src/main/db/migrations/1713590784186-create-segment.js create mode 100644 enjoy/src/main/db/migrations/1713690537982-create-note.js create mode 100644 enjoy/src/main/db/models/note.ts create mode 100644 enjoy/src/main/db/models/segment.ts create mode 100644 enjoy/src/renderer/components/medias/media-captions/index.ts create mode 100644 enjoy/src/renderer/components/medias/media-captions/media-caption-tabs.tsx create mode 100644 enjoy/src/renderer/components/medias/media-captions/tab-content-analysis.tsx create mode 100644 enjoy/src/renderer/components/medias/media-captions/tab-content-note.tsx rename enjoy/src/renderer/components/medias/{media-caption-tabs.tsx => media-captions/tab-content-translation.tsx} (58%) create mode 100644 enjoy/src/renderer/components/notes/index.ts create mode 100644 enjoy/src/renderer/components/notes/note-card.tsx create mode 100644 enjoy/src/renderer/components/notes/note-form.tsx create mode 100644 enjoy/src/renderer/components/notes/note-segment-group.tsx create mode 100644 enjoy/src/renderer/components/notes/note-segment.tsx create mode 100644 enjoy/src/renderer/components/posts/post-note.tsx create mode 100644 enjoy/src/renderer/components/widgets/wavesurfer-player.tsx create mode 100644 enjoy/src/renderer/hooks/use-notes.tsx create mode 100644 enjoy/src/renderer/hooks/use-segments.tsx create mode 100644 enjoy/src/renderer/pages/notes.tsx create mode 100644 enjoy/src/types/note.d.ts create mode 100644 enjoy/src/types/segment.d.ts diff --git a/enjoy/src/api/client.ts b/enjoy/src/api/client.ts index fc5e7079..59d8aba3 100644 --- a/enjoy/src/api/client.ts +++ b/enjoy/src/api/client.ts @@ -166,7 +166,7 @@ export class Client { page?: number; items?: number; userId?: string; - type?: "all" | "recording" | "medium" | "story" | "prompt" | "text" | "gpt"; + type?: "all" | "recording" | "medium" | "story" | "prompt" | "text" | "gpt" | "note"; by?: "following" | "all"; }): Promise< { @@ -232,6 +232,18 @@ export class Client { return this.api.post("/api/transcriptions", decamelizeKeys(transcription)); } + syncSegment(segment: Partial>) { + return this.api.post("/api/segments", decamelizeKeys(segment)); + } + + syncNote(note: Partial>) { + return this.api.post("/api/notes", decamelizeKeys(note)); + } + + deleteNote(id: string) { + return this.api.delete(`/api/notes/${id}`); + } + syncRecording(recording: Partial) { if (!recording) return; diff --git a/enjoy/src/i18n/en.json b/enjoy/src/i18n/en.json index 00296f85..b9c8e21a 100644 --- a/enjoy/src/i18n/en.json +++ b/enjoy/src/i18n/en.json @@ -141,7 +141,8 @@ "translator": "Translator", "mine": "Mine", "preferences": "Preferences", - "profile": "My Profile" + "profile": "My Profile", + "notes": "Note" }, "form": { "lengthMustBeAtLeast": "{{field}} must be at least {{length}} characters", @@ -166,6 +167,7 @@ "autoScroll": "auto scroll", "translate": "translate", "displayIpa": "display IPA", + "displayNotes": "display Notes", "detail": "detail", "remove": "remove", "share": "share", @@ -470,6 +472,9 @@ "shareGpt": "Share GPT", "sharedGpt": "Shared a GPT", "areYouSureToShareThisGptToCommunity": "Are you sure to share this GPT to community?", + "shareNote": "Share note", + "sharedNote": "Shared a note", + "areYouSureToShareThisNoteToCommunity": "Are you sure to share this note to community?", "saveAiAssistant": "Save this AI assistant", "addToLibary": "Add to library", "areYouSureToAddThisVideoToYourLibrary": "Are you sure to add this video to library?", @@ -550,8 +555,17 @@ "storyType": "Story", "promptType": "Prompt", "gptType": "GPT", + "noteType": "Note", "follow": "follow", "unfollow": "unfollow", "noFollowersYet": "No followers yet", - "notFollowingAnyoneYet": "Not following anyone yet" + "notFollowingAnyoneYet": "Not following anyone yet", + "startToNote": "Start to note", + "newNote": "New note", + "writeNoteHere": "Write your note here", + "deleteNote": "Delete note", + "areYouSureToDeleteThisNote": "Are you sure to delete this note?", + "notesCount": "{{count}} notes", + "source": "source", + "noNotesYet": "No notes yet" } diff --git a/enjoy/src/i18n/zh-CN.json b/enjoy/src/i18n/zh-CN.json index df3b8a8f..7c0de682 100644 --- a/enjoy/src/i18n/zh-CN.json +++ b/enjoy/src/i18n/zh-CN.json @@ -141,7 +141,8 @@ "translator": "翻译助手", "mine": "我的", "preferences": "软件设置", - "profile": "个人主页" + "profile": "个人主页", + "notes": "笔记" }, "form": { "lengthMustBeAtLeast": "{{field}} 长度不可超过 {{length}} 个字符", @@ -166,6 +167,7 @@ "autoScroll": "自动滚动", "translate": "翻译", "displayIpa": "标注音标", + "displayNotes": "显示笔记", "detail": "详情", "remove": "删除", "share": "分享", @@ -449,7 +451,7 @@ "square": "广场", "noOneSharedYet": "还没有人分享", "sharedSuccessfully": "分享成功", - "sharedFailed": "分享失败", + "shareFailed": "分享失败", "shareAudio": "分享音频", "sharedAudio": "分享了一个音频材料", "areYouSureToShareThisAudioToCommunity": "您确定要分享此音频到社区吗？", @@ -469,6 +471,9 @@ "shareGpt": "分享智能助手", "sharedGpt": "分享了一个智能助手", "areYouSureToShareThisGptToCommunity": "您确定要将这个智能助手分享到社区吗？", + "shareNote": "分享笔记", + "sharedNote": "分享了一条笔记", + "areYouSureToShareThisNoteToCommunity": "您确定要将这条笔记分享到社区吗？", "saveAiAssistant": "保存智能助手", "addToLibary": "添加到资源库", "areYouSureToAddThisVideoToYourLibrary": "您确定要添加此视频到资料库吗？", @@ -522,7 +527,7 @@ "autoGroup": "智能断句", "captionTabs": { "selected": "选词", - "translation": "整句翻译", + "translation": "翻译", "analysis": "句子分析", "note": "笔记" }, @@ -549,8 +554,17 @@ "storyType": "文章", "promptType": "提示语", "gptType": "智能助手", + "noteType": "笔记", "follow": "关注", "unfollow": "取消关注", "noFollowersYet": "还没有人关注", - "notFollowingAnyoneYet": "还没有关注任何人" + "notFollowingAnyoneYet": "还没有关注任何人", + "startToNote": "开始做笔记", + "newNote": "新笔记", + "writeNoteHere": "开始做笔记", + "deleteNote": "删除笔记", + "areYouSureToDeleteThisNote": "您确定要删除这条笔记吗？", + "notesCount": "{{count}} 条笔记", + "source": "来源", + "noNotesYet": "还没有笔记" } diff --git a/enjoy/src/main.ts b/enjoy/src/main.ts index 8f7089ac..8ade6951 100644 --- a/enjoy/src/main.ts +++ b/enjoy/src/main.ts @@ -37,6 +37,7 @@ contextMenu({ showInspectElement: false, showLookUpSelection: false, showLearnSpelling: false, + showSelectAll: false, labels: { copy: t("copy"), cut: t("cut"), @@ -44,7 +45,7 @@ contextMenu({ selectAll: t("selectAll"), }, shouldShowMenu: (_event, params) => { - return params.isEditable; + return params.isEditable || !!params.selectionText; }, }); @@ -75,7 +76,7 @@ protocol.registerSchemesAsPrivileged([ app.on("ready", async () => { protocol.handle("enjoy", (request) => { let url = request.url.replace("enjoy://", ""); - if (url.match(/library\/(audios|videos|recordings|speeches)/g)) { + if (url.match(/library\/(audios|videos|recordings|speeches|segments)/g)) { url = url.replace("library/", ""); url = path.join(settings.userDataPath(), url); } else if (url.startsWith("library")) { diff --git a/enjoy/src/main/db/handlers/index.ts b/enjoy/src/main/db/handlers/index.ts index d3a994a2..5ef6e0e0 100644 --- a/enjoy/src/main/db/handlers/index.ts +++ b/enjoy/src/main/db/handlers/index.ts @@ -1,8 +1,10 @@ export * from './audios-handler'; -export * from './recordings-handler'; -export * from './messages-handler'; -export * from './conversations-handler'; export * from './cache-objects-handler'; +export * from './conversations-handler'; +export * from './messages-handler'; +export * from './notes-handler'; +export * from './recordings-handler'; export * from './speeches-handler'; +export * from './segments-handler'; export * from './transcriptions-handler'; export * from './videos-handler'; diff --git a/enjoy/src/main/db/handlers/notes-handler.ts b/enjoy/src/main/db/handlers/notes-handler.ts new file mode 100644 index 00000000..5e53203e --- /dev/null +++ b/enjoy/src/main/db/handlers/notes-handler.ts @@ -0,0 +1,170 @@ +import { ipcMain, IpcMainEvent } from "electron"; +import { Note, Segment } from "@main/db/models"; +import { Sequelize } from "sequelize"; + +class NotesHandler { + private async groupByTarget( + _event: IpcMainEvent, + params: { + limit?: number; + offset?: number; + } + ) { + const { limit, offset } = params; + + return Note.findAll({ + include: [Segment], + attributes: [ + "targetId", + "targetType", + [Sequelize.fn("COUNT", Sequelize.col("note.id")), "count"], + ], + group: ["targetId", "targetType"], + order: [["created_at", "DESC"]], + limit, + offset, + }).then((notes) => notes.map((note) => note.toJSON())); + } + + private async groupBySegment( + _event: IpcMainEvent, + targetId: string, + targetType: string + ) { + return Note.findAll({ + include: [ + { + model: Segment, + as: "segment", + attributes: ["id", "segmentIndex"], + }, + ], + attributes: [ + "targetId", + "targetType", + [Sequelize.fn("COUNT", Sequelize.col("note.id")), "count"], + ], + group: ["targetId", "targetType"], + where: { + "$segment.target_id$": targetId, + "$segment.target_type$": targetType, + }, + }).then((notes) => notes.map((note) => note.toJSON())); + } + + private async findAll( + _event: IpcMainEvent, + params: { + targetId?: string; + targetType?: string; + limit?: number; + offset?: number; + } + ) { + const { targetId, targetType, limit, offset } = params; + + const where: any = {}; + if (targetId && targetType) { + where["targetId"] = targetId; + where["targetType"] = targetType; + } + + const notes = await Note.findAll({ + where, + limit: limit, + offset: offset, + include: [Segment], + order: [["createdAt", "DESC"]], + }); + + return notes.map((note) => note.toJSON()); + } + + private async find(_event: IpcMainEvent, id: string) { + const note = await Note.findByPk(id); + return note.toJSON(); + } + + private async update( + _event: IpcMainEvent, + id: string, + params: { + content: string; + parameters: any; + } + ) { + const note = await Note.findByPk(id); + if (!note) { + throw new Error("Note not found"); + } + + await note.update({ + content: params.content, + parameters: params.parameters, + }); + + return note.toJSON(); + } + + private async delete(_event: IpcMainEvent, id: string) { + const note = await Note.findByPk(id); + if (!note) { + throw new Error("Note not found"); + } + + note.destroy(); + } + + private async create( + _event: IpcMainEvent, + params: { + targetId: string; + targetType: string; + content: string; + parameters: any; + } + ) { + const { targetId, targetType, content, parameters } = params; + + switch (targetType) { + case "Segment": + const segment = await Segment.findByPk(targetId); + if (!segment) { + throw new Error("Segment not found"); + } + break; + default: + throw new Error("Invalid target"); + } + + return Note.create({ + targetId, + targetType, + content, + parameters, + }); + } + + private async sync(_event: IpcMainEvent, id: string) { + const note = await Note.findByPk(id); + if (!note) { + throw new Error("Note not found"); + } + + await note.sync(); + return note.toJSON(); + } + + register() { + ipcMain.handle("notes-group-by-target", this.groupByTarget); + ipcMain.handle("notes-group-by-segment", this.groupBySegment); + ipcMain.handle("notes-find-all", this.findAll); + ipcMain.handle("notes-find", this.find); + ipcMain.handle("notes-update", this.update); + ipcMain.handle("notes-delete", this.delete); + ipcMain.handle("notes-create", this.create); + ipcMain.handle("notes-sync", this.sync); + } +} + +export const notesHandler = new NotesHandler(); diff --git a/enjoy/src/main/db/handlers/segments-handler.ts b/enjoy/src/main/db/handlers/segments-handler.ts new file mode 100644 index 00000000..178a5547 --- /dev/null +++ b/enjoy/src/main/db/handlers/segments-handler.ts @@ -0,0 +1,62 @@ +import { ipcMain, IpcMainEvent } from "electron"; +import { Audio, Segment, Video } from "@main/db/models"; + +class SegmentsHandler { + private async find(_event: IpcMainEvent, id: string) { + const segment = await Segment.findByPk(id); + return segment.toJSON(); + } + + private async findAll( + _event: IpcMainEvent, + params: { + targetId: string; + targetType: string; + segmentIndex: number; + } + ) { + const { targetId, targetType, segmentIndex } = params; + const segments = await Segment.findAll({ + where: { + targetId, + targetType, + segmentIndex, + }, + include: [Audio, Video], + }); + + return segments.map((segment) => segment.toJSON()); + } + + private async create( + _event: IpcMainEvent, + params: { + targetId: string; + targetType: string; + segmentIndex: number; + } + ) { + const segment = await Segment.generate({ + targetId: params.targetId, + targetType: params.targetType, + segmentIndex: params.segmentIndex, + }); + return segment.toJSON(); + } + + private async sync(_event: IpcMainEvent, id: string) { + const segment = await Segment.findByPk(id); + await segment.sync(); + await segment.upload(); + return segment.toJSON(); + } + + register() { + ipcMain.handle("segments-create", this.create); + ipcMain.handle("segments-find", this.find); + ipcMain.handle("segments-find-all", this.findAll); + ipcMain.handle("segments-sync", this.sync); + } +} + +export const segmentsHandler = new SegmentsHandler(); diff --git a/enjoy/src/main/db/index.ts b/enjoy/src/main/db/index.ts index 64141041..eacda7a0 100644 --- a/enjoy/src/main/db/index.ts +++ b/enjoy/src/main/db/index.ts @@ -8,7 +8,9 @@ import { CacheObject, Conversation, Message, + Note, PronunciationAssessment, + Segment, Speech, Transcription, Video, @@ -18,7 +20,9 @@ import { cacheObjectsHandler, conversationsHandler, messagesHandler, + notesHandler, recordingsHandler, + segmentsHandler, speechesHandler, transcriptionsHandler, videosHandler, @@ -47,8 +51,10 @@ db.connect = async () => { CacheObject, Conversation, Message, + Note, PronunciationAssessment, Recording, + Segment, Speech, Transcription, Video, @@ -92,9 +98,11 @@ db.connect = async () => { // register handlers audiosHandler.register(); cacheObjectsHandler.register(); - recordingsHandler.register(); conversationsHandler.register(); messagesHandler.register(); + notesHandler.register(); + recordingsHandler.register(); + segmentsHandler.register(); speechesHandler.register(); transcriptionsHandler.register(); videosHandler.register(); diff --git a/enjoy/src/main/db/migrations/1713590784186-create-segment.js b/enjoy/src/main/db/migrations/1713590784186-create-segment.js new file mode 100644 index 00000000..3dc75390 --- /dev/null +++ b/enjoy/src/main/db/migrations/1713590784186-create-segment.js @@ -0,0 +1,70 @@ +import { DataTypes } from "sequelize"; + +async function up({ context: queryInterface }) { + queryInterface.createTable( + "segments", + { + id: { + type: DataTypes.UUID, + defaultValue: DataTypes.UUIDV4, + primaryKey: true, + allowNull: false, + }, + target_type: { + type: DataTypes.STRING, + allowNull: false, + }, + target_id: { + type: DataTypes.UUID, + allowNull: false, + }, + md5: { + type: DataTypes.STRING, + allowNull: false, + }, + segment_index: { + type: DataTypes.INTEGER, + allowNull: false, + }, + caption: { + type: DataTypes.JSON, + }, + start_time: { + type: DataTypes.NUMBER, + }, + end_time: { + type: DataTypes.NUMBER, + }, + synced_at: { + type: DataTypes.DATE, + }, + uploaded_at: { + type: DataTypes.DATE, + }, + created_at: { + type: DataTypes.DATE, + allowNull: false, + }, + updated_at: { + type: DataTypes.DATE, + allowNull: false, + }, + }, + { + indexes: [ + { + fields: ["target_type", "target_id"], + }, + { + fields: ["md5"], + }, + ], + } + ); +} + +async function down({ context: queryInterface }) { + queryInterface.dropTable("segments"); +} + +export { up, down }; diff --git a/enjoy/src/main/db/migrations/1713690537982-create-note.js b/enjoy/src/main/db/migrations/1713690537982-create-note.js new file mode 100644 index 00000000..aec29e11 --- /dev/null +++ b/enjoy/src/main/db/migrations/1713690537982-create-note.js @@ -0,0 +1,45 @@ +import { DataTypes } from "sequelize"; + +async function up({ context: queryInterface }) { + queryInterface.createTable("notes", { + id: { + type: DataTypes.UUID, + defaultValue: DataTypes.UUIDV4, + primaryKey: true, + allowNull: false, + }, + target_type: { + type: DataTypes.STRING, + allowNull: false, + }, + target_id: { + type: DataTypes.UUID, + allowNull: false, + }, + parameters: { + type: DataTypes.JSON, + defaultValue: {}, + }, + content: { + type: DataTypes.TEXT, + allowNull: false, + }, + synced_at: { + type: DataTypes.DATE, + }, + created_at: { + type: DataTypes.DATE, + allowNull: false, + }, + updated_at: { + type: DataTypes.DATE, + allowNull: false, + }, + }); +} + +async function down({ context: queryInterface }) { + queryInterface.dropTable("notes"); +} + +export { up, down }; diff --git a/enjoy/src/main/db/models/index.ts b/enjoy/src/main/db/models/index.ts index 808fc2fa..c55d7b15 100644 --- a/enjoy/src/main/db/models/index.ts +++ b/enjoy/src/main/db/models/index.ts @@ -1,9 +1,11 @@ -export * from './audio'; -export * from './recording'; -export * from './conversation'; -export * from './message'; -export * from './speech'; -export * from './pronunciation-assessment'; -export * from './cache-object'; -export * from './transcription'; -export * from './video'; +export * from "./audio"; +export * from "./cache-object"; +export * from "./conversation"; +export * from "./message"; +export * from "./note"; +export * from "./pronunciation-assessment"; +export * from "./recording"; +export * from "./segment"; +export * from "./speech"; +export * from "./transcription"; +export * from "./video"; diff --git a/enjoy/src/main/db/models/note.ts b/enjoy/src/main/db/models/note.ts new file mode 100644 index 00000000..f91bb527 --- /dev/null +++ b/enjoy/src/main/db/models/note.ts @@ -0,0 +1,155 @@ +import { + AfterUpdate, + AfterDestroy, + BelongsTo, + Table, + Column, + Default, + IsUUID, + Model, + DataType, + AfterCreate, + AllowNull, + AfterFind, +} from "sequelize-typescript"; +import mainWindow from "@main/window"; +import log from "@main/logger"; +import { Client } from "@/api"; +import { WEB_API_URL } from "@/constants"; +import settings from "@main/settings"; +import { Segment } from "@main/db/models"; + +const logger = log.scope("db/models/note"); +@Table({ + modelName: "Note", + tableName: "notes", + underscored: true, + timestamps: true, +}) +export class Note extends Model { + @IsUUID("all") + @Default(DataType.UUIDV4) + @Column({ primaryKey: true, type: DataType.UUID }) + id: string; + + @Column(DataType.UUID) + targetId: string; + + @Column(DataType.STRING) + targetType: string; + + @AllowNull(false) + @Column(DataType.TEXT) + content: string; + + @Default({}) + @Column(DataType.JSON) + parameters: any; + + @Column(DataType.DATE) + syncedAt: Date; + + @BelongsTo(() => Segment, { foreignKey: "targetId", constraints: false }) + segment: Segment; + + @Column(DataType.VIRTUAL) + get isSynced(): boolean { + return Boolean(this.syncedAt) && this.syncedAt >= this.updatedAt; + } + + async sync(): Promise { + if (this.isSynced) return; + + const webApi = new Client({ + baseUrl: process.env.WEB_API_URL || WEB_API_URL, + accessToken: settings.getSync("user.accessToken") as string, + logger, + }); + + // Sync the segment if the note is related to a segment + if (this.targetType === "Segment") { + const segment = await Segment.findByPk(this.targetId); + if (!segment) { + throw new Error("Segment not found"); + } + + await segment.sync(); + } + + return webApi.syncNote(this.toJSON()).then(() => { + const now = new Date(); + this.update({ syncedAt: now, updatedAt: now }); + }); + } + + @AfterFind + static async syncAfterFind(notes: Note[]) { + if (!notes.length) return; + + const unsyncedNotes = notes.filter((note) => note.id && !note.isSynced); + if (!unsyncedNotes.length) return; + + unsyncedNotes.forEach((note) => { + note.sync().catch((err) => { + logger.error("sync note error", note.id, err); + }); + }); + } + + @AfterCreate + static syncAndUploadAfterCreate(note: Note) { + note.sync(); + } + + @AfterCreate + static notifyForCreate(note: Note) { + this.notify(note, "create"); + } + + @AfterUpdate + static notifyForUpdate(note: Note) { + this.notify(note, "update"); + } + + @AfterUpdate + static syncAfterUpdate(note: Note) { + note.sync().catch((err) => { + logger.error("sync error", err); + }); + } + + @AfterDestroy + static destroyRemote(note: Note) { + const webApi = new Client({ + baseUrl: process.env.WEB_API_URL || WEB_API_URL, + accessToken: settings.getSync("user.accessToken") as string, + logger, + }); + + webApi.deleteNote(note.id).catch((err) => { + logger.error("delete remote note failed:", err.message); + }); + } + + @AfterDestroy + static notifyForDestroy(note: Note) { + this.notify(note, "destroy"); + } + + static async notify(note: Note, action: "create" | "update" | "destroy") { + if (!mainWindow.win) return; + + const segment = await Segment.findOne({ where: { id: note.targetId } }); + const record = note.toJSON(); + + if (segment) { + record.segment = segment.toJSON(); + } + mainWindow.win.webContents.send("db-on-transaction", { + model: "Note", + id: note.id, + action, + record, + }); + } +} diff --git a/enjoy/src/main/db/models/segment.ts b/enjoy/src/main/db/models/segment.ts new file mode 100644 index 00000000..f1478175 --- /dev/null +++ b/enjoy/src/main/db/models/segment.ts @@ -0,0 +1,246 @@ +import { + AfterUpdate, + AfterDestroy, + BelongsTo, + Table, + Column, + Default, + IsUUID, + Model, + DataType, + Unique, + AfterCreate, + AllowNull, + AfterFind, +} from "sequelize-typescript"; +import { Audio, Transcription, Video } from "@main/db/models"; +import mainWindow from "@main/window"; +import log from "@main/logger"; +import { Client } from "@/api"; +import { WEB_API_URL } from "@/constants"; +import settings from "@main/settings"; +import storage from "@/main/storage"; +import path from "path"; +import { TimelineEntry } from "echogarden/dist/utilities/Timeline.d.js"; +import FfmpegWrapper from "@/main/ffmpeg"; +import { hashFile } from "@/main/utils"; +import fs from "fs-extra"; +import { v5 as uuidv5 } from "uuid"; + +const logger = log.scope("db/models/segment"); +const OUTPUT_FORMAT = "mp3"; +@Table({ + modelName: "Segment", + tableName: "segments", + underscored: true, + timestamps: true, +}) +export class Segment extends Model { + @IsUUID("all") + @Default(DataType.UUIDV4) + @Column({ primaryKey: true, type: DataType.UUID }) + id: string; + + @Column(DataType.UUID) + targetId: string; + + @Column(DataType.STRING) + targetType: string; + + @AllowNull(false) + @Column(DataType.INTEGER) + segmentIndex: number; + + @Unique + @Column(DataType.STRING) + md5: string; + + @Column(DataType.JSON) + caption: TimelineEntry; + + @Column(DataType.NUMBER) + startTime: number; + + @Column(DataType.NUMBER) + endTime: number; + + @Column(DataType.DATE) + syncedAt: Date; + + @Column(DataType.DATE) + uploadedAt: Date; + + @BelongsTo(() => Audio, { foreignKey: "targetId", constraints: false }) + audio: Audio; + + @BelongsTo(() => Video, { foreignKey: "targetId", constraints: false }) + video: Video; + + @Column(DataType.VIRTUAL) + get isSynced(): boolean { + return Boolean(this.syncedAt) && this.syncedAt >= this.updatedAt; + } + + @Column(DataType.VIRTUAL) + get isUploaded(): boolean { + return Boolean(this.uploadedAt) && this.uploadedAt >= this.updatedAt; + } + + @Column(DataType.VIRTUAL) + get src(): string { + return `enjoy://${path.posix.join( + "library", + "segments", + this.getDataValue("md5") + "." + OUTPUT_FORMAT + )}`; + } + + get filePath(): string { + return path.join( + settings.userDataPath(), + "segments", + this.getDataValue("md5") + "." + OUTPUT_FORMAT + ); + } + + async sync() { + if (this.isSynced) return; + + const webApi = new Client({ + baseUrl: process.env.WEB_API_URL || WEB_API_URL, + accessToken: settings.getSync("user.accessToken") as string, + logger, + }); + return webApi.syncSegment(this.toJSON()).then(() => { + const now = new Date(); + this.update({ syncedAt: now, updatedAt: now }); + }); + } + + async upload() { + if (this.isUploaded) return; + + return storage + .put(this.md5, this.filePath) + .then((result) => { + logger.debug("upload result:", result.data); + if (result.data.success) { + this.update({ uploadedAt: new Date() }); + } else { + throw new Error(result.data); + } + }) + .catch((err) => { + logger.error("upload failed:", err.message); + throw err; + }); + } + + static async generate(params: { + targetId: string; + targetType: string; + segmentIndex: number; + }) { + let target: Video | Audio; + if (params.targetType === "Video") { + target = await Video.findByPk(params.targetId); + } else if (params.targetType === "Audio") { + target = await Audio.findByPk(params.targetId); + } else { + throw new Error("Invalid targetType"); + } + + const { targetId, targetType, segmentIndex } = params; + const transcription = await Transcription.findOne({ + where: { targetId, targetType }, + }); + + if (!transcription) { + throw new Error("Transcription not found"); + } + + const caption = transcription.result.timeline[segmentIndex]; + if (!caption) { + throw new Error("Caption not found"); + } + + const ffmpeg = new FfmpegWrapper(); + const output = path.join( + settings.cachePath(), + `${target.md5}-${segmentIndex}.${OUTPUT_FORMAT}` + ); + await ffmpeg.crop(target.filePath, { + startTime: caption.startTime, + endTime: caption.endTime, + output, + }); + + const md5 = await hashFile(output, { algo: "md5" }); + const userId = settings.getSync("user.id"); + const id = uuidv5(`${userId}/${md5}`, uuidv5.URL); + const dir = path.join(settings.userDataPath(), "segments"); + fs.ensureDirSync(dir); + fs.moveSync(output, path.join(dir, `${md5}.${OUTPUT_FORMAT}`), { + overwrite: true, + }); + + return Segment.create({ + id, + targetId, + targetType, + segmentIndex, + md5, + caption, + startTime: caption.startTime, + endTime: caption.endTime, + }); + } + + @AfterFind + static async syncAfterFind(segments: Segment[]) { + if (!segments.length) return; + + const unsyncedSegments = segments.filter((segment) => !segment.isSynced); + if (!unsyncedSegments.length) return; + + unsyncedSegments.forEach((segment) => { + segment.sync().catch((err) => { + logger.error("sync error", err); + }); + }); + } + + @AfterCreate + static syncAndUploadAfterCreate(segment: Segment) { + segment.sync(); + segment.upload(); + } + + @AfterUpdate + static notifyForUpdate(segment: Segment) { + this.notify(segment, "update"); + } + + @AfterUpdate + static syncAfterUpdate(segment: Segment) { + segment.sync().catch((err) => { + logger.error("sync error", err); + }); + } + + @AfterDestroy + static notifyForDestroy(segment: Segment) { + this.notify(segment, "destroy"); + } + + static notify(segment: Segment, action: "create" | "update" | "destroy") { + if (!mainWindow.win) return; + + mainWindow.win.webContents.send("db-on-transaction", { + model: "Segment", + id: segment.id, + action: action, + record: segment.toJSON(), + }); + } +} diff --git a/enjoy/src/main/db/models/transcription.ts b/enjoy/src/main/db/models/transcription.ts index d0fbc0c3..6f64e6ed 100644 --- a/enjoy/src/main/db/models/transcription.ts +++ b/enjoy/src/main/db/models/transcription.ts @@ -17,6 +17,7 @@ import log from "@main/logger"; import { Client } from "@/api"; import { WEB_API_URL, PROCESS_TIMEOUT } from "@/constants"; import settings from "@main/settings"; +import { AlignmentResult } from "echogarden/dist/api/Alignment"; const logger = log.scope("db/models/transcription"); @Table({ @@ -52,7 +53,7 @@ export class Transcription extends Model { model: string; @Column(DataType.JSON) - result: any; + result: Partial & { originalText?: string }; @Column(DataType.DATE) syncedAt: Date; diff --git a/enjoy/src/main/ffmpeg.ts b/enjoy/src/main/ffmpeg.ts index d3f6efd4..1d28a262 100644 --- a/enjoy/src/main/ffmpeg.ts +++ b/enjoy/src/main/ffmpeg.ts @@ -233,6 +233,44 @@ export default class FfmpegWrapper { }); } + // Crop video or audio from start to end time to a mp3 file + // Save the file to the output path + crop( + input: string, + options: { + startTime: number; + endTime: number; + output: string; + } + ) { + const { startTime, endTime, output } = options; + const ffmpeg = Ffmpeg(); + + return new Promise((resolve, reject) => { + ffmpeg + .input(input) + .outputOptions( + "-ss", + startTime.toString(), + "-to", + endTime.toString() + ) + .on("start", (commandLine) => { + logger.info("Spawned FFmpeg with command: " + commandLine); + fs.ensureDirSync(path.dirname(output)); + }) + .on("end", () => { + logger.info(`File ${output} created`); + resolve(output); + }) + .on("error", (err) => { + logger.error(err); + reject(err); + }) + .save(output); + }); + } + registerIpcHandlers() { ipcMain.handle("ffmpeg-check-command", async (_event) => { return await this.checkCommand(); diff --git a/enjoy/src/main/utils.ts b/enjoy/src/main/utils.ts index 5e472d6d..2daa863d 100644 --- a/enjoy/src/main/utils.ts +++ b/enjoy/src/main/utils.ts @@ -49,7 +49,7 @@ export function enjoyUrlToPath(enjoyUrl: string): string { let filePath = enjoyUrl; if ( - enjoyUrl.match(/enjoy:\/\/library\/(audios|videos|recordings|speeches)/g) + enjoyUrl.match(/enjoy:\/\/library\/(audios|videos|recordings|speeches|segments)/g) ) { filePath = path.posix.join( settings.userDataPath(), diff --git a/enjoy/src/preload.ts b/enjoy/src/preload.ts index 851c1319..bf11d2e0 100644 --- a/enjoy/src/preload.ts +++ b/enjoy/src/preload.ts @@ -473,4 +473,51 @@ contextBridge.exposeInMainWorld("__ENJOY_APP__", { return ipcRenderer.invoke("waveforms-save", id, data); }, }, + segments: { + findAll: (params: { + targetId?: string; + targetType?: string; + offset?: number; + limit?: number; + }) => { + return ipcRenderer.invoke("segments-find-all", params); + }, + find: (id: string) => { + return ipcRenderer.invoke("segments-find", id); + }, + create: (params: any) => { + return ipcRenderer.invoke("segments-create", params); + }, + sync: (id: string) => { + return ipcRenderer.invoke("segments-sync", id); + }, + }, + notes: { + groupByTarget: (params?: { limit?: number; offset?: number }) => { + return ipcRenderer.invoke("notes-group-by-target", params); + }, + groupBySegment: (targetId: string, targetType: string) => { + return ipcRenderer.invoke("notes-group-by-segment", targetId, targetType); + }, + findAll: (params: { + targetId?: string; + targetType?: string; + offset?: number; + limit?: number; + }) => { + return ipcRenderer.invoke("notes-find-all", params); + }, + update: (id: string, params: any) => { + return ipcRenderer.invoke("notes-update", id, params); + }, + delete: (id: string) => { + return ipcRenderer.invoke("notes-delete", id); + }, + create: (params: any) => { + return ipcRenderer.invoke("notes-create", params); + }, + sync: (id: string) => { + return ipcRenderer.invoke("notes-sync", id); + }, + }, }); diff --git a/enjoy/src/renderer/components/audios/audio-player.tsx b/enjoy/src/renderer/components/audios/audio-player.tsx index f4053ff6..eaa6d475 100644 --- a/enjoy/src/renderer/components/audios/audio-player.tsx +++ b/enjoy/src/renderer/components/audios/audio-player.tsx @@ -11,9 +11,15 @@ import { } from "@renderer/components"; import { useAudio } from "@renderer/hooks"; -export const AudioPlayer = (props: { id?: string; md5?: string }) => { - const { id, md5 } = props; - const { setMedia, layout } = useContext(MediaPlayerProviderContext); +export const AudioPlayer = (props: { + id?: string; + md5?: string; + segmentIndex?: number; +}) => { + const { id, md5, segmentIndex } = props; + const { setMedia, layout, setCurrentSegmentIndex } = useContext( + MediaPlayerProviderContext + ); const { audio } = useAudio({ id, md5 }); useEffect(() => { @@ -21,12 +27,17 @@ export const AudioPlayer = (props: { id?: string; md5?: string }) => { setMedia(audio); }, [audio]); + useEffect(() => { + if (!segmentIndex) return; + setCurrentSegmentIndex(segmentIndex); + }, []); + if (!layout) return ; return (
-
+
@@ -39,11 +50,11 @@ export const AudioPlayer = (props: { id?: string; md5?: string }) => {
-
+
-
+
diff --git a/enjoy/src/renderer/components/conversations/conversation-shortcuts.tsx b/enjoy/src/renderer/components/conversations/conversation-shortcuts.tsx index 2e0aed8e..10f498d9 100644 --- a/enjoy/src/renderer/components/conversations/conversation-shortcuts.tsx +++ b/enjoy/src/renderer/components/conversations/conversation-shortcuts.tsx @@ -154,7 +154,7 @@ export const ConversationShortcuts = (props: {
ask(conversation)} - className="bg-background text-primary rounded-full w-full mb-2 py-2 px-4 hover:bg-primary hover:text-white cursor-pointer flex items-center border" + className="bg-background text-primary rounded-full w-full mb-2 py-2 px-4 hover:bg-muted hover:text-muted-foreground cursor-pointer flex items-center border" style={{ borderLeftColor: `#${conversation.id .replaceAll("-", "") diff --git a/enjoy/src/renderer/components/index.ts b/enjoy/src/renderer/components/index.ts index 0db55b97..4c69d9a1 100644 --- a/enjoy/src/renderer/components/index.ts +++ b/enjoy/src/renderer/components/index.ts @@ -2,15 +2,13 @@ export * from "./audios"; export * from "./conversations"; export * from "./meanings"; export * from "./messages"; +export * from "./medias"; +export * from "./notes"; +export * from "./posts"; export * from "./preferences"; export * from "./pronunciation-assessments"; export * from "./recordings"; export * from "./stories"; -export * from "./videos"; - -export * from "./medias"; - -export * from "./posts"; export * from "./users"; - +export * from "./videos"; export * from "./widgets"; diff --git a/enjoy/src/renderer/components/medias/index.ts b/enjoy/src/renderer/components/medias/index.ts index 28f0e2aa..ba93a3a5 100644 --- a/enjoy/src/renderer/components/medias/index.ts +++ b/enjoy/src/renderer/components/medias/index.ts @@ -1,6 +1,5 @@ export * from "./media-player-controls"; export * from "./media-caption"; -export * from "./media-caption-tabs"; export * from "./media-info-panel"; export * from "./media-recordings"; export * from "./media-current-recording"; diff --git a/enjoy/src/renderer/components/medias/media-caption.tsx b/enjoy/src/renderer/components/medias/media-caption.tsx index f0cb0a50..1c023aaf 100644 --- a/enjoy/src/renderer/components/medias/media-caption.tsx +++ b/enjoy/src/renderer/components/medias/media-caption.tsx @@ -2,19 +2,25 @@ import { useEffect, useState, useContext } from "react"; import { MediaPlayerProviderContext } from "@renderer/context"; import cloneDeep from "lodash/cloneDeep"; import { Button, toast } from "@renderer/components/ui"; -import { ConversationShortcuts, MediaCaptionTabs } from "@renderer/components"; +import { ConversationShortcuts } from "@renderer/components"; import { t } from "i18next"; -import { BotIcon, CopyIcon, CheckIcon, SpeechIcon } from "lucide-react"; +import { + BotIcon, + CopyIcon, + CheckIcon, + SpeechIcon, + NotebookPenIcon, +} from "lucide-react"; import { Timeline, TimelineEntry, } from "echogarden/dist/utilities/Timeline.d.js"; import { convertIpaToNormal } from "@/utils"; import { useCopyToClipboard } from "@uidotdev/usehooks"; +import { MediaCaptionTabs } from "./media-captions"; export const MediaCaption = () => { const { - wavesurfer, currentSegmentIndex, currentTime, transcription, @@ -30,6 +36,7 @@ export const MediaCaption = () => { const [multiSelecting, setMultiSelecting] = useState(false); const [displayIpa, setDisplayIpa] = useState(true); + const [displayNotes, setDisplayNotes] = useState(true); const [_, copyToClipboard] = useCopyToClipboard(); const [copied, setCopied] = useState(false); @@ -39,29 +46,71 @@ export const MediaCaption = () => { setMultiSelecting(event.shiftKey && event.type === "keydown"); }; - const toggleRegion = (index: number) => { + const toggleSeletedIndex = (index: number) => { if (!activeRegion) return; if (editingRegion) { toast.warning(t("currentRegionIsBeingEdited")); return; } - const word = caption.timeline[index]; - if (!word) return; + const startWord = caption.timeline[index]; + if (!startWord) return; - const start = word.startTime; - const end = word.endTime; + if (multiSelecting) { + const min = Math.min(index, ...selectedIndices); + const max = Math.max(index, ...selectedIndices); + + // Select all the words between the min and max indices. + setSelectedIndices( + Array.from({ length: max - min + 1 }, (_, i) => i + min) + ); + } else if (selectedIndices.includes(index)) { + setSelectedIndices([]); + } else { + setSelectedIndices([index]); + } + }; + + const toggleRegion = (params: number[]) => { + if (!activeRegion) return; + if (editingRegion) { + toast.warning(t("currentRegionIsBeingEdited")); + return; + } + if (params.length === 0) { + if (activeRegion.id.startsWith("word-region")) { + activeRegion.remove(); + setActiveRegion( + regions.getRegions().find((r) => r.id.startsWith("segment-region")) + ); + } + return; + } + + const startIndex = Math.min(...params); + const endIndex = Math.max(...params); + + const startWord = caption.timeline[startIndex]; + if (!startWord) return; + + const endWord = caption.timeline[endIndex] || startWord; + + const start = startWord.startTime; + const end = endWord.endTime; const regionStart = activeRegion.start; const regionEnd = activeRegion.end; + // If the active region is a word region, then merge the selected words into a single region. if (activeRegion.id.startsWith("word-region")) { + activeRegion.remove(); + if (start >= regionStart && end <= regionEnd) { setActiveRegion( regions.getRegions().find((r) => r.id.startsWith("segment-region")) ); - } else if (multiSelecting) { + } else { const region = regions.addRegion({ - id: `word-region-${index}`, + id: `word-region-${startIndex}`, start: Math.min(start, regionStart), end: Math.max(end, regionEnd), color: "#fb6f9233", @@ -69,27 +118,17 @@ export const MediaCaption = () => { resize: editingRegion, }); - setActiveRegion(region); - } else { - const region = regions.addRegion({ - id: `word-region-${index}`, - start, - end, - color: "#fb6f9233", - drag: false, - resize: editingRegion, - }); - setActiveRegion(region); } - activeRegion?.remove(); + // If the active region is a meaning group region, then active the segment region. } else if (activeRegion.id.startsWith("meaning-group-region")) { setActiveRegion( regions.getRegions().find((r) => r.id.startsWith("segment-region")) ); + // If the active region is a segment region, then create a new word region. } else { const region = regions.addRegion({ - id: `word-region-${index}`, + id: `word-region-${startIndex}`, start, end, color: "#fb6f9233", @@ -101,43 +140,6 @@ export const MediaCaption = () => { } }; - const markPhoneRegions = () => { - const phoneRegions = regions - .getRegions() - .filter((r) => r.id.startsWith("phone-region")); - if (phoneRegions.length > 0) { - phoneRegions.forEach((r) => { - r.remove(); - r.unAll(); - }); - return; - } - - if (!activeRegion) return; - if (!activeRegion.id.startsWith("word-region")) return; - if (!selectedIndices) return; - - selectedIndices.forEach((index) => { - const word = caption.timeline[index]; - - word.timeline.forEach((token) => { - token.timeline.forEach((phone) => { - const region = regions.addRegion({ - id: `phone-region-${index}`, - start: phone.startTime, - end: phone.endTime, - color: "#efefefef", - drag: false, - resize: editingRegion, - }); - region.on("click", () => { - region.play(); - }); - }); - }); - }); - }; - useEffect(() => { if (!caption) return; @@ -154,25 +156,8 @@ export const MediaCaption = () => { if (!caption?.timeline) return; if (!activeRegion) return; - if (activeRegion.id.startsWith("segment-region")) { - setSelectedIndices([]); - return; - } - - const indices: number[] = []; - caption.timeline.forEach((w, index) => { - if ( - w.startTime >= activeRegion.start && - (w.endTime <= activeRegion.end || - // The last word's end time may be a little greater than the duration of the audio in somehow. - w.endTime > wavesurfer.getDuration()) - ) { - indices.push(index); - } - }); - - setSelectedIndices(indices); - }, [caption, activeRegion]); + toggleRegion(selectedIndices); + }, [caption, selectedIndices]); useEffect(() => { if (!activeRegion) return; @@ -254,6 +239,10 @@ export const MediaCaption = () => { ); }, [currentSegmentIndex, transcription]); + useEffect(() => { + return () => setSelectedIndices([]); + }, [caption]); + useEffect(() => { document.addEventListener("keydown", (event: KeyboardEvent) => toggleMultiSelect(event) @@ -275,91 +264,19 @@ export const MediaCaption = () => {
-
- {/* use the words splitted by caption text if it is matched with the timeline length, otherwise use the timeline */} - {caption.text.split(" ").length !== caption.timeline.length - ? (caption.timeline || []).map((w, index) => ( -
toggleRegion(index)} - > -
-
- {w.text} -
- {displayIpa && ( -
- {w.timeline - .map((t) => - t.timeline - .map((s) => convertIpaToNormal(s.text)) - .join("") - ) - .join(" · ")} -
- )} -
-
- )) - : caption.text.split(" ").map((word, index) => ( -
toggleRegion(index)} - > -
-
- {word} -
- {displayIpa && ( -
- {caption.timeline[index].timeline - .map((t) => - t.timeline - .map((s) => convertIpaToNormal(s.text)) - .join("") - ) - .join(" · ")} -
- )} -
-
- ))} -
+
@@ -375,6 +292,17 @@ export const MediaCaption = () => { + + {
); }; + +const Caption = (props: { + caption: TimelineEntry; + selectedIndices: number[]; + currentSegmentIndex: number; + activeIndex: number; + displayIpa: boolean; + displayNotes: boolean; + onClick: (index: number) => void; +}) => { + const { + caption, + selectedIndices, + currentSegmentIndex, + activeIndex, + displayIpa, + displayNotes, + onClick, + } = props; + + const { currentNotes } = useContext(MediaPlayerProviderContext); + const notes = currentNotes.filter((note) => note.parameters?.quoteIndices); + const [notedquoteIndices, setNotedquoteIndices] = useState([]); + + let words = caption.text.split(" "); + const ipas = caption.timeline.map((w) => + w.timeline.map((t) => t.timeline.map((s) => s.text)) + ); + + if (words.length !== caption.timeline.length) { + words = caption.timeline.map((w) => w.text); + } + + return ( +
+ {/* use the words splitted by caption text if it is matched with the timeline length, otherwise use the timeline */} + {words.map((word, index) => ( +
+
onClick(index)} + > + {word} +
+ + {displayIpa && ( +
+ {ipas[index]} +
+ )} + + {displayNotes && + notes + .filter((note) => note.parameters.quoteIndices[0] === index) + .map((note) => ( +
+ setNotedquoteIndices(note.parameters.quoteIndices) + } + onMouseLeave={() => setNotedquoteIndices([])} + onClick={() => + document.getElementById("note-" + note.id)?.scrollIntoView() + } + > + {note.parameters.quoteIndices[0] === index && note.content} +
+ ))} +
+ ))} +
+ ); +}; diff --git a/enjoy/src/renderer/components/medias/media-captions/index.ts b/enjoy/src/renderer/components/medias/media-captions/index.ts new file mode 100644 index 00000000..2938ec5f --- /dev/null +++ b/enjoy/src/renderer/components/medias/media-captions/index.ts @@ -0,0 +1,4 @@ +export * from "./media-caption-tabs"; +export * from "./tab-content-analysis"; +export * from "./tab-content-note"; +export * from "./tab-content-translation"; \ No newline at end of file diff --git a/enjoy/src/renderer/components/medias/media-captions/media-caption-tabs.tsx b/enjoy/src/renderer/components/medias/media-captions/media-caption-tabs.tsx new file mode 100644 index 00000000..336fa861 --- /dev/null +++ b/enjoy/src/renderer/components/medias/media-captions/media-caption-tabs.tsx @@ -0,0 +1,67 @@ +import { useState } from "react"; +import { + Tabs, + TabsList, + TabsTrigger, + ScrollArea, +} from "@renderer/components/ui"; +import { t } from "i18next"; +import { TimelineEntry } from "echogarden/dist/utilities/Timeline.d.js"; +import { TabContentTranslation } from "./tab-content-translation"; +import { TabContentAnalysis } from "./tab-content-analysis"; +import { TabContentNote } from "./tab-content-note"; + +export const MediaCaptionTabs = (props: { + caption: TimelineEntry; + currentSegmentIndex: number; + selectedIndices: number[]; + setSelectedIndices: (indices: number[]) => void; + children?: React.ReactNode; +}) => { + const { + caption, + currentSegmentIndex, + selectedIndices, + setSelectedIndices, + children, + } = props; + + const [tab, setTab] = useState("note"); + + if (!caption) return null; + + return ( + + setTab(value)} className=""> + {children} + +
+ + + + + +
+ + + + {t("captionTabs.note")} + + + {t("captionTabs.translation")} + + + {t("captionTabs.analysis")} + + + + + ); +}; diff --git a/enjoy/src/renderer/components/medias/media-captions/tab-content-analysis.tsx b/enjoy/src/renderer/components/medias/media-captions/tab-content-analysis.tsx new file mode 100644 index 00000000..6342f346 --- /dev/null +++ b/enjoy/src/renderer/components/medias/media-captions/tab-content-analysis.tsx @@ -0,0 +1,133 @@ +import { useEffect, useState, useContext } from "react"; +import { AppSettingsProviderContext } from "@renderer/context"; +import { Button, toast, TabsContent } from "@renderer/components/ui"; +import { ConversationShortcuts } from "@renderer/components"; +import { t } from "i18next"; +import { BotIcon } from "lucide-react"; +import { useAiCommand } from "@renderer/hooks"; +import { LoaderIcon } from "lucide-react"; +import { md5 } from "js-md5"; +import Markdown from "react-markdown"; + +export function TabContentAnalysis(props: { text: string; }) { + const { text } = props; + const { EnjoyApp } = useContext(AppSettingsProviderContext); + const [analyzing, setAnalyzing] = useState(false); + const [analysisResult, setAnalysisResult] = useState(); + + const { analyzeText } = useAiCommand(); + + const analyzeSetence = async () => { + if (analyzing) return; + + setAnalyzing(true); + analyzeText(text, `analyze-${md5(text)}`) + .then((result) => { + if (result) { + setAnalysisResult(result); + } + }) + .catch((err) => toast.error(err.message)) + .finally(() => { + setAnalyzing(false); + }); + }; + + /* + * If the caption is changed, then reset the analysis. + * Also, check if the translation is cached, then use it. + */ + useEffect(() => { + EnjoyApp.cacheObjects.get(`analyze-${md5(text)}`).then((cached) => { + setAnalysisResult(cached); + }); + }, [text]); + + return ( + + {analysisResult ? ( + <> + {children}; + }, + }} + > + {analysisResult} + + +
+ + { + const result = replies.map((m) => m.content).join("\n"); + setAnalysisResult(result); + EnjoyApp.cacheObjects.set(`analyze-${md5(text)}`, result); + } } + tooltip={t("useAIAssistantToAnalyze")} /> +
+ + ) : ( +
+ + { + const result = replies.map((m) => m.content).join("\n"); + setAnalysisResult(result); + EnjoyApp.cacheObjects.set(`analyze-${md5(text)}`, result); + } } + tooltip={t("useAIAssistantToAnalyze")} /> +
+ )} + + ); +} + +const AIButton = (props: { + prompt: string; + onReply?: (replies: MessageType[]) => void; + tooltip: string; +}) => { + const { prompt, onReply, tooltip } = props; + return ( + + + + } + /> + ); +}; diff --git a/enjoy/src/renderer/components/medias/media-captions/tab-content-note.tsx b/enjoy/src/renderer/components/medias/media-captions/tab-content-note.tsx new file mode 100644 index 00000000..47438785 --- /dev/null +++ b/enjoy/src/renderer/components/medias/media-captions/tab-content-note.tsx @@ -0,0 +1,93 @@ +import { MediaPlayerProviderContext } from "@renderer/context"; +import { Button, TabsContent, toast } from "@renderer/components/ui"; +import { t } from "i18next"; +import { useContext, useState } from "react"; +import { NoteCard, NoteForm } from "@renderer/components"; + +/* + * Note tab content. + */ +export const TabContentNote = (props: { + currentSegmentIndex: number; + selectedIndices: number[]; + setSelectedIndices: (indices: number[]) => void; +}) => { + const { selectedIndices, setSelectedIndices } = props; + const { currentSegment, createSegment, currentNotes } = useContext( + MediaPlayerProviderContext + ); + const [editingNote, setEditingNote] = useState(); + + if (!currentSegment) { + return ( + +
+ +
+ + ); + } + + return ( + +
+
+ {!editingNote && ( +
+ + currentSegment?.caption?.timeline?.[index]?.text + ) + .join(" "), + }} + onParametersChange={(param) => { + if (param.quoteIndices) { + setSelectedIndices(param.quoteIndices); + } + }} + /> +
+ )} + +
+ {currentNotes.map((note) => ( +
+ {editingNote?.id === note.id ? ( + + currentSegment?.caption?.timeline?.[index]?.text + ) + .join(" "), + }} + onParametersChange={(param) => { + if (param.quoteIndices) { + setSelectedIndices(param.quoteIndices); + } + }} + note={note} + onCancel={() => setEditingNote(null)} + onSave={() => setEditingNote(null)} + /> + ) : ( + setEditingNote(note)} /> + )} +
+ ))} +
+
+
+ + ); +}; diff --git a/enjoy/src/renderer/components/medias/media-caption-tabs.tsx b/enjoy/src/renderer/components/medias/media-captions/tab-content-translation.tsx similarity index 58% rename from enjoy/src/renderer/components/medias/media-caption-tabs.tsx rename to enjoy/src/renderer/components/medias/media-captions/tab-content-translation.tsx index f4ea287d..aaf4c321 100644 --- a/enjoy/src/renderer/components/medias/media-caption-tabs.tsx +++ b/enjoy/src/renderer/components/medias/media-captions/tab-content-translation.tsx @@ -3,116 +3,108 @@ import { AppSettingsProviderContext, MediaPlayerProviderContext, } from "@renderer/context"; -import { - Button, - toast, - Tabs, - TabsList, - TabsTrigger, - TabsContent, - Separator, - ScrollArea, -} from "@renderer/components/ui"; -import { ConversationShortcuts } from "@renderer/components"; +import { Button, toast, TabsContent, Separator } from "@renderer/components/ui"; import { t } from "i18next"; -import { BotIcon } from "lucide-react"; -import { TimelineEntry } from "echogarden/dist/utilities/Timeline.d.js"; import { useAiCommand, useCamdict } from "@renderer/hooks"; import { LoaderIcon, Volume2Icon } from "lucide-react"; -import { convertIpaToNormal } from "@/utils"; import { md5 } from "js-md5"; import Markdown from "react-markdown"; +import { TimelineEntry } from "echogarden/dist/utilities/Timeline"; +import { convertIpaToNormal } from "@/utils"; /* - * Tabs below the caption text. - * It provides the translation, analysis, and note features. + * Translation tab content. */ -export const MediaCaptionTabs = (props: { +export function TabContentTranslation(props: { caption: TimelineEntry; selectedIndices: number[]; - toggleRegion: (index: number) => void; - children?: React.ReactNode; -}) => { - const { caption, selectedIndices, toggleRegion, children } = props; +}) { + const { caption } = props; + const { EnjoyApp } = useContext(AppSettingsProviderContext); + const [translation, setTranslation] = useState(); + const [translating, setTranslating] = useState(false); + const { translate } = useAiCommand(); - const [tab, setTab] = useState("selected"); + const translateSetence = async () => { + if (translating) return; - if (!caption) return null; + setTranslating(true); + translate(caption.text, `translate-${md5(caption.text)}`) + .then((result) => { + if (result) { + setTranslation(result); + } + }) + .catch((err) => toast.error(err.message)) + .finally(() => { + setTranslating(false); + }); + }; + + /* + * If the caption is changed, then reset the translation. + * Also, check if the translation is cached, then use it. + */ + useEffect(() => { + EnjoyApp.cacheObjects + .get(`translate-${md5(caption.text)}`) + .then((cached) => { + setTranslation(cached); + }); + }, [caption.text]); return ( - - setTab(value)} className=""> - {children} + + -
- + - + {translation ? ( +
+
+ {t("translateSetence")} +
+ + {translation} + - - - -
- Comming soon -
- +
+ +
- - - - {t("captionTabs.selected")} - - - {t("captionTabs.translation")} - - - {t("captionTabs.analysis")} - - - {t("captionTabs.note")} - - - - + ) : ( +
+ +
+ )} + ); -}; +} -const AIButton = (props: { - prompt: string; - onReply?: (replies: MessageType[]) => void; - tooltip: string; -}) => { - const { prompt, onReply, tooltip } = props; - return ( - - - - } - /> - ); -}; - -const SelectedTabContent = (props: { +const SelectedWords = (props: { caption: TimelineEntry; selectedIndices: number[]; - toggleRegion: (index: number) => void; }) => { - const { selectedIndices, caption, toggleRegion } = props; + const { selectedIndices, caption } = props; const { transcription } = useContext(MediaPlayerProviderContext); const { webApi } = useContext(AppSettingsProviderContext); @@ -187,15 +179,13 @@ const SelectedTabContent = (props: { if (selectedIndices.length === 0) return ( - -
- {t("clickAnyWordToSelect")} -
- +
+ {t("clickAnyWordToSelect")} +
); return ( - + <>
{selectedIndices.map((index, i) => { const word = caption.timeline[index]; @@ -330,191 +320,6 @@ const SelectedTabContent = (props: {
)} - -
- -
- - ); -}; - -/* - * Translation tab content. - */ -const TranslationTabContent = (props: { text: string }) => { - const { text } = props; - const { EnjoyApp } = useContext(AppSettingsProviderContext); - const [translation, setTranslation] = useState(); - const [translating, setTranslating] = useState(false); - const { translate } = useAiCommand(); - - const translateSetence = async () => { - if (translating) return; - - setTranslating(true); - translate(text, `translate-${md5(text)}`) - .then((result) => { - if (result) { - setTranslation(result); - } - }) - .catch((err) => toast.error(err.message)) - .finally(() => { - setTranslating(false); - }); - }; - - /* - * If the caption is changed, then reset the translation. - * Also, check if the translation is cached, then use it. - */ - useEffect(() => { - EnjoyApp.cacheObjects.get(`translate-${md5(text)}`).then((cached) => { - setTranslation(cached); - }); - }, [text]); - - return ( - - {translation ? ( - <> - - {translation} - - -
- -
- - ) : ( -
- -
- )} - - ); -}; - -const AnalysisTabContent = (props: { text: string }) => { - const { text } = props; - const { EnjoyApp } = useContext(AppSettingsProviderContext); - const [analyzing, setAnalyzing] = useState(false); - const [analysisResult, setAnalysisResult] = useState(); - - const { analyzeText } = useAiCommand(); - - const analyzeSetence = async () => { - if (analyzing) return; - - setAnalyzing(true); - analyzeText(text, `analyze-${md5(text)}`) - .then((result) => { - if (result) { - setAnalysisResult(result); - } - }) - .catch((err) => toast.error(err.message)) - .finally(() => { - setAnalyzing(false); - }); - }; - - /* - * If the caption is changed, then reset the analysis. - * Also, check if the translation is cached, then use it. - */ - useEffect(() => { - EnjoyApp.cacheObjects.get(`analyze-${md5(text)}`).then((cached) => { - setAnalysisResult(cached); - }); - }, [text]); - - return ( - - {analysisResult ? ( - <> - {children}; - }, - }} - > - {analysisResult} - - -
- - { - const result = replies.map((m) => m.content).join("\n"); - setAnalysisResult(result); - EnjoyApp.cacheObjects.set(`analyze-${md5(text)}`, result); - }} - tooltip={t("useAIAssistantToAnalyze")} - /> -
- - ) : ( -
- - { - const result = replies.map((m) => m.content).join("\n"); - setAnalysisResult(result); - EnjoyApp.cacheObjects.set(`analyze-${md5(text)}`, result); - }} - tooltip={t("useAIAssistantToAnalyze")} - /> -
- )} - + ); }; diff --git a/enjoy/src/renderer/components/medias/media-player-controls.tsx b/enjoy/src/renderer/components/medias/media-player-controls.tsx index c0e39901..e0e7d90f 100644 --- a/enjoy/src/renderer/components/medias/media-player-controls.tsx +++ b/enjoy/src/renderer/components/medias/media-player-controls.tsx @@ -57,7 +57,9 @@ export const MediaPlayerControls = () => { setTranscriptionDraft, } = useContext(MediaPlayerProviderContext); const { EnjoyApp } = useContext(AppSettingsProviderContext); - const { currentHotkeys, enabled } = useContext(HotKeysSettingsProviderContext) + const { currentHotkeys, enabled } = useContext( + HotKeysSettingsProviderContext + ); const [playMode, setPlayMode] = useState<"loop" | "single" | "all">("single"); const [playbackRate, setPlaybackRate] = useState(1); const [grouping, setGrouping] = useState(false); @@ -325,8 +327,7 @@ export const MediaPlayerControls = () => { if (!decoded) return; if (!wavesurfer) return; - setCurrentSegmentIndex(0); - const segment = transcription.result.timeline[0]; + const segment = transcription.result.timeline[currentSegmentIndex]; wavesurfer.seekTo( Math.floor((segment.startTime / wavesurfer.getDuration()) * 1e8) / 1e8 ); @@ -374,7 +375,13 @@ export const MediaPlayerControls = () => { }, [wavesurfer, decoded, playMode, activeRegion, currentTime]); useHotkeys( - [currentHotkeys.PlayOrPause, currentHotkeys.PlayPreviousSegment, currentHotkeys.PlayNextSegment, currentHotkeys.StartOrStopRecording, currentHotkeys.Compare], + [ + currentHotkeys.PlayOrPause, + currentHotkeys.PlayPreviousSegment, + currentHotkeys.PlayNextSegment, + currentHotkeys.StartOrStopRecording, + currentHotkeys.Compare, + ], (keyboardEvent, hotkeyEvent) => { if (!wavesurfer) return; keyboardEvent.preventDefault(); @@ -396,8 +403,9 @@ export const MediaPlayerControls = () => { document.getElementById("media-compare-button").click(); break; } - },{ - enabled + }, + { + enabled, }, [wavesurfer, currentHotkeys] ); diff --git a/enjoy/src/renderer/components/medias/media-transcription.tsx b/enjoy/src/renderer/components/medias/media-transcription.tsx index 81618bb8..c3f1cf0d 100644 --- a/enjoy/src/renderer/components/medias/media-transcription.tsx +++ b/enjoy/src/renderer/components/medias/media-transcription.tsx @@ -18,7 +18,12 @@ import { AlertDialogAction, PingPoint, } from "@renderer/components/ui"; -import { LoaderIcon, CheckCircleIcon, MicIcon } from "lucide-react"; +import { + LoaderIcon, + CheckCircleIcon, + MicIcon, + PencilLineIcon, +} from "lucide-react"; import { AlignmentResult } from "echogarden/dist/api/API.d.js"; import { formatDuration } from "@renderer/lib/utils"; @@ -40,6 +45,15 @@ export const MediaTranscription = () => { const [recordingStats, setRecordingStats] = useState([]); + const [notesStats, setNotesStats] = useState< + { + targetId: string; + targetType: string; + count: number; + segment: SegmentType; + }[] + >([]); + const fetchSegmentStats = async () => { if (!media) return; @@ -48,6 +62,10 @@ export const MediaTranscription = () => { .then((stats) => { setRecordingStats(stats); }); + + EnjoyApp.notes.groupBySegment(media.id, media.mediaType).then((stats) => { + setNotesStats(stats); + }); }; useEffect(() => { @@ -134,8 +152,9 @@ export const MediaTranscription = () => {
{ const duration = wavesurfer.getDuration(); wavesurfer.seekTo( @@ -151,6 +170,9 @@ export const MediaTranscription = () => { {(recordingStats || []).findIndex( (s) => s.referenceId === index ) !== -1 && } + {(notesStats || []).findIndex( + (s) => s.segment?.segmentIndex === index + ) !== -1 && } {formatDuration(sentence.startTime, "s")} diff --git a/enjoy/src/renderer/components/notes/index.ts b/enjoy/src/renderer/components/notes/index.ts new file mode 100644 index 00000000..56b904e2 --- /dev/null +++ b/enjoy/src/renderer/components/notes/index.ts @@ -0,0 +1,4 @@ +export * from './note-card'; +export * from './note-form'; +export * from './note-segment'; +export * from './note-segment-group'; \ No newline at end of file diff --git a/enjoy/src/renderer/components/notes/note-card.tsx b/enjoy/src/renderer/components/notes/note-card.tsx new file mode 100644 index 00000000..6b44db48 --- /dev/null +++ b/enjoy/src/renderer/components/notes/note-card.tsx @@ -0,0 +1,165 @@ +import { AppSettingsProviderContext } from "@renderer/context"; +import { useContext, useState } from "react"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogContent, + AlertDialogCancel, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, + Button, + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, + toast, +} from "@renderer/components/ui"; +import { MoreHorizontalIcon } from "lucide-react"; +import Markdown from "react-markdown"; +import { t } from "i18next"; + +export const NoteCard = (props: { + note: NoteType; + onEdit?: (note: NoteType) => void; +}) => { + if (props.note.targetType === "Segment") { + return ; + } +}; + +export const SegmentNoteCard = (props: { + note: NoteType; + onEdit?: (note: NoteType) => void; +}) => { + const { note } = props; + + return ( +
+ + {note.content} + + +
+ {note.parameters?.quote ? ( +
+ + {note.parameters.quote} + +
+ ) : ( +
+ )} + + +
+
+ ); +}; + +const NoteActionsDropdownMenu = (props: { + note: NoteType; + onEdit?: (note: NoteType) => void; +}) => { + const { EnjoyApp, webApi } = useContext(AppSettingsProviderContext); + const { note, onEdit } = props; + const [deleting, setDeleting] = useState(false); + const [sharing, setSharing] = useState(false); + + const handleDelete = () => { + EnjoyApp.notes.delete(note.id); + }; + + const handleShare = async () => { + try { + if ( + note.segment && + (!note.segment.syncedAt || !note.segment.uploadedAt) + ) { + await EnjoyApp.segments.sync(note.segment.id); + } + if (!note.syncedAt) { + await EnjoyApp.notes.sync(note.id); + } + } catch (e) { + toast.error(t("shareFailed"), { description: e.message }); + } + + webApi + .createPost({ + targetId: note.id, + targetType: "Note", + }) + .then(() => { + toast.success(t("sharedSuccessfully")); + }) + .catch((e) => { + toast.error(t("shareFailed"), { description: e.message }); + }); + }; + + return ( + <> + + + + + + {onEdit && ( + onEdit(note)}> + {t("edit")} + + )} + + setSharing(true)}> + {t("share")} + + + setDeleting(true)}> + {t("delete")} + + + + + setSharing(value)}> + + + {t("shareNote")} + + {t("areYouSureToShareThisNoteToCommunity")} + + + + {t("cancel")} + + + + + + + + setDeleting(value)}> + + + {t("deleteNote")} + + {t("areYouSureToDeleteThisNote")} + + + + {t("cancel")} + + + + + + + + ); +}; diff --git a/enjoy/src/renderer/components/notes/note-form.tsx b/enjoy/src/renderer/components/notes/note-form.tsx new file mode 100644 index 00000000..fb9a0928 --- /dev/null +++ b/enjoy/src/renderer/components/notes/note-form.tsx @@ -0,0 +1,110 @@ +import { AppSettingsProviderContext } from "@renderer/context"; +import { useContext, useEffect, useRef, useState } from "react"; +import { Button, Textarea, toast } from "@renderer/components/ui"; +import { t } from "i18next"; + +export const NoteForm = (props: { + segment: SegmentType; + note?: NoteType; + parameters: { quoteIndices: number[]; quote: string }; + onParametersChange?: (parameters: any) => void; + onCancel?: () => void; + onSave?: (note: NoteType) => void; +}) => { + const { segment, note, parameters, onParametersChange, onCancel, onSave } = + props; + const [content, setContent] = useState(note?.content ?? ""); + const { EnjoyApp } = useContext(AppSettingsProviderContext); + + const inputRef = useRef(null); + + const resizeTextarea = () => { + if (!inputRef.current) return; + + inputRef.current.style.height = "auto"; + inputRef.current.style.height = `${inputRef.current.scrollHeight}px`; + }; + + const handleSubmit = () => { + if (!content) return; + + if (note) { + EnjoyApp.notes + .update(note.id, { + content, + parameters, + }) + .then((note) => { + onSave && onSave(note); + }) + .catch((err) => { + toast.error(err.message); + }); + } else { + EnjoyApp.notes + .create({ + targetId: segment.id, + targetType: "Segment", + parameters, + content, + }) + .then((note) => { + onSave && onSave(note); + setContent(""); + }) + .catch((err) => { + toast.error(err.message); + }); + } + }; + + useEffect(() => { + resizeTextarea(); + }, [content]); + + useEffect(() => { + if (!note) return; + if (note.parameters === parameters) return; + + onParametersChange && onParametersChange(note.parameters); + }, [note]); + + return ( +
+
+