diff --git a/enjoy/package.json b/enjoy/package.json index 7e72c7b9..80114b97 100644 --- a/enjoy/package.json +++ b/enjoy/package.json @@ -51,6 +51,7 @@ "@types/intl-tel-input": "^18.1.4", "@types/lodash": "^4.17.12", "@types/mark.js": "^8.11.12", + "@types/mime-types": "^2", "@types/mustache": "^4.2.5", "@types/node": "^22.7.7", "@types/prop-types": "^15.7.13", @@ -120,6 +121,7 @@ "@radix-ui/react-toggle": "^1.1.0", "@radix-ui/react-tooltip": "^1.1.3", "@rails/actioncable": "7.2.101", + "@types/turndown": "^5.0.5", "@uidotdev/usehooks": "^2.4.1", "@vidstack/react": "^1.12.11", "ahoy.js": "^0.4.4", @@ -145,7 +147,9 @@ "electron-settings": "^4.0.4", "electron-squirrel-startup": "^1.0.1", "ffmpeg-static": "^5.2.0", + "file-type": "^19.6.0", "fluent-ffmpeg": "^2.1.3", + "foliate-js": "https://github.com/johnfactotum/foliate-js", "fs-extra": "^11.2.0", "html-to-text": "^9.0.5", "https-proxy-agent": "^7.0.5", @@ -158,6 +162,7 @@ "lucide-react": "^0.453.0", "mark.js": "^8.11.1", "microsoft-cognitiveservices-speech-sdk": "^1.41.0", + "mime-types": "^2.1.35", "mustache": "^4.2.0", "next-themes": "^0.3.0", "openai": "^4.68.1", @@ -179,6 +184,7 @@ "react-shadow-root": "^6.2.0", "react-tooltip": "^5.28.0", "reflect-metadata": "^0.2.2", + "remark-gfm": "^4.0.0", "rimraf": "^6.0.1", "semver": "^7.6.3", "sequelize": "^6.37.4", @@ -186,6 +192,7 @@ "sonner": "^1.5.0", "sqlite3": "^5.1.7", "tailwind-scrollbar-hide": "^1.1.7", + "turndown": "^7.2.0", "umzug": "^3.8.2", "unzipper": "^0.12.3", "update-electron-app": "^3.0.0", diff --git a/enjoy/src/api/client.ts b/enjoy/src/api/client.ts index 55703440..a2751ab6 100644 --- a/enjoy/src/api/client.ts +++ b/enjoy/src/api/client.ts @@ -69,8 +69,8 @@ export class Client { this.logger.error( err.response.status, err.response.config.method.toUpperCase(), - err.response.config.baseURL + err.response.config.url, - err.response.data + err.response.config.baseURL + err.response.config.url + // err.response.data ); if (err.response.data) { @@ -86,7 +86,6 @@ export class Client { return Promise.reject(err); } - this.logger.error(err.message); return Promise.reject(err); } ); @@ -616,4 +615,37 @@ export class Client { params: decamelizeKeys(params), }); } + + syncDocument(document: Partial) { + return this.api.post("/api/mine/documents", decamelizeKeys(document)); + } + + deleteDocument(id: string) { + return this.api.delete(`/api/mine/documents/${id}`); + } + + translations(params?: { + md5?: string; + translatedLanguage?: string; + engine?: string; + }): Promise< + { + translations: TranslationType[]; + } & PagyResponseType + > { + return this.api.get("/api/translations", { + params: decamelizeKeys(params), + }); + } + + createTranslation(params: { + md5: string; + content: string; + translatedContent: string; + language: string; + translatedLanguage: string; + engine: string; + }): Promise { + return this.api.post("/api/translations", decamelizeKeys(params)); + } } diff --git a/enjoy/src/constants/index.ts b/enjoy/src/constants/index.ts index e8334250..96d78b1a 100644 --- a/enjoy/src/constants/index.ts +++ b/enjoy/src/constants/index.ts @@ -61,6 +61,8 @@ export const AudioFormats = ["mp3", "wav", "ogg", "flac", "m4a", "wma", "aac"]; export const VideoFormats = ["mp4", "mkv", "avi", "mov", "wmv", "flv", "webm"]; +export const DocumentFormats = ["epub", "md", "markdown", "html", "txt"]; + export const PROCESS_TIMEOUT = 1000 * 60 * 15; export const NOT_SUPPORT_JSON_FORMAT_MODELS = [ diff --git a/enjoy/src/i18n/en.json b/enjoy/src/i18n/en.json index 8628260d..619ef545 100644 --- a/enjoy/src/i18n/en.json +++ b/enjoy/src/i18n/en.json @@ -189,6 +189,11 @@ "onlyGPTAgentCanBeAddedToThisChat": "Only GPT agent can be added to this chat", "invalidAgentType": "Invalid agent type", "invalidMembers": "Invalid members" + }, + "document": { + "fileNotFound": "File not found {{file}}", + "fileNotSupported": "File not supported {{file}}", + "failedToCopyFile": "Failed to copy file {{file}}" } }, "sidebar": { @@ -199,6 +204,7 @@ "audios": "Audios", "videos": "Videos", "stories": "Stories", + "documents": "Documents", "books": "Books", "vocabulary": "Vocabulary", "library": "Library", @@ -529,6 +535,8 @@ "addedStories": "added stories", "addedAudios": "added audios", "addedVideos": "added videos", + "addedDocuments": "added documents", + "document": "document", "frontSide": "front side", "backSide": "back side", "aiExtractVocabulary": "AI extract vocabulary", @@ -891,5 +899,13 @@ "diskUsageDescription": "The disk usage of Enjoy App.", "releaseDiskSpace": "Release", "bulkDeleteRecordings": "Delete Recordings", - "bulkDeleteAborted": "Bulk delete aborted" + "bulkDeleteAborted": "Bulk delete aborted", + "notFound": "Not found", + "saved": "Saved", + "autoTranslate": "Auto translate", + "autoNextSpeech": "Auto speech", + "failedToLoadLink": "Failed to load link", + "refreshSpeech": "Refresh speech", + "locateParagraph": "Locate paragraph", + "close": "Close" } diff --git a/enjoy/src/i18n/zh-CN.json b/enjoy/src/i18n/zh-CN.json index f9fe7574..d029de96 100644 --- a/enjoy/src/i18n/zh-CN.json +++ b/enjoy/src/i18n/zh-CN.json @@ -189,6 +189,11 @@ "onlyGPTAgentCanBeAddedToThisChat": "只有 GPT 智能体可以添加到此聊天", "invalidAgentType": "无效的智能体类型", "invalidMembers": "无效的成员" + }, + "document": { + "fileNotFound": "文件未找到 {{file}}", + "fileNotSupported": "文件格式不支持 {{file}}", + "failedToCopyFile": "复制文件失败 {{file}}" } }, "sidebar": { @@ -199,6 +204,7 @@ "audios": "音频", "videos": "视频", "stories": "文章", + "documents": "文档", "books": "电子书", "vocabulary": "生词本", "library": "资料库", @@ -529,6 +535,8 @@ "addedStories": "添加的文章", "addedAudios": "添加的音频", "addedVideos": "添加的视频", + "addedDocuments": "添加的文档", + "document": "文档", "frontSide": "正面", "backSide": "反面", "aiExtractVocabulary": "AI 提取生词", @@ -891,5 +899,13 @@ "diskUsageDescription": "Enjoy App 的磁盘使用情况", "releaseDiskSpace": "释放磁盘", "bulkDeleteRecordings": "删除录音", - "bulkDeleteAborted": "批量删除已中止" + "bulkDeleteAborted": "批量删除已中止", + "notFound": "未找到", + "saved": "已保存", + "autoTranslate": "自动翻译", + "autoNextSpeech": "连续朗读", + "failedToLoadLink": "加载链接失败", + "refreshSpeech": "刷新语音", + "locateParagraph": "定位段落", + "close": "关闭" } diff --git a/enjoy/src/main.ts b/enjoy/src/main.ts index 20888a48..19aa372e 100644 --- a/enjoy/src/main.ts +++ b/enjoy/src/main.ts @@ -117,7 +117,11 @@ app.on("ready", async () => { protocol.handle("enjoy", (request) => { let url = request.url.replace("enjoy://", ""); - if (url.match(/library\/(audios|videos|recordings|speeches|segments)/g)) { + if ( + url.match( + /library\/(audios|videos|recordings|speeches|segments|documents)/g + ) + ) { url = url.replace("library/", ""); url = path.join(settings.userDataPath(), url); } else if (url.startsWith("library")) { diff --git a/enjoy/src/main/db/handlers/documents-handler.ts b/enjoy/src/main/db/handlers/documents-handler.ts new file mode 100644 index 00000000..3e8b4d9a --- /dev/null +++ b/enjoy/src/main/db/handlers/documents-handler.ts @@ -0,0 +1,158 @@ +import { ipcMain, IpcMainEvent } from "electron"; +import { Document } from "@main/db/models"; +import { FindOptions, WhereOptions, Attributes, Op } from "sequelize"; +import downloader from "@main/downloader"; +import log from "@main/logger"; +import { t } from "i18next"; + +const logger = log.scope("db/handlers/documents-handler"); + +class DocumentsHandler { + private async findAll( + _event: IpcMainEvent, + options: FindOptions> & { query?: string } + ) { + const { query, where = {} } = options || {}; + delete options.query; + delete options.where; + + if (query) { + (where as any).title = { + [Op.like]: `%${query}%`, + }; + } + const documents = await Document.findAll({ + order: [ + ["lastReadAt", "DESC"], + ["updatedAt", "DESC"], + ], + where, + ...options, + }); + + if (!documents) { + return []; + } + return documents.map((document) => document.toJSON()); + } + + private async findOne( + _event: IpcMainEvent, + where: WhereOptions> + ) { + const document = await Document.findOne({ + where: { + ...where, + }, + }); + if (!document) return; + + if (!document.isSynced) { + document.sync().catch(() => {}); + } + + return document.toJSON(); + } + + private async create( + event: IpcMainEvent, + params: { + uri: string; + title?: string; + config?: Record; + source?: string; + } + ) { + let { uri, title, config, source } = params; + if (uri.startsWith("http")) { + uri = await downloader.download(uri, { + webContents: event.sender, + }); + if (!uri) throw new Error("Failed to download file"); + } + + try { + const document = await Document.buildFromLocalFile(uri, { + title, + config, + source, + }); + + return document.toJSON(); + } catch (err) { + logger.error(err.message); + throw err; + } + } + + private async update( + _event: IpcMainEvent, + id: string, + params: Attributes + ) { + const { title, metadata, lastReadPosition, lastReadAt, config } = params; + + const document = await Document.findByPk(id); + + if (!document) { + throw new Error(t("models.document.notFound")); + } + return await document.update({ + title, + metadata, + lastReadPosition, + lastReadAt, + config, + }); + } + + private async destroy(_event: IpcMainEvent, id: string) { + const document = await Document.findByPk(id); + + if (!document) { + throw new Error(t("models.document.notFound")); + } + return await document.destroy(); + } + + private async upload(event: IpcMainEvent, id: string) { + const document = await Document.findByPk(id); + if (!document) { + throw new Error(t("models.document.notFound")); + } + + return await document.upload(); + } + + private async cleanUp() { + const documents = await Document.findAll(); + + for (const document of documents) { + if (!document.src) { + document.destroy(); + } + } + } + + register() { + ipcMain.handle("documents-find-all", this.findAll); + ipcMain.handle("documents-find-one", this.findOne); + ipcMain.handle("documents-create", this.create); + ipcMain.handle("documents-update", this.update); + ipcMain.handle("documents-destroy", this.destroy); + ipcMain.handle("documents-upload", this.upload); + ipcMain.handle("documents-clean-up", this.cleanUp); + } + + unregister() { + ipcMain.removeHandler("documents-find-all"); + ipcMain.removeHandler("documents-find-one"); + ipcMain.removeHandler("documents-create"); + ipcMain.removeHandler("documents-update"); + ipcMain.removeHandler("documents-destroy"); + ipcMain.removeHandler("documents-upload"); + ipcMain.removeHandler("documents-clean-up"); + } +} + +export const documentsHandler = new DocumentsHandler(); diff --git a/enjoy/src/main/db/handlers/index.ts b/enjoy/src/main/db/handlers/index.ts index 57887a81..43801d49 100644 --- a/enjoy/src/main/db/handlers/index.ts +++ b/enjoy/src/main/db/handlers/index.ts @@ -14,3 +14,4 @@ export * from "./segments-handler"; export * from "./transcriptions-handler"; export * from "./user-settings-handler"; export * from "./videos-handler"; +export * from "./documents-handler"; diff --git a/enjoy/src/main/db/handlers/speeches-handler.ts b/enjoy/src/main/db/handlers/speeches-handler.ts index 9e73a7b0..1889f5cb 100644 --- a/enjoy/src/main/db/handlers/speeches-handler.ts +++ b/enjoy/src/main/db/handlers/speeches-handler.ts @@ -25,6 +25,8 @@ class SpeechesHandler { sourceId: string; sourceType: string; text: string; + section?: number; + segment?: number; configuration: { engine: string; model: string; @@ -55,14 +57,20 @@ class SpeechesHandler { }); } + private async delete(event: IpcMainEvent, id: string) { + await Speech.destroy({ where: { id } }); + } + register() { ipcMain.handle("speeches-find-one", this.findOne); ipcMain.handle("speeches-create", this.create); + ipcMain.handle("speeches-delete", this.delete); } unregister() { ipcMain.removeHandler("speeches-find-one"); ipcMain.removeHandler("speeches-create"); + ipcMain.removeHandler("speeches-delete"); } } diff --git a/enjoy/src/main/db/index.ts b/enjoy/src/main/db/index.ts index 93dce88a..a59b7e63 100644 --- a/enjoy/src/main/db/index.ts +++ b/enjoy/src/main/db/index.ts @@ -11,6 +11,7 @@ import { ChatMember, ChatMessage, Conversation, + Document, Message, Note, PronunciationAssessment, @@ -28,6 +29,7 @@ import { chatMessagesHandler, chatsHandler, conversationsHandler, + documentsHandler, messagesHandler, notesHandler, pronunciationAssessmentsHandler, @@ -68,6 +70,7 @@ const handlers = [ chatMessagesHandler, chatsHandler, conversationsHandler, + documentsHandler, messagesHandler, notesHandler, pronunciationAssessmentsHandler, @@ -107,6 +110,7 @@ db.connect = async () => { ChatMember, ChatMessage, Conversation, + Document, Message, Note, PronunciationAssessment, diff --git a/enjoy/src/main/db/migrations/1729731114150-create-documents.js b/enjoy/src/main/db/migrations/1729731114150-create-documents.js new file mode 100644 index 00000000..66b207b5 --- /dev/null +++ b/enjoy/src/main/db/migrations/1729731114150-create-documents.js @@ -0,0 +1,77 @@ +import { DataTypes } from "sequelize"; + +async function up({ context: queryInterface }) { + queryInterface.createTable( + "documents", + { + id: { + type: DataTypes.UUID, + defaultValue: DataTypes.UUIDV4, + primaryKey: true, + allowNull: false, + }, + title: { + type: DataTypes.STRING, + allowNull: false, + }, + md5: { + type: DataTypes.STRING, + allowNull: false, + }, + language: { + type: DataTypes.STRING, + }, + cover_url: { + type: DataTypes.STRING, + }, + source: { + type: DataTypes.STRING, + }, + metadata: { + type: DataTypes.JSON, + allowNull: false, + defaultValue: {}, + }, + config: { + type: DataTypes.JSON, + allowNull: false, + defaultValue: {}, + }, + last_read_position: { + type: DataTypes.JSON, + defaultValue: {}, + }, + last_read_at: { + type: DataTypes.DATE, + }, + synced_at: { + type: DataTypes.DATE, + }, + uploaded_at: { + type: DataTypes.DATE, + }, + created_at: { + type: DataTypes.DATE, + allowNull: false, + }, + updated_at: { + type: DataTypes.DATE, + allowNull: false, + }, + }, + { + indexes: [ + { + fields: ["md5"], + unique: true, + }, + ], + } + ); +} + +async function down({ context: queryInterface }) { + queryInterface.dropTable("documents"); +} + +export { up, down }; diff --git a/enjoy/src/main/db/migrations/1730343273827-add-section-and-segment-to-speeches.js b/enjoy/src/main/db/migrations/1730343273827-add-section-and-segment-to-speeches.js new file mode 100644 index 00000000..7e1fdfab --- /dev/null +++ b/enjoy/src/main/db/migrations/1730343273827-add-section-and-segment-to-speeches.js @@ -0,0 +1,21 @@ +import { DataTypes } from "sequelize"; + +async function up({ context: queryInterface }) { + await queryInterface.addColumn("speeches", "section", { + type: DataTypes.INTEGER, + allowNull: true, + defaultValue: 0, + }); + await queryInterface.addColumn("speeches", "segment", { + type: DataTypes.INTEGER, + allowNull: true, + defaultValue: 0, + }); +} + +async function down({ context: queryInterface }) { + await queryInterface.removeColumn("speeches", "section"); + await queryInterface.removeColumn("speeches", "segment"); +} + +export { up, down }; diff --git a/enjoy/src/main/db/models/document.ts b/enjoy/src/main/db/models/document.ts new file mode 100644 index 00000000..0e1e0c69 --- /dev/null +++ b/enjoy/src/main/db/models/document.ts @@ -0,0 +1,344 @@ +import { + AfterUpdate, + AfterDestroy, + Table, + Column, + Default, + IsUUID, + Model, + DataType, + AfterCreate, + AfterFind, + Unique, +} from "sequelize-typescript"; +import mainWindow from "@main/window"; +import log from "@main/logger"; +import { Client } from "@/api"; +import settings from "@main/settings"; +import { UserSetting } from "@main/db/models"; +import fs from "fs-extra"; +import { t } from "i18next"; +import path from "path"; +import { DocumentFormats } from "@/constants"; +import { enjoyUrlToPath, hashFile } from "@/main/utils"; +import { v5 as uuidv5 } from "uuid"; +import { fileTypeFromFile } from "file-type"; +import mime from "mime-types"; +import storage from "@/main/storage"; + +const logger = log.scope("db/models/document"); +@Table({ + modelName: "Document", + tableName: "documents", + underscored: true, + timestamps: true, +}) +export class Document extends Model { + @IsUUID("all") + @Default(DataType.UUIDV4) + @Column({ primaryKey: true, type: DataType.UUID }) + id: string; + + @Column(DataType.STRING) + language: string; + + @Unique + @Column(DataType.STRING) + md5: string; + + @Column(DataType.STRING) + title: string; + + @Column(DataType.STRING) + coverUrl: string; + + @Column(DataType.STRING) + source: string; + + @Column(DataType.JSON) + metadata: Record; + + @Column(DataType.JSON) + config: Record; + + @Column(DataType.JSON) + lastReadPosition: Record; + + @Column(DataType.DATE) + lastReadAt: Date; + + @Column(DataType.DATE) + syncedAt: Date; + + @Column(DataType.DATE) + uploadedAt: Date; + + @Column(DataType.VIRTUAL) + get autoTranslate(): boolean { + return this.config.autoTranslate || false; + } + + @Column(DataType.VIRTUAL) + get autoNextSpeech(): boolean { + return this.config.autoNextSpeech || false; + } + + @Column(DataType.VIRTUAL) + get ttsConfig(): Record { + return this.config.tts || {}; + } + + @Column(DataType.VIRTUAL) + get filePath(): string { + const file = path.join( + settings.userDataPath(), + "documents", + `${this.md5}.${this.metadata.extension}` + ); + if (fs.existsSync(file)) { + return file; + } + return null; + } + + @Column(DataType.VIRTUAL) + get src(): string { + if (!this.filePath) return null; + + return `enjoy://${path.posix.join( + "library", + "documents", + `${this.md5}.${this.metadata.extension}` + )}`; + } + + @Column(DataType.VIRTUAL) + get isSynced(): boolean { + return Boolean(this.syncedAt) && this.syncedAt >= this.updatedAt; + } + + @Column(DataType.VIRTUAL) + get isUploaded(): boolean { + return Boolean(this.uploadedAt) && this.uploadedAt >= this.updatedAt; + } + + async sync(): Promise { + if (this.isSynced) return; + + const webApi = new Client({ + baseUrl: settings.apiUrl(), + accessToken: (await UserSetting.accessToken()) as string, + logger, + }); + + return webApi.syncDocument(this.toJSON()).then(() => { + const now = new Date(); + this.update({ syncedAt: now, updatedAt: now }); + }); + } + + async upload(force: boolean = false): Promise { + if (this.isUploaded && !force) return; + + return storage + .put(this.md5, this.filePath) + .then((result) => { + logger.debug("upload result:", result.data); + if (result.data.success) { + this.update({ uploadedAt: new Date() }); + } else { + throw new Error(result.data); + } + }) + .catch((err) => { + logger.error("upload failed:", err.message); + throw err; + }); + } + + @AfterFind + static async syncAfterFind(documents: Document[]) { + if (!documents?.length) return; + + const unsyncedDocuments = documents.filter( + (document) => document.id && !document.isSynced + ); + if (!unsyncedDocuments.length) return; + + unsyncedDocuments.forEach((document) => { + document.sync().catch((err) => { + logger.error(err.message); + }); + }); + } + + @AfterCreate + static syncAndUploadAfterCreate(document: Document) { + document.sync().catch((err) => { + logger.error(err.message); + }); + } + + @AfterCreate + static notifyForCreate(document: Document) { + this.notify(document, "create"); + } + + @AfterUpdate + static notifyForUpdate(document: Document) { + if (document.changed("config") || document.changed("title")) { + this.notify(document, "update"); + } + } + + @AfterUpdate + static syncAfterUpdate(document: Document) { + document.sync().catch((err) => { + logger.error(err.message); + }); + } + + @AfterDestroy + static async destroyRemote(document: Document) { + const webApi = new Client({ + baseUrl: settings.apiUrl(), + accessToken: (await UserSetting.accessToken()) as string, + logger, + }); + + webApi.deleteDocument(document.id).catch((err) => { + logger.error("delete remote document failed:", err.message); + }); + } + + @AfterDestroy + static notifyForDestroy(document: Document) { + this.notify(document, "destroy"); + } + + static async buildFromLocalFile( + filePath: string, + params: { + title?: string; + config?: Record; + source?: string; + } + ): Promise { + // Check if file exists + if (filePath.startsWith("enjoy://")) { + filePath = enjoyUrlToPath(filePath); + } + try { + fs.accessSync(filePath, fs.constants.R_OK); + } catch (error) { + throw new Error(t("models.document.fileNotFound", { file: filePath })); + } + + // calculate md5 + const md5 = await hashFile(filePath, { algo: "md5" }); + + const existing = await Document.findOne({ + where: { + md5, + }, + }); + if (existing) { + return existing; + } + + // Check if file format is supported + let mimeType: string; + let extension: string; + const fileType = await fileTypeFromFile(filePath); + if (fileType) { + mimeType = fileType.mime; + extension = fileType.ext; + } else { + mimeType = mime.lookup(filePath) || ""; + extension = mime.extension(mimeType) || ""; + } + + logger.debug("detected file type", filePath, mimeType, extension); + if (!DocumentFormats.includes(extension)) { + logger.error("unsupported file type", filePath, extension); + throw new Error( + t("models.document.fileNotSupported", { file: filePath }) + ); + } + + // get file's metadata + const stat = await fs.promises.stat(filePath); + + const metadata = { + size: stat.size, + created: stat.birthtime, + modified: stat.mtime, + mimeType, + extension, + extname: extension, + }; + + // generate ID + const userId = settings.getSync("user.id"); + const id = uuidv5(`${userId}/${md5}`, uuidv5.URL); + + const destDir = path.join(settings.userDataPath(), "documents"); + fs.ensureDirSync(destDir); + const destFile = path.join(destDir, `${md5}.${extension}`); + + try { + // copy file to library + fs.copyFileSync(filePath, destFile); + } catch (error) { + logger.error("failed to copy file", filePath, error); + throw new Error( + t("models.document.failedToCopyFile", { file: filePath }) + ); + } + + const { + title = path.basename(filePath, `.${extension}`), + config = { + autoTranslate: false, + autoNextSpeech: true, + tts: { + engine: "enjoyai", + model: "openai/tts-1", + voice: "alloy", + }, + }, + source, + } = params || {}; + + const record = this.build({ + id, + md5, + title, + metadata, + config, + source, + }); + + return record.save().catch((err) => { + // remove copied file + fs.removeSync(destFile); + throw err; + }); + } + + static async notify( + document: Document, + action: "create" | "update" | "destroy" + ) { + if (!mainWindow.win) return; + + const record = document.toJSON(); + + mainWindow.win.webContents.send("db-on-transaction", { + model: "Document", + id: document.id, + action, + record, + }); + } +} diff --git a/enjoy/src/main/db/models/index.ts b/enjoy/src/main/db/models/index.ts index e38189b3..846e05a4 100644 --- a/enjoy/src/main/db/models/index.ts +++ b/enjoy/src/main/db/models/index.ts @@ -14,3 +14,4 @@ export * from "./speech"; export * from "./user-setting"; export * from "./transcription"; export * from "./video"; +export * from "./document"; diff --git a/enjoy/src/main/db/models/speech.ts b/enjoy/src/main/db/models/speech.ts index f99fcb91..6de597b5 100644 --- a/enjoy/src/main/db/models/speech.ts +++ b/enjoy/src/main/db/models/speech.ts @@ -21,7 +21,7 @@ import settings from "@main/settings"; import OpenAI, { type ClientOptions } from "openai"; import { t } from "i18next"; import { hashFile } from "@main/utils"; -import { Audio, Message, UserSetting } from "@main/db/models"; +import { Audio, Document, Message, UserSetting } from "@main/db/models"; import log from "@main/logger"; import proxyAgent from "@main/proxy-agent"; @@ -55,11 +55,14 @@ export class Speech extends Model { sourceType: string; @Column(DataType.VIRTUAL) - source: Message; + source: Message | Document; @BelongsTo(() => Message, { foreignKey: "sourceId", constraints: false }) message: Message; + @BelongsTo(() => Document, { foreignKey: "sourceId", constraints: false }) + document: Document; + @HasOne(() => Audio, "md5") audio: Audio; @@ -67,6 +70,14 @@ export class Speech extends Model { @Column(DataType.TEXT) text: string; + @AllowNull(true) + @Column(DataType.INTEGER) + section: number; + + @AllowNull(true) + @Column(DataType.INTEGER) + segment: number; + @AllowNull(false) @Column(DataType.JSON) configuration: any; @@ -125,9 +136,15 @@ export class Speech extends Model { if (!instance) continue; if (instance.sourceType === "Message" && instance.message !== undefined) { instance.source = instance.message; + } else if ( + instance.sourceType === "Document" && + instance.document !== undefined + ) { + instance.source = instance.document; } // To prevent mistakes: delete instance.dataValues.message; + delete instance.dataValues.document; } } diff --git a/enjoy/src/main/window.ts b/enjoy/src/main/window.ts index 5c4e0067..849306f2 100644 --- a/enjoy/src/main/window.ts +++ b/enjoy/src/main/window.ts @@ -257,7 +257,32 @@ main.init = async () => { view.setVisible(false); mainWindow.contentView.addChildView(view); + // Add timeout handler + const timeout = setTimeout(() => { + logger.debug("view-scrape timeout", url); + event.sender.send("view-on-state", { + state: "did-fail-load", + error: "Request timed out", + url: url, + }); + (view.webContents as any)?.destroy(); + mainWindow.contentView.removeChildView(view); + }, 30000); // 30 second timeout + + view.webContents.on("did-start-loading", () => { + logger.debug("view-scrape did-start-loading", url); + }); + + view.webContents.on("did-stop-loading", () => { + logger.debug("view-scrape did-stop-loading", url); + }); + + view.webContents.on("dom-ready", () => { + logger.debug("view-scrape dom-ready", url); + }); + view.webContents.on("did-navigate", (_event, url) => { + clearTimeout(timeout); event.sender.send("view-on-state", { state: "did-navigate", url, @@ -266,6 +291,7 @@ main.init = async () => { view.webContents.on( "did-fail-load", (_event, _errorCode, errrorDescription, validatedURL) => { + clearTimeout(timeout); event.sender.send("view-on-state", { state: "did-fail-load", error: errrorDescription, @@ -276,19 +302,31 @@ main.init = async () => { } ); view.webContents.on("did-finish-load", () => { + clearTimeout(timeout); + logger.debug("view-scrape did-finish-load", url); view.webContents .executeJavaScript(`document.documentElement.innerHTML`) .then((html) => { event.sender.send("view-on-state", { state: "did-finish-load", html, + url, }); (view.webContents as any).destroy(); mainWindow.contentView.removeChildView(view); }); }); - view.webContents.loadURL(url); + view.webContents.loadURL(url).catch((err) => { + logger.error("view-scrape loadURL error", err); + (view.webContents as any).destroy(); + mainWindow.contentView.removeChildView(view); + event.sender.send("view-on-state", { + state: "did-fail-load", + error: err.message, + url: url, + }); + }); }); // App options diff --git a/enjoy/src/preload.ts b/enjoy/src/preload.ts index cc520d18..aa511124 100644 --- a/enjoy/src/preload.ts +++ b/enjoy/src/preload.ts @@ -457,6 +457,9 @@ contextBridge.exposeInMainWorld("__ENJOY_APP__", { ) => { return ipcRenderer.invoke("speeches-create", params, blob); }, + delete: (id: string) => { + return ipcRenderer.invoke("speeches-delete", id); + }, }, audiowaveform: { generate: ( @@ -699,4 +702,27 @@ contextBridge.exposeInMainWorld("__ENJOY_APP__", { return ipcRenderer.invoke("chat-messages-destroy", id); }, }, + documents: { + findAll: (params: any) => { + return ipcRenderer.invoke("documents-find-all", params); + }, + findOne: (params: any) => { + return ipcRenderer.invoke("documents-find-one", params); + }, + create: (params: any) => { + return ipcRenderer.invoke("documents-create", params); + }, + update: (id: string, params: any) => { + return ipcRenderer.invoke("documents-update", id, params); + }, + destroy: (id: string) => { + return ipcRenderer.invoke("documents-destroy", id); + }, + upload: (id: string) => { + return ipcRenderer.invoke("documents-upload", id); + }, + cleanUp: () => { + return ipcRenderer.invoke("documents-clean-up"); + }, + }, }); diff --git a/enjoy/src/renderer/components/audios/audios-segment.tsx b/enjoy/src/renderer/components/audios/audios-segment.tsx index 88f6b88d..939193a8 100644 --- a/enjoy/src/renderer/components/audios/audios-segment.tsx +++ b/enjoy/src/renderer/components/audios/audios-segment.tsx @@ -62,7 +62,7 @@ export const AudiosSegment = (props: { limit?: number }) => { {audios.length === 0 ? (
- +
) : ( diff --git a/enjoy/src/renderer/components/chats/chat-agent-form.tsx b/enjoy/src/renderer/components/chats/chat-agent-form.tsx index c7245c3d..403b1350 100644 --- a/enjoy/src/renderer/components/chats/chat-agent-form.tsx +++ b/enjoy/src/renderer/components/chats/chat-agent-form.tsx @@ -30,7 +30,7 @@ import { toast, } from "@renderer/components/ui"; import { t } from "i18next"; -import { ChatTTSForm } from "@renderer/components"; +import { TTSForm } from "@renderer/components"; import { AISettingsProviderContext, AppSettingsProviderContext, @@ -366,7 +366,7 @@ export const ChatAgentForm = (props: { )} {form.watch("type") === ChatAgentTypeEnum.TTS && ( - + )}
diff --git a/enjoy/src/renderer/components/chats/chat-agent-message.tsx b/enjoy/src/renderer/components/chats/chat-agent-message.tsx index 66d0b17a..8a65c93f 100644 --- a/enjoy/src/renderer/components/chats/chat-agent-message.tsx +++ b/enjoy/src/renderer/components/chats/chat-agent-message.tsx @@ -174,7 +174,7 @@ const ChatAgentMessageActions = (props: { setTranslation, autoSpeech, } = props; - const { chat, setShadowing, deleteMessage } = useContext( + const { setShadowing, deleteMessage } = useContext( ChatSessionProviderContext ); const { EnjoyApp } = useContext(AppSettingsProviderContext); diff --git a/enjoy/src/renderer/components/chats/chat-form.tsx b/enjoy/src/renderer/components/chats/chat-form.tsx index b262ad13..b7e90e12 100644 --- a/enjoy/src/renderer/components/chats/chat-form.tsx +++ b/enjoy/src/renderer/components/chats/chat-form.tsx @@ -43,8 +43,8 @@ import { SttEngineOptionEnum, } from "@/types/enums"; import { ChevronDownIcon, ChevronUpIcon, RefreshCwIcon } from "lucide-react"; -import { useAiCommand } from "@/renderer/hooks"; -import { cn } from "@/renderer/lib/utils"; +import { useAiCommand } from "@renderer/hooks"; +import { cn } from "@renderer/lib/utils"; export const ChatForm = (props: { chat: ChatType; onFinish?: () => void }) => { const { chat, onFinish } = props; diff --git a/enjoy/src/renderer/components/chats/chat-member-form.tsx b/enjoy/src/renderer/components/chats/chat-member-form.tsx index fd82c84d..e0df2dd6 100644 --- a/enjoy/src/renderer/components/chats/chat-member-form.tsx +++ b/enjoy/src/renderer/components/chats/chat-member-form.tsx @@ -30,7 +30,7 @@ import { t } from "i18next"; import { useContext } from "react"; import { AppSettingsProviderContext } from "@renderer/context"; import Mustache from "mustache"; -import { ChatGPTForm, ChatTTSForm } from "@renderer/components"; +import { GPTForm, TTSForm } from "@renderer/components"; export const ChatMemberForm = (props: { chat: ChatType; @@ -164,7 +164,7 @@ export const ChatMemberForm = (props: { {t("models.chatMember.gptSettings")} - + @@ -173,7 +173,7 @@ export const ChatMemberForm = (props: { {t("models.chatMember.ttsSettings")} - + diff --git a/enjoy/src/renderer/components/chats/index.ts b/enjoy/src/renderer/components/chats/index.ts index ad23edbc..a3ba4df7 100644 --- a/enjoy/src/renderer/components/chats/index.ts +++ b/enjoy/src/renderer/components/chats/index.ts @@ -13,7 +13,5 @@ export * from "./chat-agent-card"; export * from "./chat-settings"; export * from "./chat-member-form"; export * from "./chat-header"; -export * from "./chat-tts-form"; -export * from "./chat-gpt-form"; export * from "./chat-suggestion-button"; export * from "./chat-mentioning"; diff --git a/enjoy/src/renderer/components/documents/document-add-button.tsx b/enjoy/src/renderer/components/documents/document-add-button.tsx new file mode 100644 index 00000000..a25e0a34 --- /dev/null +++ b/enjoy/src/renderer/components/documents/document-add-button.tsx @@ -0,0 +1,176 @@ +import { + Dialog, + DialogTrigger, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, + DialogFooter, + Input, + Button, + toast, +} from "@renderer/components/ui"; +import { PlusCircleIcon, LoaderIcon } from "lucide-react"; +import { t } from "i18next"; +import { useState, useContext, useEffect } from "react"; +import { DocumentFormats } from "@/constants"; +import { AppSettingsProviderContext } from "@renderer/context"; +import { useNavigate } from "react-router-dom"; +import { Readability } from "@mozilla/readability"; +import { Buffer } from "buffer"; + +export const DocumentAddButton = () => { + const { EnjoyApp } = useContext(AppSettingsProviderContext); + const navigate = useNavigate(); + const [uri, setUri] = useState(""); + const [open, setOpen] = useState(false); + const [submitting, setSubmitting] = useState(false); + + const handleOpen = (value: boolean) => { + if (submitting) { + setOpen(true); + } else { + setOpen(value); + } + }; + + const handleSubmit = async () => { + if (!uri) return; + + setSubmitting(true); + if (uri.startsWith("http")) { + EnjoyApp.view.scrape(uri); + } else { + createFromLocalFile(uri, uri); + } + }; + + const createFromLocalFile = async (path: string, source?: string) => { + EnjoyApp.documents + .create({ + uri: path, + config: { + autoTranslate: false, + autoNextSpeech: true, + tts: { + engine: "enjoyai", + model: "openai/tts-1", + voice: "alloy", + }, + }, + source, + }) + .then((doc) => { + navigate(`/documents/${doc.id}`); + }) + .catch((err) => { + toast.error(err.message); + }) + .finally(() => { + setSubmitting(false); + setOpen(false); + }); + }; + + const onViewState = async (event: { + state: string; + url?: string; + error?: string; + html?: string; + }) => { + const { state, html, error, url } = event; + if (state === "did-finish-load") { + const doc = new DOMParser().parseFromString(html, "text/html"); + const reader = new Readability(doc); + const article = reader.parse(); + + const file = await EnjoyApp.cacheObjects.writeFile( + `${doc.title}.html`, + Buffer.from(article.content) + ); + createFromLocalFile(file, url); + } else if (state === "did-fail-load") { + setSubmitting(false); + toast.error(error || t("failedToLoadLink")); + } + }; + + useEffect(() => { + EnjoyApp.view.onViewState((_event, state) => onViewState(state)); + + return () => { + EnjoyApp.view.removeViewStateListeners(); + EnjoyApp.view.remove(); + }; + }, []); + + return ( + + + + + + + {t("addResource")} + + {t("addResourceFromUrlOrLocal")} + + + +
+ { + setUri(element.target.value); + }} + /> + +
+ + + + + + + + ); +}; diff --git a/enjoy/src/renderer/components/documents/document-card.tsx b/enjoy/src/renderer/components/documents/document-card.tsx new file mode 100644 index 00000000..ff1fe604 --- /dev/null +++ b/enjoy/src/renderer/components/documents/document-card.tsx @@ -0,0 +1,104 @@ +import { cn } from "@renderer/lib/utils"; +import { Link } from "react-router-dom"; +import { + AlertDialog, + AlertDialogContent, + AlertDialogHeader, + AlertDialogTitle, + AlertDialogDescription, + Button, + DropdownMenu, + DropdownMenuTrigger, + DropdownMenuContent, + DropdownMenuItem, + AlertDialogFooter, +} from "@renderer/components/ui"; +import { MoreVerticalIcon, TrashIcon } from "lucide-react"; +import { t } from "i18next"; +import { useState } from "react"; + +export const DocumentCard = (props: { + document: DocumentEType; + className?: string; +}) => { + const { document, className } = props; + const [deleting, setDeleting] = useState(false); + const handleDelete = (event: React.MouseEvent) => { + event.stopPropagation(); + setDeleting(true); + }; + return ( +
+ +
+ {/* Book body */} +
+ + {/* Book spine */} +
+ + {/* Book title */} +
+

+ {document.title} +

+
+ {/* drop menu */} +
+ + + + + + + + + + {t("delete")} + + + + +
+ +
+ {document.metadata?.extension} +
+
+ + + + + {t("delete")} + + {t("deleteConfirm")} + + + + + + + + +
+ ); +}; diff --git a/enjoy/src/renderer/components/documents/document-config-button.tsx b/enjoy/src/renderer/components/documents/document-config-button.tsx new file mode 100644 index 00000000..5ed1723a --- /dev/null +++ b/enjoy/src/renderer/components/documents/document-config-button.tsx @@ -0,0 +1,46 @@ +import { + Button, + Popover, + PopoverContent, + PopoverTrigger, + toast, +} from "@renderer/components/ui"; +import { SettingsIcon } from "lucide-react"; +import { useContext, useState } from "react"; +import { DocumentConfigForm } from "@renderer/components"; +import { AppSettingsProviderContext } from "@/renderer/context"; +import { t } from "i18next"; + +export const DocumentConfigButton = (props: { document: DocumentEType }) => { + const { document } = props; + const [configOpen, setConfigOpen] = useState(false); + const { EnjoyApp } = useContext(AppSettingsProviderContext); + + return ( + + + + + + { + return EnjoyApp.documents + .update(document.id, { + ...data, + }) + .then(() => { + toast.success(t("saved")); + setConfigOpen(false); + }) + .catch((err) => { + toast.error(err.message); + }); + }} + /> + + + ); +}; diff --git a/enjoy/src/renderer/components/documents/document-config-form.tsx b/enjoy/src/renderer/components/documents/document-config-form.tsx new file mode 100644 index 00000000..5707dd55 --- /dev/null +++ b/enjoy/src/renderer/components/documents/document-config-form.tsx @@ -0,0 +1,110 @@ +import { zodResolver } from "@hookform/resolvers/zod"; +import { useForm } from "react-hook-form"; +import { z } from "zod"; +import { + Button, + Form, + FormField, + FormItem, + FormLabel, + Switch, +} from "@renderer/components/ui"; +import { t } from "i18next"; +import { TTSForm } from "@renderer/components"; +import { LoaderIcon } from "lucide-react"; +import { useState } from "react"; + +const documentConfigSchema = z.object({ + config: z.object({ + autoTranslate: z.boolean(), + autoNextSpeech: z.boolean(), + tts: z.object({ + engine: z.string(), + model: z.string(), + voice: z.string(), + language: z.string(), + }), + }), +}); + +export const DocumentConfigForm = (props: { + config?: DocumentEType["config"]; + onSubmit: (data: z.infer) => Promise; +}) => { + const { config, onSubmit } = props; + const [submitting, setSubmitting] = useState(false); + + const form = useForm>({ + resolver: zodResolver(documentConfigSchema), + defaultValues: config + ? { config } + : { + config: { + autoTranslate: true, + autoNextSpeech: true, + tts: { + engine: "openai", + model: "openai/tts-1", + language: "en-US", + voice: "alloy", + }, + }, + }, + }); + + return ( +
+ { + setSubmitting(true); + onSubmit(data).finally(() => { + setSubmitting(false); + }); + })} + > +
+ ( + +
+ {t("autoTranslate")} + +
+ + )} + /> + + ( + +
+ {t("autoNextSpeech")} + +
+ + )} + /> + + +
+ +
+ +
+
+ + ); +}; diff --git a/enjoy/src/renderer/components/documents/document-epub-renderer.tsx b/enjoy/src/renderer/components/documents/document-epub-renderer.tsx new file mode 100644 index 00000000..296fa0b5 --- /dev/null +++ b/enjoy/src/renderer/components/documents/document-epub-renderer.tsx @@ -0,0 +1,218 @@ +import { useCallback, useContext, useEffect, useState } from "react"; +import { + DocumentConfigButton, + LoaderSpin, + MarkdownWrapper, +} from "@renderer/components"; +import { makeBook } from "foliate-js/view.js"; +import { EPUB } from "foliate-js/epub.js"; +import { blobToDataUrl } from "@renderer/lib/utils"; +import Turndown from "turndown"; +import { + DropdownMenu, + DropdownMenuTrigger, + DropdownMenuContent, + DropdownMenuItem, + Button, + toast, +} from "@renderer/components/ui"; +import { ChevronLeftIcon, ChevronRightIcon, MenuIcon } from "lucide-react"; +import { + AppSettingsProviderContext, + DocumentProviderContext, +} from "@renderer/context"; + +export const DocumentEpubRenderer = () => { + const { + ref, + document, + onSpeech, + section, + setSection, + onSegmentVisible, + content, + setContent, + } = useContext(DocumentProviderContext); + const { EnjoyApp } = useContext(AppSettingsProviderContext); + + const [book, setBook] = useState(); + const [title, setTitle] = useState(""); + const [loading, setLoading] = useState(true); + + const refreshBookMetadata = () => { + if (!book) return; + + if (document.title !== book.metadata.title) { + EnjoyApp.documents.update(document.id, { + title: book.metadata.title, + language: book.metadata.language, + }); + } + }; + + const renderCurrentSection = async () => { + setLoading(true); + + try { + const sectionDoc = await book.sections[section].createDocument(); + const tocItem = book.toc.find((item: any) => item.href === sectionDoc.id); + setTitle(tocItem?.label || sectionDoc.title); + + for (const img of sectionDoc.body.querySelectorAll("img")) { + let image: any; + if (img.src) { + image = book.resources.manifest.find((resource: any) => + resource.href.endsWith(new URL(img.src).pathname) + ); + } else if (img.id) { + image = book.resources.manifest.find( + (resource: any) => resource.id === img.id + ); + } + if (!image) continue; + + const blob = new Blob([await book.loadBlob(image.href)], { + type: image.mediaType, + }); + const url = await blobToDataUrl(blob); + img.setAttribute("src", url); + } + + const markdownContent = new Turndown().turndown( + sectionDoc.body.innerHTML + ); + setContent(markdownContent); + } catch (err) { + toast.error(err.message); + } finally { + setLoading(false); + } + }; + + const handlePrevSection = () => { + if (section === 0) return; + if (!book) return; + + setSection(section - 1); + }; + + const handleNextSection = () => { + if (section === book.sections.length - 1) return; + if (!book) return; + + setSection(section + 1); + }; + + const handleSectionClick = useCallback( + (id: string) => { + const sec = book.sections.findIndex((sec: any) => sec.id.endsWith(id)); + if (sec === -1) return; + + setSection(sec); + }, + [book] + ); + + const handleLinkClick = useCallback( + (e: React.MouseEvent) => { + e.preventDefault(); + handleSectionClick(new URL(e.currentTarget.href).pathname); + e.currentTarget.blur(); + }, + [handleSectionClick] + ); + + useEffect(() => { + makeBook(document.src).then((epub: typeof EPUB) => { + setBook(epub); + setLoading(false); + }); + }, [document?.src]); + + useEffect(() => { + if (!book) return; + + refreshBookMetadata(); + renderCurrentSection(); + }, [book, section]); + + if (!book) return ; + + return ( +
+
+
+ + + + + + {(book?.toc as any[]).map((item: any) => ( +
+ handleSectionClick(item.href)} + > + {item.label} + + {(item.subitems || []).map((subitem: any) => ( + handleSectionClick(subitem.href)} + > + {subitem.label} + + ))} +
+ ))} + + + +
+
{title}
+
+ + +
+
+
+ {loading ? ( + + ) : ( + + {content} + + )} +
+ ); +}; diff --git a/enjoy/src/renderer/components/documents/document-html-renderer.tsx b/enjoy/src/renderer/components/documents/document-html-renderer.tsx new file mode 100644 index 00000000..45d509c6 --- /dev/null +++ b/enjoy/src/renderer/components/documents/document-html-renderer.tsx @@ -0,0 +1,84 @@ +import { Readability } from "@mozilla/readability"; +import { useContext, useEffect, useState } from "react"; +import { + DocumentConfigButton, + LoaderSpin, + MarkdownWrapper, +} from "@renderer/components"; +import Turndown from "turndown"; +import { + AppSettingsProviderContext, + DocumentProviderContext, +} from "@/renderer/context"; +import { Button } from "../ui"; +import { LinkIcon } from "lucide-react"; + +export const DocumentHtmlRenderer = () => { + const { document, onSpeech, onSegmentVisible, content, setContent } = + useContext(DocumentProviderContext); + const { EnjoyApp } = useContext(AppSettingsProviderContext); + const [title, setTitle] = useState(""); + + const fetchContent = async () => { + const res = await fetch(document.src); + const text = await res.text(); + const doc = new DOMParser().parseFromString(text, "text/html"); + setTitle(doc.title || document.title); + const readability = new Readability(doc); + const article = readability.parse(); + const markdownContent = new Turndown().turndown(article.content); + setContent(markdownContent); + }; + + useEffect(() => { + fetchContent(); + }, [document.src]); + + useEffect(() => { + if (!title) return; + + if (document.title !== title) { + EnjoyApp.documents.update(document.id, { + title, + }); + } + }, [title]); + + if (!content) return ; + + return ( +
+
+
+ +
+
+ {title} +
+
+ {document.metadata?.source && ( + + )} +
+
+ + {content} + +
+ ); +}; diff --git a/enjoy/src/renderer/components/documents/document-player.tsx b/enjoy/src/renderer/components/documents/document-player.tsx new file mode 100644 index 00000000..7ee1a5e5 --- /dev/null +++ b/enjoy/src/renderer/components/documents/document-player.tsx @@ -0,0 +1,238 @@ +import { + AppSettingsProviderContext, + DocumentProviderContext, +} from "@renderer/context"; +import { useSpeech } from "@renderer/hooks"; +import { useContext, useEffect, useState } from "react"; +import { Button, toast } from "@renderer/components/ui"; +import { + AudioPlayer, + LoaderSpin, + WavesurferPlayer, +} from "@renderer/components"; +import { t } from "i18next"; +import { + LoaderIcon, + LocateFixedIcon, + RefreshCcwIcon, + XIcon, +} from "lucide-react"; + +export const DocumentPlayer = () => { + const { + ref, + document, + section, + togglePlayingSegment, + locateSegment, + playingSegment, + nextSegment, + } = useContext(DocumentProviderContext); + const { EnjoyApp } = useContext(AppSettingsProviderContext); + const [speech, setSpeech] = useState(null); + const [speeching, setSpeeching] = useState(false); + const [resourcing, setResourcing] = useState(false); + const { tts } = useSpeech(); + const [audio, setAudio] = useState(null); + + const startShadow = async () => { + if (!speech) return; + + const audio = await EnjoyApp.audios.findOne({ + md5: speech.md5, + }); + + if (audio) { + setAudio(audio); + } else { + setResourcing(true); + EnjoyApp.audios + .create(speech.filePath, { + name: `[S${section}P${playingSegment.index}]-${document.title}`, + originalText: speech.text, + }) + .then((audio) => setAudio(audio)) + .catch((err) => toast.error(err.message)) + .finally(() => setResourcing(false)); + } + }; + + const findOrCreateSpeech = async () => { + if (typeof section !== "number" || !playingSegment) return; + + const existingSpeech = await EnjoyApp.speeches.findOne({ + sourceId: document.id, + sourceType: "Document", + section, + segment: playingSegment.index, + }); + + if (existingSpeech) { + setSpeech(existingSpeech); + } else { + createSpeech(playingSegment); + } + }; + + const refreshSpeech = async () => { + if (speech) { + await EnjoyApp.speeches.delete(speech.id); + setSpeech(null); + } + findOrCreateSpeech(); + }; + + const createSpeech = async (segment: { index: number; text: string }) => { + if (speeching) return; + const { index, text } = segment; + + setSpeeching(true); + tts({ + sourceId: document.id, + sourceType: "Document", + section, + segment: index, + text, + configuration: document.config.tts, + }) + .then((res) => { + setSpeech(res); + }) + .catch((err) => { + toast.error(err.message); + }) + .finally(() => { + setSpeeching(false); + }); + }; + + useEffect(() => { + if (typeof section !== "number" || !playingSegment) return; + findOrCreateSpeech(); + + return () => { + setSpeech(null); + setAudio(null); + }; + }, [playingSegment]); + + // Close the player when the section changes + useEffect(() => { + return () => { + togglePlayingSegment(null); + }; + }, [section]); + + if (typeof section !== "number" || !playingSegment) { + return ; + } + + if (speeching) { + return ( +
+
+ +
+
+ {t("creatingSpeech")} +
+
+ ); + } + + if (resourcing) { + return ( +
+
+ +
+
+ {t("preparingAudio")} +
+
+ ); + } + + if (!speech) { + return ( +
+ + +
+ ); + } + + if (!audio) { + return ( +
+ { + if (nextSegment) { + togglePlayingSegment(nextSegment.id); + } + }} + className="w-full h-full" + /> +
+ + + + +
+
+ ); + } + + return ( +
+
+ + {audio.name} +
+ +
+ ); +}; diff --git a/enjoy/src/renderer/components/documents/document-text-renderer.tsx b/enjoy/src/renderer/components/documents/document-text-renderer.tsx new file mode 100644 index 00000000..83238e63 --- /dev/null +++ b/enjoy/src/renderer/components/documents/document-text-renderer.tsx @@ -0,0 +1,68 @@ +import { useContext, useEffect } from "react"; +import { + DocumentConfigButton, + LoaderSpin, + MarkdownWrapper, +} from "@renderer/components"; +import { + AppSettingsProviderContext, + DocumentProviderContext, +} from "@renderer/context"; +import { Button } from "@renderer/components/ui"; +import { LinkIcon } from "lucide-react"; + +export const DocumentTextRenderer = () => { + const { document, onSpeech, onSegmentVisible, content, setContent } = + useContext(DocumentProviderContext); + const { EnjoyApp } = useContext(AppSettingsProviderContext); + + const fetchContent = async () => { + const res = await fetch(document.src); + console.log("res", res); + const text = await res.text(); + console.log("text", text); + setContent(text); + }; + + useEffect(() => { + fetchContent(); + }, [document.src]); + + if (!content) return ; + + return ( +
+
+
+ +
+
+ {document.title} +
+
+ {document.metadata?.source && ( + + )} +
+
+ + {content} + +
+ ); +}; diff --git a/enjoy/src/renderer/components/documents/documents-segment.tsx b/enjoy/src/renderer/components/documents/documents-segment.tsx new file mode 100644 index 00000000..8737da06 --- /dev/null +++ b/enjoy/src/renderer/components/documents/documents-segment.tsx @@ -0,0 +1,55 @@ +import { DocumentCard } from "@renderer/components"; +import { Button, ScrollArea, ScrollBar } from "@renderer/components/ui"; +import { t } from "i18next"; +import { Link } from "react-router-dom"; +import { useState, useContext, useEffect } from "react"; +import { AppSettingsProviderContext } from "@renderer/context"; + +export const DocumentsSegment = () => { + const [documents, setDocuments] = useState([]); + const { EnjoyApp } = useContext(AppSettingsProviderContext); + + const fetchDocuments = async () => { + EnjoyApp.documents.findAll({ limit: 10 }).then((docs) => { + setDocuments(docs); + }); + }; + + useEffect(() => { + fetchDocuments(); + }, []); + + if (documents.length == 0) return null; + + return ( +
+
+
+

+ {t("addedDocuments")} +

+
+
+ + + +
+
+ + +
+ {documents.map((document) => ( + + ))} +
+ + +
+ ); +}; diff --git a/enjoy/src/renderer/components/documents/index.ts b/enjoy/src/renderer/components/documents/index.ts new file mode 100644 index 00000000..a96ae7b7 --- /dev/null +++ b/enjoy/src/renderer/components/documents/index.ts @@ -0,0 +1,9 @@ +export * from "./document-card"; +export * from "./document-html-renderer"; +export * from "./document-text-renderer"; +export * from "./document-epub-renderer"; +export * from "./document-player"; +export * from "./document-config-form"; +export * from "./document-add-button"; +export * from "./document-config-button"; +export * from "./documents-segment"; diff --git a/enjoy/src/renderer/components/index.ts b/enjoy/src/renderer/components/index.ts index f738ee08..ebafc31f 100644 --- a/enjoy/src/renderer/components/index.ts +++ b/enjoy/src/renderer/components/index.ts @@ -3,6 +3,7 @@ export * from "./chats"; export * from "./conversations"; export * from "./copilots"; export * from "./courses"; +export * from "./documents"; export * from "./llm-chats"; export * from "./meanings"; export * from "./messages"; diff --git a/enjoy/src/renderer/components/medias/media-bottom-panel/media-current-recording.tsx b/enjoy/src/renderer/components/medias/media-bottom-panel/media-current-recording.tsx index 806c3009..89b437bb 100644 --- a/enjoy/src/renderer/components/medias/media-bottom-panel/media-current-recording.tsx +++ b/enjoy/src/renderer/components/medias/media-bottom-panel/media-current-recording.tsx @@ -547,68 +547,7 @@ export const MediaCurrentRecording = () => { ]; if (isRecording || isPaused) { - return ( -
- - - {Math.floor(recordingTime / 60)}: - {String(recordingTime % 60).padStart(2, "0")} - -
- - - -
-
- ); + return ; } if (!currentRecording?.src) @@ -801,3 +740,108 @@ export const MediaRecordButton = () => { ); }; + +const MediaRecorder = () => { + const { + mediaRecorder, + recordingTime, + isRecording, + isPaused, + cancelRecording, + togglePauseResume, + stopRecording, + } = useContext(MediaShadowProviderContext); + const ref = useRef(null); + const [size, setSize] = useState<{ width: number; height: number } | null>( + null + ); + + const calContainerSize = () => { + const size = ref?.current?.getBoundingClientRect(); + if (!size) return; + + setSize({ width: size.width, height: size.height }); + }; + const debouncedCalContainerSize = debounce(calContainerSize, 100); + + useEffect(() => { + if (!ref?.current) return; + + const observer = new ResizeObserver(() => { + debouncedCalContainerSize(); + }); + observer.observe(ref.current); + + return () => { + observer.disconnect(); + }; + }, [ref]); + + return ( +
+ {size?.width && size?.width > 1024 && ( + + )} + + {Math.floor(recordingTime / 60)}: + {String(recordingTime % 60).padStart(2, "0")} + +
+ + + +
+
+ ); +}; diff --git a/enjoy/src/renderer/components/medias/media-bottom-panel/media-waveform.tsx b/enjoy/src/renderer/components/medias/media-bottom-panel/media-waveform.tsx index 70a55999..42e3c180 100644 --- a/enjoy/src/renderer/components/medias/media-bottom-panel/media-waveform.tsx +++ b/enjoy/src/renderer/components/medias/media-bottom-panel/media-waveform.tsx @@ -269,6 +269,7 @@ export const MediaWaveform = () => { variant={`${action.active ? "secondary" : "ghost"}`} data-tooltip-id="media-shadow-tooltip" data-tooltip-content={action.label} + data-tooltip-place="left" className="relative p-0 w-full h-full rounded-none" onClick={action.onClick} > @@ -284,6 +285,7 @@ export const MediaWaveform = () => { size="icon" data-tooltip-id="media-shadow-tooltip" data-tooltip-content={t("more")} + data-tooltip-place="left" className="relative p-0 w-full h-full rounded-none" > diff --git a/enjoy/src/renderer/components/medias/media-left-panel/media-left-panel.tsx b/enjoy/src/renderer/components/medias/media-left-panel/media-left-panel.tsx index 5f59525d..de068fd9 100644 --- a/enjoy/src/renderer/components/medias/media-left-panel/media-left-panel.tsx +++ b/enjoy/src/renderer/components/medias/media-left-panel/media-left-panel.tsx @@ -7,6 +7,7 @@ import { MediaRecordings, } from "@renderer/components"; import { + Button, ScrollArea, Tabs, TabsContent, @@ -14,9 +15,15 @@ import { TabsTrigger, } from "@renderer/components/ui"; import { t } from "i18next"; +import { cn } from "@renderer/lib/utils"; +import { ArrowLeftRightIcon } from "lucide-react"; -export const MediaLeftPanel = () => { - const { media, decoded } = useContext(MediaShadowProviderContext); +export const MediaLeftPanel = (props: { + className?: string; + setDisplayPanel?: (displayPanel: "left" | "right" | null) => void; +}) => { + const { className, setDisplayPanel } = props; + const { media, decoded, layout } = useContext(MediaShadowProviderContext); const [tab, setTab] = useState("provider"); useEffect(() => { @@ -28,36 +35,53 @@ export const MediaLeftPanel = () => { if (!media) return null; return ( - - - {media?.mediaType === "Video" && ( + +
+ {layout === "compact" && ( + + )} + + + {media?.mediaType === "Video" && ( + + {t("player")} + + )} - {t("player")} + {t("transcription")} - )} - - {t("transcription")} - - - {t("myRecordings")} - - - {t("mediaInfo")} - - + + {t("myRecordings")} + + + {t("mediaInfo")} + + +
diff --git a/enjoy/src/renderer/components/medias/media-left-panel/media-provider.tsx b/enjoy/src/renderer/components/medias/media-left-panel/media-provider.tsx index a49f40d3..7c8f03b1 100644 --- a/enjoy/src/renderer/components/medias/media-left-panel/media-provider.tsx +++ b/enjoy/src/renderer/components/medias/media-left-panel/media-provider.tsx @@ -20,8 +20,10 @@ import { import { TimelineEntry } from "echogarden/dist/utilities/Timeline.d.js"; import { milisecondsToTimestamp } from "@/utils"; import { toast } from "@renderer/components/ui"; +import { cn } from "@renderer/lib/utils"; -export const MediaProvider = () => { +export const MediaProvider = (props: { className?: string }) => { + const { className } = props; const { theme } = useContext(ThemeProviderContext); const { media, setMediaProvider, setDecodeError, transcription } = useContext( MediaShadowProviderContext @@ -63,7 +65,7 @@ export const MediaProvider = () => { if (!media?.src) return null; return ( -
+
{ transcribingProgress, transcribingOutput, generateTranscription, + onCancel, } = useContext(MediaShadowProviderContext); if (decoded) { // Decoded and transcription created but not ready @@ -109,7 +110,12 @@ const LoadingContent = () => {
- @@ -132,7 +138,12 @@ const LoadingContent = () => { )}
- diff --git a/enjoy/src/renderer/components/medias/media-right-panel/media-caption-actions.tsx b/enjoy/src/renderer/components/medias/media-right-panel/media-caption-actions.tsx index 0ce0f68d..706cdde2 100644 --- a/enjoy/src/renderer/components/medias/media-right-panel/media-caption-actions.tsx +++ b/enjoy/src/renderer/components/medias/media-right-panel/media-caption-actions.tsx @@ -173,6 +173,7 @@ export const MediaCaptionActions = (props: { className="rounded-full w-8 h-8 p-0" data-tooltip-id="media-shadow-tooltip" data-tooltip-content={t("displayIpa")} + data-tooltip-place="left" onClick={() => setDisplayIpa(!displayIpa)} > @@ -184,6 +185,7 @@ export const MediaCaptionActions = (props: { className="rounded-full w-8 h-8 p-0" data-tooltip-id="media-shadow-tooltip" data-tooltip-content={t("displayNotes")} + data-tooltip-place="left" onClick={() => setDisplayNotes(!displayNotes)} > @@ -195,6 +197,7 @@ export const MediaCaptionActions = (props: { + )} + + + {t("captionTabs.translation")} + + + {t("captionTabs.note")} + + + {t("captionTabs.analysis")} + + +
{ return ( @@ -18,15 +23,7 @@ export const MediaShadowPlayer = () => { direction="vertical" > - - - - - - - - - + @@ -38,3 +35,37 @@ export const MediaShadowPlayer = () => { ); }; + +const TopPanel = () => { + const { layout } = useContext(MediaShadowProviderContext); + const [displayPanel, setDisplayPanel] = useState<"left" | "right" | null>( + "right" + ); + + if (layout === "normal") { + return ( + + + + + + + + + + ); + } + + return ( +
+ + +
+ ); +}; diff --git a/enjoy/src/renderer/components/chats/chat-gpt-form.tsx b/enjoy/src/renderer/components/misc/gpt-form.tsx similarity index 98% rename from enjoy/src/renderer/components/chats/chat-gpt-form.tsx rename to enjoy/src/renderer/components/misc/gpt-form.tsx index c5357cf7..d575c9d7 100644 --- a/enjoy/src/renderer/components/chats/chat-gpt-form.tsx +++ b/enjoy/src/renderer/components/misc/gpt-form.tsx @@ -18,7 +18,7 @@ import { t } from "i18next"; import { useContext } from "react"; import { AISettingsProviderContext } from "@renderer/context"; -export const ChatGPTForm = (props: { form: ReturnType }) => { +export const GPTForm = (props: { form: ReturnType }) => { const { form } = props; const { gptProviders } = useContext(AISettingsProviderContext); diff --git a/enjoy/src/renderer/components/misc/index.ts b/enjoy/src/renderer/components/misc/index.ts index 19388a84..cd8a0e4d 100644 --- a/enjoy/src/renderer/components/misc/index.ts +++ b/enjoy/src/renderer/components/misc/index.ts @@ -12,3 +12,5 @@ export * from "./page-placeholder"; export * from "./universal-player"; export * from "./sidebar"; export * from "./wavesurfer-player"; +export * from "./tts-form"; +export * from "./gpt-form"; diff --git a/enjoy/src/renderer/components/misc/markdown-wrapper.tsx b/enjoy/src/renderer/components/misc/markdown-wrapper.tsx index f2fba3f7..da4a1503 100644 --- a/enjoy/src/renderer/components/misc/markdown-wrapper.tsx +++ b/enjoy/src/renderer/components/misc/markdown-wrapper.tsx @@ -1,14 +1,34 @@ -import Markdown from "react-markdown"; +import Markdown, { defaultUrlTransform } from "react-markdown"; import { visitParents } from "unist-util-visit-parents"; import { Sentence } from "@renderer/components"; import { cn } from "@renderer/lib/utils"; +import remarkGfm from "remark-gfm"; +import { + useCallback, + useEffect, + useState, + useMemo, + memo, + useContext, +} from "react"; +import { + LanguagesIcon, + LoaderIcon, + PlayIcon, + RefreshCwIcon, +} from "lucide-react"; +import { Button, toast } from "@renderer/components/ui"; +import { useIntersectionObserver } from "@uidotdev/usehooks"; +import { md5 } from "js-md5"; +import { AppSettingsProviderContext } from "@/renderer/context"; +import { useAiCommand } from "@/renderer/hooks"; function rehypeWrapText() { return function wrapTextTransform(tree: any) { visitParents(tree, "text", (node, ancestors) => { const parent = ancestors.at(-1); - if (parent.tagName !== "vocabulary") { + if (parent.tagName !== "vocabulary" && parent.tagName !== "a") { node.type = "element"; node.tagName = "vocabulary"; node.properties = { text: node.value }; @@ -18,35 +38,244 @@ function rehypeWrapText() { }; } -export const MarkdownWrapper = ({ - children, - className, - ...props -}: { - children: string; - className?: string; -}) => { - return ( - void; + onSpeech?: (id: string) => void; + autoTranslate?: boolean; + translatable?: boolean; + section?: number; + }) => { + const { EnjoyApp } = useContext(AppSettingsProviderContext); + const { translate } = useAiCommand(); + const [translating, setTranslating] = useState(false); + const [translation, setTranslation] = useState(""); + + const [ref, entry] = useIntersectionObserver({ + threshold: 0, + root: null, + rootMargin: "0px", + }); + + const toggleTranslation = () => { + if (translation) { + setTranslation(""); + } else { + handleTranslate(); + } + }; + + const handleTranslate = async (force = false) => { + if (translating) return; + + const content = entry.target + ?.querySelector(".segment-content") + ?.textContent?.trim(); + if (!content) return; + + const md5Hash = md5(content); + + const cacheKey = `translate-${md5Hash}`; + const cached = await EnjoyApp.cacheObjects.get(cacheKey); + if (cached && !force) { + setTranslation(cached); + } else { + setTranslating(true); + setTranslation(""); + translate(content, cacheKey) + .then((result) => { + setTranslation(result); + }) + .catch((error) => { + toast.error(error.message); + }) + .finally(() => { + setTranslating(false); + }); + } + }; + + const content = useMemo(() => { + if (!entry?.target) return ""; + return entry.target?.textContent?.trim(); + }, [entry]); + + const id = `segment-${index}`; + + useEffect(() => { + if (!onSegmentVisible) return; + if (entry?.isIntersecting) { + onSegmentVisible(`segment-${index}`); + if (autoTranslate) { + handleTranslate(); + } + } + }, [entry?.isIntersecting, autoTranslate]); + + return ( + <> + + + {content && (onSpeech || translatable) && ( + + #{index + 1} + + )} + {onSpeech && content && ( + + )} + {translatable && content && ( + + )} + + {children} + + {translation && ( + + {translation} + + + )} + + ); + } +); + +// Wrap MarkdownWrapper with React.memo and memoize callbacks +export const MarkdownWrapper = memo( + ({ + children, + className, + onLinkClick, + onSpeech, + onSegmentVisible, + translatable = false, + autoTranslate, + section, + ...props + }: { + children: string; + className?: string; + onLinkClick?: (e: React.MouseEvent) => void; + onSpeech?: (id: string) => void; + onSegmentVisible?: (id: string) => void; + translatable?: boolean; + autoTranslate?: boolean; + section?: number; + }) => { + // Memoize component callbacks + const handleLinkClick = useCallback(onLinkClick, [onLinkClick]); + + const components = useMemo(() => { + let segmentIndex = 0; + const HEADER_COMPONENTS = ["h1", "h2", "h3", "h4", "h5", "p"] as const; + + const headerComponents = Object.fromEntries( + HEADER_COMPONENTS.map((tag) => [ + tag, + ({ node, children, ...props }: any) => ( + + {children} + + ), + ]) + ); + + return { + a({ node, children, ...props }: any) { try { new URL(props.href ?? ""); props.target = "_blank"; props.rel = "noopener noreferrer"; } catch (e) {} - return {children}; + return ( + + {children} + + ); }, - vocabulary({ node, children, ...props }) { + vocabulary({ node, children, ...props }: any) { return ; }, - }} - {...props} - > - {children} - - ); -}; + ...headerComponents, + }; + }, [handleLinkClick, onSegmentVisible, onSpeech, autoTranslate, section]); + + return ( + { + if (url.startsWith("blob:") || url.startsWith("data:")) { + return url; + } + return defaultUrlTransform(url); + }} + components={components} + {...props} + > + {children} + + ); + } +); + +MarkdownWrapper.displayName = "MarkdownWrapper"; diff --git a/enjoy/src/renderer/components/misc/sidebar.tsx b/enjoy/src/renderer/components/misc/sidebar.tsx index b753d9f3..dd3c0e83 100644 --- a/enjoy/src/renderer/components/misc/sidebar.tsx +++ b/enjoy/src/renderer/components/misc/sidebar.tsx @@ -155,10 +155,10 @@ export const Sidebar = () => { /> diff --git a/enjoy/src/renderer/components/chats/chat-tts-form.tsx b/enjoy/src/renderer/components/misc/tts-form.tsx similarity index 98% rename from enjoy/src/renderer/components/chats/chat-tts-form.tsx rename to enjoy/src/renderer/components/misc/tts-form.tsx index 3216d734..9e5f320a 100644 --- a/enjoy/src/renderer/components/chats/chat-tts-form.tsx +++ b/enjoy/src/renderer/components/misc/tts-form.tsx @@ -16,7 +16,7 @@ import { t } from "i18next"; import { useContext } from "react"; import { AISettingsProviderContext } from "@renderer/context"; -export const ChatTTSForm = (props: { form: ReturnType }) => { +export const TTSForm = (props: { form: ReturnType }) => { const { form } = props; const { ttsProviders } = useContext(AISettingsProviderContext); diff --git a/enjoy/src/renderer/components/misc/wavesurfer-player.tsx b/enjoy/src/renderer/components/misc/wavesurfer-player.tsx index 25f751a4..49383570 100644 --- a/enjoy/src/renderer/components/misc/wavesurfer-player.tsx +++ b/enjoy/src/renderer/components/misc/wavesurfer-player.tsx @@ -22,6 +22,7 @@ export const WavesurferPlayer = (props: { pitchContourOptions?: any; className?: string; autoplay?: boolean; + onEnded?: () => void; }) => { const { id, @@ -33,6 +34,7 @@ export const WavesurferPlayer = (props: { pitchContourOptions, className = "", autoplay = false, + onEnded, } = props; const [initialized, setInitialized] = useState(false); const [isPlaying, setIsPlaying] = useState(false); @@ -95,6 +97,9 @@ export const WavesurferPlayer = (props: { wavesurfer.on("pause", () => { setIsPlaying(false); }), + wavesurfer.on("finish", () => { + onEnded && onEnded(); + }), wavesurfer.on("timeupdate", (time: number) => { setCurrentTime(time); onSetCurrentTime && onSetCurrentTime(time); diff --git a/enjoy/src/renderer/components/stories/story-viewer.tsx b/enjoy/src/renderer/components/stories/story-viewer.tsx index 08b61b69..7e33751b 100644 --- a/enjoy/src/renderer/components/stories/story-viewer.tsx +++ b/enjoy/src/renderer/components/stories/story-viewer.tsx @@ -5,7 +5,7 @@ import { ChevronLeftIcon, ExternalLinkIcon } from "lucide-react"; import { Button } from "@renderer/components/ui"; import uniq from "lodash/uniq"; import Mark from "mark.js"; -import { Vocabulary } from "@/renderer/components"; +import { Vocabulary } from "@renderer/components"; export const StoryViewer = (props: { story: Partial & Partial; diff --git a/enjoy/src/renderer/components/videos/videos-segment.tsx b/enjoy/src/renderer/components/videos/videos-segment.tsx index b42675e2..3fab714c 100644 --- a/enjoy/src/renderer/components/videos/videos-segment.tsx +++ b/enjoy/src/renderer/components/videos/videos-segment.tsx @@ -62,7 +62,7 @@ export const VideosSegment = (props: { limit?: number }) => { {videos.length === 0 ? (
- +
) : ( diff --git a/enjoy/src/renderer/components/widgets/sentence.tsx b/enjoy/src/renderer/components/widgets/sentence.tsx index a55c9c1f..521042ab 100644 --- a/enjoy/src/renderer/components/widgets/sentence.tsx +++ b/enjoy/src/renderer/components/widgets/sentence.tsx @@ -1,30 +1,29 @@ +import { memo } from "react"; import { Vocabulary } from "@renderer/components"; import { cn } from "@renderer/lib/utils"; -export const Sentence = ({ - sentence, - className, -}: { - sentence: string; - className?: string; -}) => { - // split by space or punctuation - // Sentence may be in other languages, so need to handle only English words - let words = sentence.split(/(\s+|[a-zA-Z]+)/); +export const Sentence = memo( + ({ sentence, className }: { sentence: string; className?: string }) => { + // split by space or punctuation + // Sentence may be in other languages, so need to handle only English words + let words = sentence.split(/(\s+|[a-zA-Z]+)/); - return ( - - {words.map((word, index) => { - return ( - - {word.match(/[a-zA-Z]+/) ? ( - - ) : ( - word - )} - - ); - })} - - ); -}; + return ( + + {words.map((word, index) => { + return ( + + {word.match(/[a-zA-Z]+/) ? ( + + ) : ( + word + )} + + ); + })} + + ); + } +); + +Sentence.displayName = "Sentence"; diff --git a/enjoy/src/renderer/components/widgets/vocabulary.tsx b/enjoy/src/renderer/components/widgets/vocabulary.tsx index d8f269a4..e225f41f 100644 --- a/enjoy/src/renderer/components/widgets/vocabulary.tsx +++ b/enjoy/src/renderer/components/widgets/vocabulary.tsx @@ -1,33 +1,22 @@ -import React, { useContext, useState } from "react"; +import { useContext, useState, memo } from "react"; import { AppSettingsProviderContext } from "@renderer/context"; -export const Vocabulary = ({ - word, - context, - children, -}: { - word: string; - context?: string; - children?: React.ReactNode; -}) => { - let [timer, setTimer] = useState>(); - const { vocabularyConfig, EnjoyApp } = useContext(AppSettingsProviderContext); +export const Vocabulary = memo( + ({ + word, + context, + children, + }: { + word: string; + context?: string; + children?: React.ReactNode; + }) => { + let [timer, setTimer] = useState>(); + const { vocabularyConfig, EnjoyApp } = useContext( + AppSettingsProviderContext + ); - const handleLookup = (e: any) => { - if (!context) { - context = e.target?.parentElement - .closest(".sentence, h2, p, div") - ?.textContent?.trim(); - } - - const { x, bottom: y } = e.target.getBoundingClientRect(); - const _word = word.replace(/[^\w\s]|_/g, ""); - - EnjoyApp.lookup(_word, context, { x, y }); - }; - - const handleMouseEnter = (e: any) => { - let _timer = setTimeout(() => { + const handleLookup = (e: any) => { if (!context) { context = e.target?.parentElement .closest(".sentence, h2, p, div") @@ -38,23 +27,40 @@ export const Vocabulary = ({ const _word = word.replace(/[^\w\s]|_/g, ""); EnjoyApp.lookup(_word, context, { x, y }); - }, 800); + }; - setTimer(_timer); - }; + const handleMouseEnter = (e: any) => { + let _timer = setTimeout(() => { + if (!context) { + context = e.target?.parentElement + .closest(".sentence, h2, p, div") + ?.textContent?.trim(); + } - const handleMouseLeave = () => { - clearTimeout(timer); - }; + const { x, bottom: y } = e.target.getBoundingClientRect(); + const _word = word.replace(/[^\w\s]|_/g, ""); - return vocabularyConfig.lookupOnMouseOver ? ( - - {word || children} - - ) : ( - {word || children} - ); -}; + EnjoyApp.lookup(_word, context, { x, y }); + }, 800); + + setTimer(_timer); + }; + + const handleMouseLeave = () => { + clearTimeout(timer); + }; + + return vocabularyConfig.lookupOnMouseOver ? ( + + {word || children} + + ) : ( + {word || children} + ); + } +); + +Vocabulary.displayName = "Vocabulary"; diff --git a/enjoy/src/renderer/context/document-provider.tsx b/enjoy/src/renderer/context/document-provider.tsx new file mode 100644 index 00000000..b4b7fab8 --- /dev/null +++ b/enjoy/src/renderer/context/document-provider.tsx @@ -0,0 +1,264 @@ +import { + createContext, + useCallback, + useContext, + useEffect, + useRef, + useState, +} from "react"; +import { + AppSettingsProviderContext, + DbProviderContext, + MediaShadowProvider, +} from "@renderer/context"; +import { toast } from "@renderer/components/ui"; +import debounce from "lodash/debounce"; +import { useSpeech } from "@renderer/hooks"; + +type DocumentProviderProps = { + ref: React.RefObject; + document: DocumentEType; + playingSegmentId: string | null; + playingSegment: { + id: string; + index: number; + text: string; + } | null; + nextSegment: { + id: string; + index: number; + text: string; + } | null; + togglePlayingSegment: (segment: string | null) => void; + section: number; + setSection: (section: number) => void; + onSpeech: (segment: string) => void; + onSegmentVisible: (id: string) => void; + locateSegment: (id: string) => HTMLElement | null; + content: string; + setContent: (content: string) => void; +}; + +export const DocumentProviderContext = createContext({ + ref: null, + document: null, + playingSegmentId: null, + playingSegment: null, + nextSegment: null, + togglePlayingSegment: () => {}, + section: 0, + setSection: () => {}, + onSpeech: () => {}, + onSegmentVisible: () => {}, + locateSegment: () => null, + content: "", + setContent: () => {}, +}); + +export function DocumentProvider({ + documentId, + children, +}: { + documentId: string; + children: React.ReactNode; +}) { + const { EnjoyApp } = useContext(AppSettingsProviderContext); + const { addDblistener, removeDbListener } = useContext(DbProviderContext); + + const { tts } = useSpeech(); + + const [document, setDocument] = useState(null); + const [section, setSection] = useState(0); + const [playingSegmentId, setPlayingSegmentId] = useState(null); + const [playingSegment, setPlayingSegment] = useState<{ + id: string; + index: number; + text: string; + } | null>(null); + const [nextSegment, setNextSegment] = useState<{ + id: string; + index: number; + text: string; + } | null>(null); + const [content, setContent] = useState(); + + const ref = useRef(null); + + const locateSegment = (id: string) => { + return ref.current?.querySelector(`#${id}`) as HTMLElement | null; + }; + + const findNextSegment = async (index: number) => { + if (!document.config.autoNextSpeech) return; + + const next: HTMLElement | null = ref.current?.querySelector( + `[data-index="${index}"]` + ); + if (!next) return; + + const text = next.querySelector(".segment-content")?.textContent?.trim(); + if (!text) { + return findNextSegment(index + 1); + } + + const existingSpeech = await EnjoyApp.speeches.findOne({ + sourceId: document.id, + sourceType: "Document", + section, + segment: index, + }); + + if (!existingSpeech) { + tts({ + sourceId: document.id, + sourceType: "Document", + section, + segment: index, + text, + configuration: document.config.tts, + }); + } + setNextSegment({ + id: next.id, + index, + text, + }); + }; + + const onSegmentVisible = useCallback( + (id: string) => { + updateDocumentPosition(id); + }, + [document] + ); + + const updateDocumentPosition = debounce((id: string) => { + if (!id) return; + + const segment = locateSegment(id); + if (!segment) return; + + const index = segment.dataset.index || "0"; + const sectionIndex = segment.dataset.section || "0"; + + EnjoyApp.documents.update(document.id, { + lastReadPosition: { + section: parseInt(sectionIndex), + segment: parseInt(index), + }, + lastReadAt: new Date(), + }); + }, 1000); + + const togglePlayingSegment = useCallback((segment: string | null) => { + setPlayingSegmentId((prev) => (prev === segment ? null : segment)); + }, []); + + const onSpeech = useCallback( + (segment: string) => { + togglePlayingSegment(segment); + }, + [togglePlayingSegment] + ); + + const fetchDocument = async () => { + if (!documentId) return; + + EnjoyApp.documents + .findOne({ id: documentId }) + .then((doc) => { + setDocument(doc); + }) + .catch((err) => { + toast.error(err.message); + }); + }; + + const handleDocumentUpdate = (event: CustomEvent) => { + const { action, record } = event.detail; + if (action === "update" && record.id === documentId) { + setDocument(record as DocumentEType); + } + }; + + useEffect(() => { + if (!ref.current) return; + if (!playingSegmentId) return; + + const element = locateSegment(playingSegmentId); + if (!element) return; + + const index = parseInt(element.dataset.index || "0"); + findNextSegment(index + 1); + setPlayingSegment({ + id: element.id, + index, + text: element.querySelector(".segment-content")?.textContent?.trim(), + }); + + element.scrollIntoView({ behavior: "smooth", block: "center" }); + element.classList.add("playing-segment", "bg-yellow-100"); + + return () => { + setPlayingSegment(null); + element?.classList?.remove("playing-segment", "bg-yellow-100"); + }; + }, [ref, playingSegmentId]); + + // auto scroll to the top when new section is rendered + useEffect(() => { + if (!content) return; + if (!ref?.current) return; + + if (document.lastReadPosition.section === section) { + const element = locateSegment( + `segment-${document.lastReadPosition.segment || 0}` + ); + if (element) { + element.scrollIntoView({ behavior: "smooth", block: "center" }); + } + } + }, [section, content]); + + useEffect(() => { + if (!document) return; + setSection(document.lastReadPosition.section || 0); + }, [document]); + + useEffect(() => { + fetchDocument(); + addDblistener(handleDocumentUpdate); + + return () => { + removeDbListener(handleDocumentUpdate); + setDocument(null); + }; + }, [documentId]); + + return ( + + togglePlayingSegment(null)} + > +
{children}
+ + + ); +} diff --git a/enjoy/src/renderer/context/index.ts b/enjoy/src/renderer/context/index.ts index b53efbd4..33de7400 100644 --- a/enjoy/src/renderer/context/index.ts +++ b/enjoy/src/renderer/context/index.ts @@ -8,3 +8,4 @@ export * from "./hotkeys-settings-provider"; export * from "./media-shadow-provider"; export * from "./theme-provider"; export * from "./dict-provider"; +export * from "./document-provider"; diff --git a/enjoy/src/renderer/context/media-shadow-provider.tsx b/enjoy/src/renderer/context/media-shadow-provider.tsx index b1e291d5..6f7ec69e 100644 --- a/enjoy/src/renderer/context/media-shadow-provider.tsx +++ b/enjoy/src/renderer/context/media-shadow-provider.tsx @@ -23,6 +23,8 @@ const ONE_MINUTE = 60; const TEN_MINUTES = 10 * ONE_MINUTE; type MediaShadowContextType = { + layout: "compact" | "normal"; + onCancel?: () => void; media: AudioType | VideoType; setMedia: (media: AudioType | VideoType) => void; setMediaProvider: (mediaProvider: HTMLAudioElement | null) => void; @@ -103,8 +105,12 @@ export const MediaShadowProviderContext = export const MediaShadowProvider = ({ children, + layout = "normal", + onCancel, }: { children: React.ReactNode; + layout?: "compact" | "normal"; + onCancel?: () => void; }) => { const minPxPerSec = 150; const { EnjoyApp, learningLanguage, recorderConfig } = useContext( @@ -636,6 +642,8 @@ export const MediaShadowProvider = ({ <> { const { EnjoyApp, webApi, nativeLanguage, learningLanguage } = useContext( @@ -99,17 +100,48 @@ export const useAiCommand = () => { text: string, cacheKey?: string ): Promise => { - return translateCommand(text, nativeLanguage, { - key: currentGptEngine.key, - modelName: - currentGptEngine.models.translate || currentGptEngine.models.default, - baseUrl: currentGptEngine.baseUrl, - }).then((res) => { - if (cacheKey) { - EnjoyApp.cacheObjects.set(cacheKey, res); + let translatedContent = ""; + const md5 = md5Hash(text.trim()); + const engine = currentGptEngine.key; + const modelName = + currentGptEngine.models.translate || currentGptEngine.models.default; + + try { + const res = await webApi.translations({ + md5, + translatedLanguage: nativeLanguage, + engine: modelName, + }); + + if (res.translations.length > 0) { + translatedContent = res.translations[0].translatedContent; } - return res; - }); + } catch (error) { + console.error(error); + } + + if (!translatedContent) { + translatedContent = await translateCommand(text, nativeLanguage, { + key: engine, + modelName, + baseUrl: currentGptEngine.baseUrl, + }); + + webApi.createTranslation({ + md5, + content: text, + translatedContent, + language: learningLanguage, + translatedLanguage: nativeLanguage, + engine: modelName, + }); + } + + if (cacheKey) { + EnjoyApp.cacheObjects.set(cacheKey, translatedContent); + } + + return translatedContent; }; const analyzeText = async (text: string, cacheKey?: string) => { diff --git a/enjoy/src/renderer/hooks/use-speech.tsx b/enjoy/src/renderer/hooks/use-speech.tsx index cc9ed225..ab3f4c06 100644 --- a/enjoy/src/renderer/hooks/use-speech.tsx +++ b/enjoy/src/renderer/hooks/use-speech.tsx @@ -29,6 +29,8 @@ export const useSpeech = () => { text: params.text, sourceType: params.sourceType, sourceId: params.sourceId, + section: params.section, + segment: params.segment, configuration: { engine, model, diff --git a/enjoy/src/renderer/lib/utils.ts b/enjoy/src/renderer/lib/utils.ts index 8de463ea..b66e1778 100644 --- a/enjoy/src/renderer/lib/utils.ts +++ b/enjoy/src/renderer/lib/utils.ts @@ -156,3 +156,12 @@ export function imgErrorToDefalut( target.onerror = null; target.src = "assets/default-img.jpg"; } + +export function blobToDataUrl(blob: Blob) { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onloadend = () => resolve(reader.result as string); + reader.onerror = reject; + reader.readAsDataURL(blob); + }); +} diff --git a/enjoy/src/renderer/pages/document.tsx b/enjoy/src/renderer/pages/document.tsx new file mode 100644 index 00000000..c80a2432 --- /dev/null +++ b/enjoy/src/renderer/pages/document.tsx @@ -0,0 +1,78 @@ +import { + Button, + ResizableHandle, + ResizablePanel, + ResizablePanelGroup, + ScrollArea, +} from "@renderer/components/ui"; +import { + DocumentHtmlRenderer, + DocumentEpubRenderer, + DocumentPlayer, + LoaderSpin, + DocumentTextRenderer, +} from "@renderer/components"; +import { useContext } from "react"; +import { useNavigate, useParams } from "react-router-dom"; +import { DocumentProvider, DocumentProviderContext } from "@renderer/context"; +import { ChevronLeftIcon } from "lucide-react"; + +export default () => { + const { id } = useParams<{ id: string }>(); + + return ( + + + + ); +}; + +const DocumentComponent = () => { + const { document, playingSegment } = useContext(DocumentProviderContext); + const navigate = useNavigate(); + + if (!document) { + return ( +
+ +
+ ); + } + + return ( +
+
+ + {document.title} +
+ + + + + {document.metadata.extension === "html" && } + {document.metadata.extension === "epub" && } + {["txt", "md", "markdown"].includes( + document.metadata.extension + ) && } + + + + + + + +
+ ); +}; diff --git a/enjoy/src/renderer/pages/documents.tsx b/enjoy/src/renderer/pages/documents.tsx new file mode 100644 index 00000000..024890d4 --- /dev/null +++ b/enjoy/src/renderer/pages/documents.tsx @@ -0,0 +1,67 @@ +import { Button, Input } from "@renderer/components/ui"; +import { + DocumentAddButton, + DocumentCard, + LoaderSpin, +} from "@renderer/components"; +import { useState, useContext, useEffect } from "react"; +import { AppSettingsProviderContext } from "@renderer/context"; +import { t } from "i18next"; +import { ChevronLeftIcon } from "lucide-react"; +import { useNavigate } from "react-router-dom"; +import { useDebounce } from "@uidotdev/usehooks"; + +export default () => { + const navigate = useNavigate(); + + const [documents, setDocuments] = useState([]); + const [query, setQuery] = useState(""); + const [loading, setLoading] = useState(true); + const { EnjoyApp } = useContext(AppSettingsProviderContext); + const debouncedQuery = useDebounce(query, 500); + + const fetchDocuments = () => { + setLoading(true); + EnjoyApp.documents + .findAll({ query: debouncedQuery }) + .then((documents) => { + setDocuments(documents); + }) + .finally(() => { + setLoading(false); + }); + }; + + useEffect(() => { + fetchDocuments(); + }, [debouncedQuery]); + + return ( +
+
+ + {t("sidebar.documents")} +
+
+ setQuery(e.target.value)} + /> + +
+ + {loading ? ( + + ) : ( +
+ {documents.map((document) => ( + + ))} +
+ )} +
+ ); +}; diff --git a/enjoy/src/renderer/pages/home.tsx b/enjoy/src/renderer/pages/home.tsx index e85d69f0..a333732d 100644 --- a/enjoy/src/renderer/pages/home.tsx +++ b/enjoy/src/renderer/pages/home.tsx @@ -1,7 +1,7 @@ import { AudiosSegment, AudibleBooksSegment, - StoriesSegment, + DocumentsSegment, VideosSegment, YoutubeVideosSegment, EnrollmentSegment, @@ -41,7 +41,7 @@ export default () => { - + {channels.map((channel) => ( diff --git a/enjoy/src/renderer/router.tsx b/enjoy/src/renderer/router.tsx index a63b38d3..c2ae0da2 100644 --- a/enjoy/src/renderer/router.tsx +++ b/enjoy/src/renderer/router.tsx @@ -11,6 +11,8 @@ import Audios from "./pages/audios"; import Videos from "./pages/videos"; import Stories from "./pages/stories"; import Story from "./pages/story"; +import Documents from "./pages/documents"; +import Document from "./pages/document"; import Books from "./pages/books"; import Profile from "./pages/profile"; import User from "./pages/user"; @@ -96,6 +98,14 @@ export default createHashRouter([ path: "/videos/:id", element: