Feat: make notes on caption (#544)

* add segment model

* add note model

* db handle segment & note

* add notes & segments handler

* refactor media caption components

* segment & note create

* fix type

* update note column & may sync

* display selected words for note

* refactor selected words

* auto select words when editing note

* refactor

* refactor caption component

* display notes

* refactor notes components

* fix

* refactor segment & notes into context

* destroy note

* update locale

* fix caption switch issue

* fix layout

* refactor caption layout

* remove deprecated code

* may share note

* improve UI

* fix notes list auto update after created

* remove console.log

* add notes page

* refactor note parameters

* refactor components

* mark note on transcription

* handle no notes

* improve style

* improve style

* show context menu on selection text

* fix utils
This commit is contained in:
an-lee
2024-04-26 15:05:36 +08:00
committed by GitHub
parent 5740b2635c
commit 0644c3bbd7
58 changed files with 2586 additions and 677 deletions

View File

@@ -166,7 +166,7 @@ export class Client {
page?: number;
items?: number;
userId?: string;
type?: "all" | "recording" | "medium" | "story" | "prompt" | "text" | "gpt";
type?: "all" | "recording" | "medium" | "story" | "prompt" | "text" | "gpt" | "note";
by?: "following" | "all";
}): Promise<
{
@@ -232,6 +232,18 @@ export class Client {
return this.api.post("/api/transcriptions", decamelizeKeys(transcription));
}
syncSegment(segment: Partial<Omit<SegmentType, "audio" | "video">>) {
return this.api.post("/api/segments", decamelizeKeys(segment));
}
syncNote(note: Partial<Omit<NoteType, "segment">>) {
return this.api.post("/api/notes", decamelizeKeys(note));
}
deleteNote(id: string) {
return this.api.delete(`/api/notes/${id}`);
}
syncRecording(recording: Partial<RecordingType>) {
if (!recording) return;

View File

@@ -141,7 +141,8 @@
"translator": "Translator",
"mine": "Mine",
"preferences": "Preferences",
"profile": "My Profile"
"profile": "My Profile",
"notes": "Note"
},
"form": {
"lengthMustBeAtLeast": "{{field}} must be at least {{length}} characters",
@@ -166,6 +167,7 @@
"autoScroll": "auto scroll",
"translate": "translate",
"displayIpa": "display IPA",
"displayNotes": "display Notes",
"detail": "detail",
"remove": "remove",
"share": "share",
@@ -470,6 +472,9 @@
"shareGpt": "Share GPT",
"sharedGpt": "Shared a GPT",
"areYouSureToShareThisGptToCommunity": "Are you sure to share this GPT to community?",
"shareNote": "Share note",
"sharedNote": "Shared a note",
"areYouSureToShareThisNoteToCommunity": "Are you sure to share this note to community?",
"saveAiAssistant": "Save this AI assistant",
"addToLibary": "Add to library",
"areYouSureToAddThisVideoToYourLibrary": "Are you sure to add this video to library?",
@@ -550,8 +555,17 @@
"storyType": "Story",
"promptType": "Prompt",
"gptType": "GPT",
"noteType": "Note",
"follow": "follow",
"unfollow": "unfollow",
"noFollowersYet": "No followers yet",
"notFollowingAnyoneYet": "Not following anyone yet"
"notFollowingAnyoneYet": "Not following anyone yet",
"startToNote": "Start to note",
"newNote": "New note",
"writeNoteHere": "Write your note here",
"deleteNote": "Delete note",
"areYouSureToDeleteThisNote": "Are you sure to delete this note?",
"notesCount": "{{count}} notes",
"source": "source",
"noNotesYet": "No notes yet"
}

View File

@@ -141,7 +141,8 @@
"translator": "翻译助手",
"mine": "我的",
"preferences": "软件设置",
"profile": "个人主页"
"profile": "个人主页",
"notes": "笔记"
},
"form": {
"lengthMustBeAtLeast": "{{field}} 长度不可超过 {{length}} 个字符",
@@ -166,6 +167,7 @@
"autoScroll": "自动滚动",
"translate": "翻译",
"displayIpa": "标注音标",
"displayNotes": "显示笔记",
"detail": "详情",
"remove": "删除",
"share": "分享",
@@ -449,7 +451,7 @@
"square": "广场",
"noOneSharedYet": "还没有人分享",
"sharedSuccessfully": "分享成功",
"sharedFailed": "分享失败",
"shareFailed": "分享失败",
"shareAudio": "分享音频",
"sharedAudio": "分享了一个音频材料",
"areYouSureToShareThisAudioToCommunity": "您确定要分享此音频到社区吗?",
@@ -469,6 +471,9 @@
"shareGpt": "分享智能助手",
"sharedGpt": "分享了一个智能助手",
"areYouSureToShareThisGptToCommunity": "您确定要将这个智能助手分享到社区吗?",
"shareNote": "分享笔记",
"sharedNote": "分享了一条笔记",
"areYouSureToShareThisNoteToCommunity": "您确定要将这条笔记分享到社区吗?",
"saveAiAssistant": "保存智能助手",
"addToLibary": "添加到资源库",
"areYouSureToAddThisVideoToYourLibrary": "您确定要添加此视频到资料库吗?",
@@ -522,7 +527,7 @@
"autoGroup": "智能断句",
"captionTabs": {
"selected": "选词",
"translation": "整句翻译",
"translation": "翻译",
"analysis": "句子分析",
"note": "笔记"
},
@@ -549,8 +554,17 @@
"storyType": "文章",
"promptType": "提示语",
"gptType": "智能助手",
"noteType": "笔记",
"follow": "关注",
"unfollow": "取消关注",
"noFollowersYet": "还没有人关注",
"notFollowingAnyoneYet": "还没有关注任何人"
"notFollowingAnyoneYet": "还没有关注任何人",
"startToNote": "开始做笔记",
"newNote": "新笔记",
"writeNoteHere": "开始做笔记",
"deleteNote": "删除笔记",
"areYouSureToDeleteThisNote": "您确定要删除这条笔记吗?",
"notesCount": "{{count}} 条笔记",
"source": "来源",
"noNotesYet": "还没有笔记"
}

View File

@@ -37,6 +37,7 @@ contextMenu({
showInspectElement: false,
showLookUpSelection: false,
showLearnSpelling: false,
showSelectAll: false,
labels: {
copy: t("copy"),
cut: t("cut"),
@@ -44,7 +45,7 @@ contextMenu({
selectAll: t("selectAll"),
},
shouldShowMenu: (_event, params) => {
return params.isEditable;
return params.isEditable || !!params.selectionText;
},
});
@@ -75,7 +76,7 @@ protocol.registerSchemesAsPrivileged([
app.on("ready", async () => {
protocol.handle("enjoy", (request) => {
let url = request.url.replace("enjoy://", "");
if (url.match(/library\/(audios|videos|recordings|speeches)/g)) {
if (url.match(/library\/(audios|videos|recordings|speeches|segments)/g)) {
url = url.replace("library/", "");
url = path.join(settings.userDataPath(), url);
} else if (url.startsWith("library")) {

View File

@@ -1,8 +1,10 @@
export * from './audios-handler';
export * from './recordings-handler';
export * from './messages-handler';
export * from './conversations-handler';
export * from './cache-objects-handler';
export * from './conversations-handler';
export * from './messages-handler';
export * from './notes-handler';
export * from './recordings-handler';
export * from './speeches-handler';
export * from './segments-handler';
export * from './transcriptions-handler';
export * from './videos-handler';

View File

@@ -0,0 +1,170 @@
import { ipcMain, IpcMainEvent } from "electron";
import { Note, Segment } from "@main/db/models";
import { Sequelize } from "sequelize";
class NotesHandler {
private async groupByTarget(
_event: IpcMainEvent,
params: {
limit?: number;
offset?: number;
}
) {
const { limit, offset } = params;
return Note.findAll({
include: [Segment],
attributes: [
"targetId",
"targetType",
[Sequelize.fn("COUNT", Sequelize.col("note.id")), "count"],
],
group: ["targetId", "targetType"],
order: [["created_at", "DESC"]],
limit,
offset,
}).then((notes) => notes.map((note) => note.toJSON()));
}
private async groupBySegment(
_event: IpcMainEvent,
targetId: string,
targetType: string
) {
return Note.findAll({
include: [
{
model: Segment,
as: "segment",
attributes: ["id", "segmentIndex"],
},
],
attributes: [
"targetId",
"targetType",
[Sequelize.fn("COUNT", Sequelize.col("note.id")), "count"],
],
group: ["targetId", "targetType"],
where: {
"$segment.target_id$": targetId,
"$segment.target_type$": targetType,
},
}).then((notes) => notes.map((note) => note.toJSON()));
}
private async findAll(
_event: IpcMainEvent,
params: {
targetId?: string;
targetType?: string;
limit?: number;
offset?: number;
}
) {
const { targetId, targetType, limit, offset } = params;
const where: any = {};
if (targetId && targetType) {
where["targetId"] = targetId;
where["targetType"] = targetType;
}
const notes = await Note.findAll({
where,
limit: limit,
offset: offset,
include: [Segment],
order: [["createdAt", "DESC"]],
});
return notes.map((note) => note.toJSON());
}
private async find(_event: IpcMainEvent, id: string) {
const note = await Note.findByPk(id);
return note.toJSON();
}
private async update(
_event: IpcMainEvent,
id: string,
params: {
content: string;
parameters: any;
}
) {
const note = await Note.findByPk(id);
if (!note) {
throw new Error("Note not found");
}
await note.update({
content: params.content,
parameters: params.parameters,
});
return note.toJSON();
}
private async delete(_event: IpcMainEvent, id: string) {
const note = await Note.findByPk(id);
if (!note) {
throw new Error("Note not found");
}
note.destroy();
}
private async create(
_event: IpcMainEvent,
params: {
targetId: string;
targetType: string;
content: string;
parameters: any;
}
) {
const { targetId, targetType, content, parameters } = params;
switch (targetType) {
case "Segment":
const segment = await Segment.findByPk(targetId);
if (!segment) {
throw new Error("Segment not found");
}
break;
default:
throw new Error("Invalid target");
}
return Note.create({
targetId,
targetType,
content,
parameters,
});
}
private async sync(_event: IpcMainEvent, id: string) {
const note = await Note.findByPk(id);
if (!note) {
throw new Error("Note not found");
}
await note.sync();
return note.toJSON();
}
register() {
ipcMain.handle("notes-group-by-target", this.groupByTarget);
ipcMain.handle("notes-group-by-segment", this.groupBySegment);
ipcMain.handle("notes-find-all", this.findAll);
ipcMain.handle("notes-find", this.find);
ipcMain.handle("notes-update", this.update);
ipcMain.handle("notes-delete", this.delete);
ipcMain.handle("notes-create", this.create);
ipcMain.handle("notes-sync", this.sync);
}
}
export const notesHandler = new NotesHandler();

View File

@@ -0,0 +1,62 @@
import { ipcMain, IpcMainEvent } from "electron";
import { Audio, Segment, Video } from "@main/db/models";
class SegmentsHandler {
private async find(_event: IpcMainEvent, id: string) {
const segment = await Segment.findByPk(id);
return segment.toJSON();
}
private async findAll(
_event: IpcMainEvent,
params: {
targetId: string;
targetType: string;
segmentIndex: number;
}
) {
const { targetId, targetType, segmentIndex } = params;
const segments = await Segment.findAll({
where: {
targetId,
targetType,
segmentIndex,
},
include: [Audio, Video],
});
return segments.map((segment) => segment.toJSON());
}
private async create(
_event: IpcMainEvent,
params: {
targetId: string;
targetType: string;
segmentIndex: number;
}
) {
const segment = await Segment.generate({
targetId: params.targetId,
targetType: params.targetType,
segmentIndex: params.segmentIndex,
});
return segment.toJSON();
}
private async sync(_event: IpcMainEvent, id: string) {
const segment = await Segment.findByPk(id);
await segment.sync();
await segment.upload();
return segment.toJSON();
}
register() {
ipcMain.handle("segments-create", this.create);
ipcMain.handle("segments-find", this.find);
ipcMain.handle("segments-find-all", this.findAll);
ipcMain.handle("segments-sync", this.sync);
}
}
export const segmentsHandler = new SegmentsHandler();

View File

@@ -8,7 +8,9 @@ import {
CacheObject,
Conversation,
Message,
Note,
PronunciationAssessment,
Segment,
Speech,
Transcription,
Video,
@@ -18,7 +20,9 @@ import {
cacheObjectsHandler,
conversationsHandler,
messagesHandler,
notesHandler,
recordingsHandler,
segmentsHandler,
speechesHandler,
transcriptionsHandler,
videosHandler,
@@ -47,8 +51,10 @@ db.connect = async () => {
CacheObject,
Conversation,
Message,
Note,
PronunciationAssessment,
Recording,
Segment,
Speech,
Transcription,
Video,
@@ -92,9 +98,11 @@ db.connect = async () => {
// register handlers
audiosHandler.register();
cacheObjectsHandler.register();
recordingsHandler.register();
conversationsHandler.register();
messagesHandler.register();
notesHandler.register();
recordingsHandler.register();
segmentsHandler.register();
speechesHandler.register();
transcriptionsHandler.register();
videosHandler.register();

View File

@@ -0,0 +1,70 @@
import { DataTypes } from "sequelize";
async function up({ context: queryInterface }) {
queryInterface.createTable(
"segments",
{
id: {
type: DataTypes.UUID,
defaultValue: DataTypes.UUIDV4,
primaryKey: true,
allowNull: false,
},
target_type: {
type: DataTypes.STRING,
allowNull: false,
},
target_id: {
type: DataTypes.UUID,
allowNull: false,
},
md5: {
type: DataTypes.STRING,
allowNull: false,
},
segment_index: {
type: DataTypes.INTEGER,
allowNull: false,
},
caption: {
type: DataTypes.JSON,
},
start_time: {
type: DataTypes.NUMBER,
},
end_time: {
type: DataTypes.NUMBER,
},
synced_at: {
type: DataTypes.DATE,
},
uploaded_at: {
type: DataTypes.DATE,
},
created_at: {
type: DataTypes.DATE,
allowNull: false,
},
updated_at: {
type: DataTypes.DATE,
allowNull: false,
},
},
{
indexes: [
{
fields: ["target_type", "target_id"],
},
{
fields: ["md5"],
},
],
}
);
}
async function down({ context: queryInterface }) {
queryInterface.dropTable("segments");
}
export { up, down };

View File

@@ -0,0 +1,45 @@
import { DataTypes } from "sequelize";
async function up({ context: queryInterface }) {
queryInterface.createTable("notes", {
id: {
type: DataTypes.UUID,
defaultValue: DataTypes.UUIDV4,
primaryKey: true,
allowNull: false,
},
target_type: {
type: DataTypes.STRING,
allowNull: false,
},
target_id: {
type: DataTypes.UUID,
allowNull: false,
},
parameters: {
type: DataTypes.JSON,
defaultValue: {},
},
content: {
type: DataTypes.TEXT,
allowNull: false,
},
synced_at: {
type: DataTypes.DATE,
},
created_at: {
type: DataTypes.DATE,
allowNull: false,
},
updated_at: {
type: DataTypes.DATE,
allowNull: false,
},
});
}
async function down({ context: queryInterface }) {
queryInterface.dropTable("notes");
}
export { up, down };

View File

@@ -1,9 +1,11 @@
export * from './audio';
export * from './recording';
export * from './conversation';
export * from './message';
export * from './speech';
export * from './pronunciation-assessment';
export * from './cache-object';
export * from './transcription';
export * from './video';
export * from "./audio";
export * from "./cache-object";
export * from "./conversation";
export * from "./message";
export * from "./note";
export * from "./pronunciation-assessment";
export * from "./recording";
export * from "./segment";
export * from "./speech";
export * from "./transcription";
export * from "./video";

View File

@@ -0,0 +1,155 @@
import {
AfterUpdate,
AfterDestroy,
BelongsTo,
Table,
Column,
Default,
IsUUID,
Model,
DataType,
AfterCreate,
AllowNull,
AfterFind,
} from "sequelize-typescript";
import mainWindow from "@main/window";
import log from "@main/logger";
import { Client } from "@/api";
import { WEB_API_URL } from "@/constants";
import settings from "@main/settings";
import { Segment } from "@main/db/models";
const logger = log.scope("db/models/note");
@Table({
modelName: "Note",
tableName: "notes",
underscored: true,
timestamps: true,
})
export class Note extends Model<Note> {
@IsUUID("all")
@Default(DataType.UUIDV4)
@Column({ primaryKey: true, type: DataType.UUID })
id: string;
@Column(DataType.UUID)
targetId: string;
@Column(DataType.STRING)
targetType: string;
@AllowNull(false)
@Column(DataType.TEXT)
content: string;
@Default({})
@Column(DataType.JSON)
parameters: any;
@Column(DataType.DATE)
syncedAt: Date;
@BelongsTo(() => Segment, { foreignKey: "targetId", constraints: false })
segment: Segment;
@Column(DataType.VIRTUAL)
get isSynced(): boolean {
return Boolean(this.syncedAt) && this.syncedAt >= this.updatedAt;
}
async sync(): Promise<void> {
if (this.isSynced) return;
const webApi = new Client({
baseUrl: process.env.WEB_API_URL || WEB_API_URL,
accessToken: settings.getSync("user.accessToken") as string,
logger,
});
// Sync the segment if the note is related to a segment
if (this.targetType === "Segment") {
const segment = await Segment.findByPk(this.targetId);
if (!segment) {
throw new Error("Segment not found");
}
await segment.sync();
}
return webApi.syncNote(this.toJSON()).then(() => {
const now = new Date();
this.update({ syncedAt: now, updatedAt: now });
});
}
@AfterFind
static async syncAfterFind(notes: Note[]) {
if (!notes.length) return;
const unsyncedNotes = notes.filter((note) => note.id && !note.isSynced);
if (!unsyncedNotes.length) return;
unsyncedNotes.forEach((note) => {
note.sync().catch((err) => {
logger.error("sync note error", note.id, err);
});
});
}
@AfterCreate
static syncAndUploadAfterCreate(note: Note) {
note.sync();
}
@AfterCreate
static notifyForCreate(note: Note) {
this.notify(note, "create");
}
@AfterUpdate
static notifyForUpdate(note: Note) {
this.notify(note, "update");
}
@AfterUpdate
static syncAfterUpdate(note: Note) {
note.sync().catch((err) => {
logger.error("sync error", err);
});
}
@AfterDestroy
static destroyRemote(note: Note) {
const webApi = new Client({
baseUrl: process.env.WEB_API_URL || WEB_API_URL,
accessToken: settings.getSync("user.accessToken") as string,
logger,
});
webApi.deleteNote(note.id).catch((err) => {
logger.error("delete remote note failed:", err.message);
});
}
@AfterDestroy
static notifyForDestroy(note: Note) {
this.notify(note, "destroy");
}
static async notify(note: Note, action: "create" | "update" | "destroy") {
if (!mainWindow.win) return;
const segment = await Segment.findOne({ where: { id: note.targetId } });
const record = note.toJSON();
if (segment) {
record.segment = segment.toJSON();
}
mainWindow.win.webContents.send("db-on-transaction", {
model: "Note",
id: note.id,
action,
record,
});
}
}

View File

@@ -0,0 +1,246 @@
import {
AfterUpdate,
AfterDestroy,
BelongsTo,
Table,
Column,
Default,
IsUUID,
Model,
DataType,
Unique,
AfterCreate,
AllowNull,
AfterFind,
} from "sequelize-typescript";
import { Audio, Transcription, Video } from "@main/db/models";
import mainWindow from "@main/window";
import log from "@main/logger";
import { Client } from "@/api";
import { WEB_API_URL } from "@/constants";
import settings from "@main/settings";
import storage from "@/main/storage";
import path from "path";
import { TimelineEntry } from "echogarden/dist/utilities/Timeline.d.js";
import FfmpegWrapper from "@/main/ffmpeg";
import { hashFile } from "@/main/utils";
import fs from "fs-extra";
import { v5 as uuidv5 } from "uuid";
const logger = log.scope("db/models/segment");
const OUTPUT_FORMAT = "mp3";
@Table({
modelName: "Segment",
tableName: "segments",
underscored: true,
timestamps: true,
})
export class Segment extends Model<Segment> {
@IsUUID("all")
@Default(DataType.UUIDV4)
@Column({ primaryKey: true, type: DataType.UUID })
id: string;
@Column(DataType.UUID)
targetId: string;
@Column(DataType.STRING)
targetType: string;
@AllowNull(false)
@Column(DataType.INTEGER)
segmentIndex: number;
@Unique
@Column(DataType.STRING)
md5: string;
@Column(DataType.JSON)
caption: TimelineEntry;
@Column(DataType.NUMBER)
startTime: number;
@Column(DataType.NUMBER)
endTime: number;
@Column(DataType.DATE)
syncedAt: Date;
@Column(DataType.DATE)
uploadedAt: Date;
@BelongsTo(() => Audio, { foreignKey: "targetId", constraints: false })
audio: Audio;
@BelongsTo(() => Video, { foreignKey: "targetId", constraints: false })
video: Video;
@Column(DataType.VIRTUAL)
get isSynced(): boolean {
return Boolean(this.syncedAt) && this.syncedAt >= this.updatedAt;
}
@Column(DataType.VIRTUAL)
get isUploaded(): boolean {
return Boolean(this.uploadedAt) && this.uploadedAt >= this.updatedAt;
}
@Column(DataType.VIRTUAL)
get src(): string {
return `enjoy://${path.posix.join(
"library",
"segments",
this.getDataValue("md5") + "." + OUTPUT_FORMAT
)}`;
}
get filePath(): string {
return path.join(
settings.userDataPath(),
"segments",
this.getDataValue("md5") + "." + OUTPUT_FORMAT
);
}
async sync() {
if (this.isSynced) return;
const webApi = new Client({
baseUrl: process.env.WEB_API_URL || WEB_API_URL,
accessToken: settings.getSync("user.accessToken") as string,
logger,
});
return webApi.syncSegment(this.toJSON()).then(() => {
const now = new Date();
this.update({ syncedAt: now, updatedAt: now });
});
}
async upload() {
if (this.isUploaded) return;
return storage
.put(this.md5, this.filePath)
.then((result) => {
logger.debug("upload result:", result.data);
if (result.data.success) {
this.update({ uploadedAt: new Date() });
} else {
throw new Error(result.data);
}
})
.catch((err) => {
logger.error("upload failed:", err.message);
throw err;
});
}
static async generate(params: {
targetId: string;
targetType: string;
segmentIndex: number;
}) {
let target: Video | Audio;
if (params.targetType === "Video") {
target = await Video.findByPk(params.targetId);
} else if (params.targetType === "Audio") {
target = await Audio.findByPk(params.targetId);
} else {
throw new Error("Invalid targetType");
}
const { targetId, targetType, segmentIndex } = params;
const transcription = await Transcription.findOne({
where: { targetId, targetType },
});
if (!transcription) {
throw new Error("Transcription not found");
}
const caption = transcription.result.timeline[segmentIndex];
if (!caption) {
throw new Error("Caption not found");
}
const ffmpeg = new FfmpegWrapper();
const output = path.join(
settings.cachePath(),
`${target.md5}-${segmentIndex}.${OUTPUT_FORMAT}`
);
await ffmpeg.crop(target.filePath, {
startTime: caption.startTime,
endTime: caption.endTime,
output,
});
const md5 = await hashFile(output, { algo: "md5" });
const userId = settings.getSync("user.id");
const id = uuidv5(`${userId}/${md5}`, uuidv5.URL);
const dir = path.join(settings.userDataPath(), "segments");
fs.ensureDirSync(dir);
fs.moveSync(output, path.join(dir, `${md5}.${OUTPUT_FORMAT}`), {
overwrite: true,
});
return Segment.create({
id,
targetId,
targetType,
segmentIndex,
md5,
caption,
startTime: caption.startTime,
endTime: caption.endTime,
});
}
@AfterFind
static async syncAfterFind(segments: Segment[]) {
if (!segments.length) return;
const unsyncedSegments = segments.filter((segment) => !segment.isSynced);
if (!unsyncedSegments.length) return;
unsyncedSegments.forEach((segment) => {
segment.sync().catch((err) => {
logger.error("sync error", err);
});
});
}
@AfterCreate
static syncAndUploadAfterCreate(segment: Segment) {
segment.sync();
segment.upload();
}
@AfterUpdate
static notifyForUpdate(segment: Segment) {
this.notify(segment, "update");
}
@AfterUpdate
static syncAfterUpdate(segment: Segment) {
segment.sync().catch((err) => {
logger.error("sync error", err);
});
}
@AfterDestroy
static notifyForDestroy(segment: Segment) {
this.notify(segment, "destroy");
}
static notify(segment: Segment, action: "create" | "update" | "destroy") {
if (!mainWindow.win) return;
mainWindow.win.webContents.send("db-on-transaction", {
model: "Segment",
id: segment.id,
action: action,
record: segment.toJSON(),
});
}
}

View File

@@ -17,6 +17,7 @@ import log from "@main/logger";
import { Client } from "@/api";
import { WEB_API_URL, PROCESS_TIMEOUT } from "@/constants";
import settings from "@main/settings";
import { AlignmentResult } from "echogarden/dist/api/Alignment";
const logger = log.scope("db/models/transcription");
@Table({
@@ -52,7 +53,7 @@ export class Transcription extends Model<Transcription> {
model: string;
@Column(DataType.JSON)
result: any;
result: Partial<AlignmentResult> & { originalText?: string };
@Column(DataType.DATE)
syncedAt: Date;

View File

@@ -233,6 +233,44 @@ export default class FfmpegWrapper {
});
}
// Crop video or audio from start to end time to a mp3 file
// Save the file to the output path
crop(
input: string,
options: {
startTime: number;
endTime: number;
output: string;
}
) {
const { startTime, endTime, output } = options;
const ffmpeg = Ffmpeg();
return new Promise((resolve, reject) => {
ffmpeg
.input(input)
.outputOptions(
"-ss",
startTime.toString(),
"-to",
endTime.toString()
)
.on("start", (commandLine) => {
logger.info("Spawned FFmpeg with command: " + commandLine);
fs.ensureDirSync(path.dirname(output));
})
.on("end", () => {
logger.info(`File ${output} created`);
resolve(output);
})
.on("error", (err) => {
logger.error(err);
reject(err);
})
.save(output);
});
}
registerIpcHandlers() {
ipcMain.handle("ffmpeg-check-command", async (_event) => {
return await this.checkCommand();

View File

@@ -49,7 +49,7 @@ export function enjoyUrlToPath(enjoyUrl: string): string {
let filePath = enjoyUrl;
if (
enjoyUrl.match(/enjoy:\/\/library\/(audios|videos|recordings|speeches)/g)
enjoyUrl.match(/enjoy:\/\/library\/(audios|videos|recordings|speeches|segments)/g)
) {
filePath = path.posix.join(
settings.userDataPath(),

View File

@@ -473,4 +473,51 @@ contextBridge.exposeInMainWorld("__ENJOY_APP__", {
return ipcRenderer.invoke("waveforms-save", id, data);
},
},
segments: {
findAll: (params: {
targetId?: string;
targetType?: string;
offset?: number;
limit?: number;
}) => {
return ipcRenderer.invoke("segments-find-all", params);
},
find: (id: string) => {
return ipcRenderer.invoke("segments-find", id);
},
create: (params: any) => {
return ipcRenderer.invoke("segments-create", params);
},
sync: (id: string) => {
return ipcRenderer.invoke("segments-sync", id);
},
},
notes: {
groupByTarget: (params?: { limit?: number; offset?: number }) => {
return ipcRenderer.invoke("notes-group-by-target", params);
},
groupBySegment: (targetId: string, targetType: string) => {
return ipcRenderer.invoke("notes-group-by-segment", targetId, targetType);
},
findAll: (params: {
targetId?: string;
targetType?: string;
offset?: number;
limit?: number;
}) => {
return ipcRenderer.invoke("notes-find-all", params);
},
update: (id: string, params: any) => {
return ipcRenderer.invoke("notes-update", id, params);
},
delete: (id: string) => {
return ipcRenderer.invoke("notes-delete", id);
},
create: (params: any) => {
return ipcRenderer.invoke("notes-create", params);
},
sync: (id: string) => {
return ipcRenderer.invoke("notes-sync", id);
},
},
});

View File

@@ -11,9 +11,15 @@ import {
} from "@renderer/components";
import { useAudio } from "@renderer/hooks";
export const AudioPlayer = (props: { id?: string; md5?: string }) => {
const { id, md5 } = props;
const { setMedia, layout } = useContext(MediaPlayerProviderContext);
export const AudioPlayer = (props: {
id?: string;
md5?: string;
segmentIndex?: number;
}) => {
const { id, md5, segmentIndex } = props;
const { setMedia, layout, setCurrentSegmentIndex } = useContext(
MediaPlayerProviderContext
);
const { audio } = useAudio({ id, md5 });
useEffect(() => {
@@ -21,12 +27,17 @@ export const AudioPlayer = (props: { id?: string; md5?: string }) => {
setMedia(audio);
}, [audio]);
useEffect(() => {
if (!segmentIndex) return;
setCurrentSegmentIndex(segmentIndex);
}, []);
if (!layout) return <LoaderSpin />;
return (
<div data-testid="audio-player" className={layout.wrapper}>
<div className={`${layout.upperWrapper} mb-4`}>
<div className="grid grid-cols-5 xl:grid-cols-3 gap-6 px-6 h-full">
<div className="grid grid-cols-5 xl:grid-cols-3 gap-3 xl:gap-6 px-3 xl:px-6 h-full">
<div
className={`col-span-2 xl:col-span-1 rounded-lg border shadow-lg ${layout.upperWrapper}`}
>
@@ -39,11 +50,11 @@ export const AudioPlayer = (props: { id?: string; md5?: string }) => {
</div>
<div className={`flex flex-col`}>
<div className={`${layout.playerWrapper} py-2 px-6`}>
<div className={`${layout.playerWrapper} py-2 px-3 xl:px-6`}>
<MediaCurrentRecording />
</div>
<div className={`${layout.playerWrapper} py-2 px-6`}>
<div className={`${layout.playerWrapper} py-2 px-3 xl:px-6`}>
<MediaPlayer />
</div>

View File

@@ -154,7 +154,7 @@ export const ConversationShortcuts = (props: {
<div
key={conversation.id}
onClick={() => ask(conversation)}
className="bg-background text-primary rounded-full w-full mb-2 py-2 px-4 hover:bg-primary hover:text-white cursor-pointer flex items-center border"
className="bg-background text-primary rounded-full w-full mb-2 py-2 px-4 hover:bg-muted hover:text-muted-foreground cursor-pointer flex items-center border"
style={{
borderLeftColor: `#${conversation.id
.replaceAll("-", "")

View File

@@ -2,15 +2,13 @@ export * from "./audios";
export * from "./conversations";
export * from "./meanings";
export * from "./messages";
export * from "./medias";
export * from "./notes";
export * from "./posts";
export * from "./preferences";
export * from "./pronunciation-assessments";
export * from "./recordings";
export * from "./stories";
export * from "./videos";
export * from "./medias";
export * from "./posts";
export * from "./users";
export * from "./videos";
export * from "./widgets";

View File

@@ -1,6 +1,5 @@
export * from "./media-player-controls";
export * from "./media-caption";
export * from "./media-caption-tabs";
export * from "./media-info-panel";
export * from "./media-recordings";
export * from "./media-current-recording";

View File

@@ -2,19 +2,25 @@ import { useEffect, useState, useContext } from "react";
import { MediaPlayerProviderContext } from "@renderer/context";
import cloneDeep from "lodash/cloneDeep";
import { Button, toast } from "@renderer/components/ui";
import { ConversationShortcuts, MediaCaptionTabs } from "@renderer/components";
import { ConversationShortcuts } from "@renderer/components";
import { t } from "i18next";
import { BotIcon, CopyIcon, CheckIcon, SpeechIcon } from "lucide-react";
import {
BotIcon,
CopyIcon,
CheckIcon,
SpeechIcon,
NotebookPenIcon,
} from "lucide-react";
import {
Timeline,
TimelineEntry,
} from "echogarden/dist/utilities/Timeline.d.js";
import { convertIpaToNormal } from "@/utils";
import { useCopyToClipboard } from "@uidotdev/usehooks";
import { MediaCaptionTabs } from "./media-captions";
export const MediaCaption = () => {
const {
wavesurfer,
currentSegmentIndex,
currentTime,
transcription,
@@ -30,6 +36,7 @@ export const MediaCaption = () => {
const [multiSelecting, setMultiSelecting] = useState<boolean>(false);
const [displayIpa, setDisplayIpa] = useState<boolean>(true);
const [displayNotes, setDisplayNotes] = useState<boolean>(true);
const [_, copyToClipboard] = useCopyToClipboard();
const [copied, setCopied] = useState<boolean>(false);
@@ -39,29 +46,71 @@ export const MediaCaption = () => {
setMultiSelecting(event.shiftKey && event.type === "keydown");
};
const toggleRegion = (index: number) => {
const toggleSeletedIndex = (index: number) => {
if (!activeRegion) return;
if (editingRegion) {
toast.warning(t("currentRegionIsBeingEdited"));
return;
}
const word = caption.timeline[index];
if (!word) return;
const startWord = caption.timeline[index];
if (!startWord) return;
const start = word.startTime;
const end = word.endTime;
if (multiSelecting) {
const min = Math.min(index, ...selectedIndices);
const max = Math.max(index, ...selectedIndices);
// Select all the words between the min and max indices.
setSelectedIndices(
Array.from({ length: max - min + 1 }, (_, i) => i + min)
);
} else if (selectedIndices.includes(index)) {
setSelectedIndices([]);
} else {
setSelectedIndices([index]);
}
};
const toggleRegion = (params: number[]) => {
if (!activeRegion) return;
if (editingRegion) {
toast.warning(t("currentRegionIsBeingEdited"));
return;
}
if (params.length === 0) {
if (activeRegion.id.startsWith("word-region")) {
activeRegion.remove();
setActiveRegion(
regions.getRegions().find((r) => r.id.startsWith("segment-region"))
);
}
return;
}
const startIndex = Math.min(...params);
const endIndex = Math.max(...params);
const startWord = caption.timeline[startIndex];
if (!startWord) return;
const endWord = caption.timeline[endIndex] || startWord;
const start = startWord.startTime;
const end = endWord.endTime;
const regionStart = activeRegion.start;
const regionEnd = activeRegion.end;
// If the active region is a word region, then merge the selected words into a single region.
if (activeRegion.id.startsWith("word-region")) {
activeRegion.remove();
if (start >= regionStart && end <= regionEnd) {
setActiveRegion(
regions.getRegions().find((r) => r.id.startsWith("segment-region"))
);
} else if (multiSelecting) {
} else {
const region = regions.addRegion({
id: `word-region-${index}`,
id: `word-region-${startIndex}`,
start: Math.min(start, regionStart),
end: Math.max(end, regionEnd),
color: "#fb6f9233",
@@ -69,27 +118,17 @@ export const MediaCaption = () => {
resize: editingRegion,
});
setActiveRegion(region);
} else {
const region = regions.addRegion({
id: `word-region-${index}`,
start,
end,
color: "#fb6f9233",
drag: false,
resize: editingRegion,
});
setActiveRegion(region);
}
activeRegion?.remove();
// If the active region is a meaning group region, then active the segment region.
} else if (activeRegion.id.startsWith("meaning-group-region")) {
setActiveRegion(
regions.getRegions().find((r) => r.id.startsWith("segment-region"))
);
// If the active region is a segment region, then create a new word region.
} else {
const region = regions.addRegion({
id: `word-region-${index}`,
id: `word-region-${startIndex}`,
start,
end,
color: "#fb6f9233",
@@ -101,43 +140,6 @@ export const MediaCaption = () => {
}
};
const markPhoneRegions = () => {
const phoneRegions = regions
.getRegions()
.filter((r) => r.id.startsWith("phone-region"));
if (phoneRegions.length > 0) {
phoneRegions.forEach((r) => {
r.remove();
r.unAll();
});
return;
}
if (!activeRegion) return;
if (!activeRegion.id.startsWith("word-region")) return;
if (!selectedIndices) return;
selectedIndices.forEach((index) => {
const word = caption.timeline[index];
word.timeline.forEach((token) => {
token.timeline.forEach((phone) => {
const region = regions.addRegion({
id: `phone-region-${index}`,
start: phone.startTime,
end: phone.endTime,
color: "#efefefef",
drag: false,
resize: editingRegion,
});
region.on("click", () => {
region.play();
});
});
});
});
};
useEffect(() => {
if (!caption) return;
@@ -154,25 +156,8 @@ export const MediaCaption = () => {
if (!caption?.timeline) return;
if (!activeRegion) return;
if (activeRegion.id.startsWith("segment-region")) {
setSelectedIndices([]);
return;
}
const indices: number[] = [];
caption.timeline.forEach((w, index) => {
if (
w.startTime >= activeRegion.start &&
(w.endTime <= activeRegion.end ||
// The last word's end time may be a little greater than the duration of the audio in somehow.
w.endTime > wavesurfer.getDuration())
) {
indices.push(index);
}
});
setSelectedIndices(indices);
}, [caption, activeRegion]);
toggleRegion(selectedIndices);
}, [caption, selectedIndices]);
useEffect(() => {
if (!activeRegion) return;
@@ -254,6 +239,10 @@ export const MediaCaption = () => {
);
}, [currentSegmentIndex, transcription]);
useEffect(() => {
return () => setSelectedIndices([]);
}, [caption]);
useEffect(() => {
document.addEventListener("keydown", (event: KeyboardEvent) =>
toggleMultiSelect(event)
@@ -275,91 +264,19 @@ export const MediaCaption = () => {
<div className="flex-1 font-serif h-full border shadow-lg rounded-lg">
<MediaCaptionTabs
caption={caption}
currentSegmentIndex={currentSegmentIndex}
selectedIndices={selectedIndices}
toggleRegion={toggleRegion}
setSelectedIndices={setSelectedIndices}
>
<div className="flex flex-wrap px-4 py-2 rounded-t-lg bg-muted/50">
{/* use the words splitted by caption text if it is matched with the timeline length, otherwise use the timeline */}
{caption.text.split(" ").length !== caption.timeline.length
? (caption.timeline || []).map((w, index) => (
<div
key={index}
id={`word-${currentSegmentIndex}-${index}`}
className={`p-1 pb-2 rounded cursor-pointer hover:bg-red-500/10 ${
index === activeIndex ? "text-red-500" : ""
} ${
selectedIndices.includes(index)
? "bg-red-500/10 selected"
: ""
}`}
onClick={() => toggleRegion(index)}
>
<div className="">
<div className="font-serif text-lg xl:text-xl 2xl:text-2xl">
{w.text}
</div>
{displayIpa && (
<div
className={`text-sm 2xl:text-base text-muted-foreground font-code ${
index === 0 ? "before:content-['/']" : ""
}
${
index === caption.timeline.length - 1
? "after:content-['/']"
: ""
}`}
>
{w.timeline
.map((t) =>
t.timeline
.map((s) => convertIpaToNormal(s.text))
.join("")
)
.join(" · ")}
</div>
)}
</div>
</div>
))
: caption.text.split(" ").map((word, index) => (
<div
key={index}
id={`word-${currentSegmentIndex}-${index}`}
className={`p-1 pb-2 rounded cursor-pointer hover:bg-red-500/10 ${
index === activeIndex ? "text-red-500" : ""
} ${
selectedIndices.includes(index) ? "bg-red-500/10" : ""
}`}
onClick={() => toggleRegion(index)}
>
<div className="">
<div className="text-serif text-lg xl:text-xl 2xl:text-2xl">
{word}
</div>
{displayIpa && (
<div
className={`text-sm 2xl:text-base text-muted-foreground font-code ${
index === 0 ? "before:content-['/']" : ""
}
${
index === caption.text.split(" ").length - 1
? "after:content-['/']"
: ""
}`}
>
{caption.timeline[index].timeline
.map((t) =>
t.timeline
.map((s) => convertIpaToNormal(s.text))
.join("")
)
.join(" · ")}
</div>
)}
</div>
</div>
))}
</div>
<Caption
caption={caption}
selectedIndices={selectedIndices}
currentSegmentIndex={currentSegmentIndex}
activeIndex={activeIndex}
displayIpa={displayIpa}
displayNotes={displayNotes}
onClick={toggleSeletedIndex}
/>
</MediaCaptionTabs>
</div>
@@ -375,6 +292,17 @@ export const MediaCaption = () => {
<SpeechIcon className="w-4 h-4" />
</Button>
<Button
variant={displayNotes ? "secondary" : "outline"}
size="icon"
className="rounded-full w-8 h-8 p-0"
data-tooltip-id="media-player-tooltip"
data-tooltip-content={t("displayNotes")}
onClick={() => setDisplayNotes(!displayNotes)}
>
<NotebookPenIcon className="w-4 h-4" />
</Button>
<ConversationShortcuts
prompt={caption.text as string}
trigger={
@@ -433,3 +361,97 @@ export const MediaCaption = () => {
</div>
);
};
const Caption = (props: {
caption: TimelineEntry;
selectedIndices: number[];
currentSegmentIndex: number;
activeIndex: number;
displayIpa: boolean;
displayNotes: boolean;
onClick: (index: number) => void;
}) => {
const {
caption,
selectedIndices,
currentSegmentIndex,
activeIndex,
displayIpa,
displayNotes,
onClick,
} = props;
const { currentNotes } = useContext(MediaPlayerProviderContext);
const notes = currentNotes.filter((note) => note.parameters?.quoteIndices);
const [notedquoteIndices, setNotedquoteIndices] = useState<number[]>([]);
let words = caption.text.split(" ");
const ipas = caption.timeline.map((w) =>
w.timeline.map((t) => t.timeline.map((s) => s.text))
);
if (words.length !== caption.timeline.length) {
words = caption.timeline.map((w) => w.text);
}
return (
<div className="flex flex-wrap px-4 py-2 rounded-t-lg bg-muted/50">
{/* use the words splitted by caption text if it is matched with the timeline length, otherwise use the timeline */}
{words.map((word, index) => (
<div
className=""
key={`word-${currentSegmentIndex}-${index}`}
id={`word-${currentSegmentIndex}-${index}`}
>
<div
className={`font-serif text-lg xl:text-xl 2xl:text-2xl cursor-pointer p-1 pb-2 rounded hover:bg-red-500/10 ${
index === activeIndex ? "text-red-500" : ""
} ${
selectedIndices.includes(index) ? "bg-red-500/10 selected" : ""
} ${
notedquoteIndices.includes(index)
? "border-b border-red-500 border-dashed"
: ""
}`}
onClick={() => onClick(index)}
>
{word}
</div>
{displayIpa && (
<div
className={`select-text text-sm 2xl:text-base text-muted-foreground font-code mb-1 ${
index === 0 ? "before:content-['/']" : ""
} ${
index === caption.timeline.length - 1
? "after:content-['/']"
: ""
}`}
>
{ipas[index]}
</div>
)}
{displayNotes &&
notes
.filter((note) => note.parameters.quoteIndices[0] === index)
.map((note) => (
<div
key={`note-${currentSegmentIndex}-${note.id}`}
className="mb-1 text-xs 2xl:text-sm text-red-500 max-w-64 line-clamp-3 font-code cursor-pointer"
onMouseOver={() =>
setNotedquoteIndices(note.parameters.quoteIndices)
}
onMouseLeave={() => setNotedquoteIndices([])}
onClick={() =>
document.getElementById("note-" + note.id)?.scrollIntoView()
}
>
{note.parameters.quoteIndices[0] === index && note.content}
</div>
))}
</div>
))}
</div>
);
};

View File

@@ -0,0 +1,4 @@
export * from "./media-caption-tabs";
export * from "./tab-content-analysis";
export * from "./tab-content-note";
export * from "./tab-content-translation";

View File

@@ -0,0 +1,67 @@
import { useState } from "react";
import {
Tabs,
TabsList,
TabsTrigger,
ScrollArea,
} from "@renderer/components/ui";
import { t } from "i18next";
import { TimelineEntry } from "echogarden/dist/utilities/Timeline.d.js";
import { TabContentTranslation } from "./tab-content-translation";
import { TabContentAnalysis } from "./tab-content-analysis";
import { TabContentNote } from "./tab-content-note";
export const MediaCaptionTabs = (props: {
caption: TimelineEntry;
currentSegmentIndex: number;
selectedIndices: number[];
setSelectedIndices: (indices: number[]) => void;
children?: React.ReactNode;
}) => {
const {
caption,
currentSegmentIndex,
selectedIndices,
setSelectedIndices,
children,
} = props;
const [tab, setTab] = useState<string>("note");
if (!caption) return null;
return (
<ScrollArea className="h-full relative">
<Tabs value={tab} onValueChange={(value) => setTab(value)} className="">
{children}
<div className="px-4 pb-10 min-h-32">
<TabContentNote
currentSegmentIndex={currentSegmentIndex}
selectedIndices={selectedIndices}
setSelectedIndices={setSelectedIndices}
/>
<TabContentTranslation
caption={caption}
selectedIndices={selectedIndices}
/>
<TabContentAnalysis text={caption.text} />
</div>
<TabsList className="grid grid-cols-3 gap-4 rounded-none absolute w-full bottom-0 px-4">
<TabsTrigger value="note" className="block truncate px-1">
{t("captionTabs.note")}
</TabsTrigger>
<TabsTrigger value="translation" className="block truncate px-1">
{t("captionTabs.translation")}
</TabsTrigger>
<TabsTrigger value="analysis" className="block truncate px-1">
{t("captionTabs.analysis")}
</TabsTrigger>
</TabsList>
</Tabs>
</ScrollArea>
);
};

View File

@@ -0,0 +1,133 @@
import { useEffect, useState, useContext } from "react";
import { AppSettingsProviderContext } from "@renderer/context";
import { Button, toast, TabsContent } from "@renderer/components/ui";
import { ConversationShortcuts } from "@renderer/components";
import { t } from "i18next";
import { BotIcon } from "lucide-react";
import { useAiCommand } from "@renderer/hooks";
import { LoaderIcon } from "lucide-react";
import { md5 } from "js-md5";
import Markdown from "react-markdown";
export function TabContentAnalysis(props: { text: string; }) {
const { text } = props;
const { EnjoyApp } = useContext(AppSettingsProviderContext);
const [analyzing, setAnalyzing] = useState<boolean>(false);
const [analysisResult, setAnalysisResult] = useState<string>();
const { analyzeText } = useAiCommand();
const analyzeSetence = async () => {
if (analyzing) return;
setAnalyzing(true);
analyzeText(text, `analyze-${md5(text)}`)
.then((result) => {
if (result) {
setAnalysisResult(result);
}
})
.catch((err) => toast.error(err.message))
.finally(() => {
setAnalyzing(false);
});
};
/*
* If the caption is changed, then reset the analysis.
* Also, check if the translation is cached, then use it.
*/
useEffect(() => {
EnjoyApp.cacheObjects.get(`analyze-${md5(text)}`).then((cached) => {
setAnalysisResult(cached);
});
}, [text]);
return (
<TabsContent value="analysis">
{analysisResult ? (
<>
<Markdown
className="select-text prose dark:prose-invert prose-sm prose-h3:text-base max-w-full mb-4"
components={{
a({ node, children, ...props }) {
try {
new URL(props.href ?? "");
props.target = "_blank";
props.rel = "noopener noreferrer";
} catch (e) { }
return <a {...props}>{children}</a>;
},
}}
>
{analysisResult}
</Markdown>
<div className="flex items-center space-x-2">
<Button
variant="secondary"
size="sm"
disabled={analyzing}
onClick={analyzeSetence}
>
{analyzing && (
<LoaderIcon className="animate-spin w-4 h-4 mr-2" />
)}
{t("reAnalyze")}
</Button>
<AIButton
prompt={text as string}
onReply={(replies) => {
const result = replies.map((m) => m.content).join("\n");
setAnalysisResult(result);
EnjoyApp.cacheObjects.set(`analyze-${md5(text)}`, result);
} }
tooltip={t("useAIAssistantToAnalyze")} />
</div>
</>
) : (
<div className="flex items-center justify-center space-x-2 py-4">
<Button size="sm" disabled={analyzing} onClick={analyzeSetence}>
{analyzing && <LoaderIcon className="animate-spin w-4 h-4 mr-2" />}
<span>{t("analyzeSetence")}</span>
</Button>
<AIButton
prompt={text as string}
onReply={(replies) => {
const result = replies.map((m) => m.content).join("\n");
setAnalysisResult(result);
EnjoyApp.cacheObjects.set(`analyze-${md5(text)}`, result);
} }
tooltip={t("useAIAssistantToAnalyze")} />
</div>
)}
</TabsContent>
);
}
const AIButton = (props: {
prompt: string;
onReply?: (replies: MessageType[]) => void;
tooltip: string;
}) => {
const { prompt, onReply, tooltip } = props;
return (
<ConversationShortcuts
prompt={prompt}
onReply={onReply}
title={tooltip}
trigger={
<Button
data-tooltip-id="media-player-tooltip"
data-tooltip-content={tooltip}
variant="outline"
size="sm"
className="p-0 w-8 h-8 rounded-full"
>
<BotIcon className="w-5 h-5" />
</Button>
}
/>
);
};

View File

@@ -0,0 +1,93 @@
import { MediaPlayerProviderContext } from "@renderer/context";
import { Button, TabsContent, toast } from "@renderer/components/ui";
import { t } from "i18next";
import { useContext, useState } from "react";
import { NoteCard, NoteForm } from "@renderer/components";
/*
* Note tab content.
*/
export const TabContentNote = (props: {
currentSegmentIndex: number;
selectedIndices: number[];
setSelectedIndices: (indices: number[]) => void;
}) => {
const { selectedIndices, setSelectedIndices } = props;
const { currentSegment, createSegment, currentNotes } = useContext(
MediaPlayerProviderContext
);
const [editingNote, setEditingNote] = useState<NoteType>();
if (!currentSegment) {
return (
<TabsContent value="note">
<div className="py-4 flex justify-center items-center">
<Button size="sm" onClick={createSegment}>
{t("startToNote")}
</Button>
</div>
</TabsContent>
);
}
return (
<TabsContent value="note">
<div className="py-4">
<div className="">
{!editingNote && (
<div className="mb-6">
<NoteForm
segment={currentSegment}
parameters={{
quoteIndices: selectedIndices,
quote: selectedIndices
.map(
(index: number) =>
currentSegment?.caption?.timeline?.[index]?.text
)
.join(" "),
}}
onParametersChange={(param) => {
if (param.quoteIndices) {
setSelectedIndices(param.quoteIndices);
}
}}
/>
</div>
)}
<div className="space-y-2 mb-4">
{currentNotes.map((note) => (
<div key={note.id} className="flex space-x-2">
{editingNote?.id === note.id ? (
<NoteForm
segment={currentSegment}
parameters={{
quoteIndices: selectedIndices,
quote: selectedIndices
.map(
(index: number) =>
currentSegment?.caption?.timeline?.[index]?.text
)
.join(" "),
}}
onParametersChange={(param) => {
if (param.quoteIndices) {
setSelectedIndices(param.quoteIndices);
}
}}
note={note}
onCancel={() => setEditingNote(null)}
onSave={() => setEditingNote(null)}
/>
) : (
<NoteCard note={note} onEdit={() => setEditingNote(note)} />
)}
</div>
))}
</div>
</div>
</div>
</TabsContent>
);
};

View File

@@ -3,116 +3,108 @@ import {
AppSettingsProviderContext,
MediaPlayerProviderContext,
} from "@renderer/context";
import {
Button,
toast,
Tabs,
TabsList,
TabsTrigger,
TabsContent,
Separator,
ScrollArea,
} from "@renderer/components/ui";
import { ConversationShortcuts } from "@renderer/components";
import { Button, toast, TabsContent, Separator } from "@renderer/components/ui";
import { t } from "i18next";
import { BotIcon } from "lucide-react";
import { TimelineEntry } from "echogarden/dist/utilities/Timeline.d.js";
import { useAiCommand, useCamdict } from "@renderer/hooks";
import { LoaderIcon, Volume2Icon } from "lucide-react";
import { convertIpaToNormal } from "@/utils";
import { md5 } from "js-md5";
import Markdown from "react-markdown";
import { TimelineEntry } from "echogarden/dist/utilities/Timeline";
import { convertIpaToNormal } from "@/utils";
/*
* Tabs below the caption text.
* It provides the translation, analysis, and note features.
* Translation tab content.
*/
export const MediaCaptionTabs = (props: {
export function TabContentTranslation(props: {
caption: TimelineEntry;
selectedIndices: number[];
toggleRegion: (index: number) => void;
children?: React.ReactNode;
}) => {
const { caption, selectedIndices, toggleRegion, children } = props;
}) {
const { caption } = props;
const { EnjoyApp } = useContext(AppSettingsProviderContext);
const [translation, setTranslation] = useState<string>();
const [translating, setTranslating] = useState<boolean>(false);
const { translate } = useAiCommand();
const [tab, setTab] = useState<string>("selected");
const translateSetence = async () => {
if (translating) return;
if (!caption) return null;
setTranslating(true);
translate(caption.text, `translate-${md5(caption.text)}`)
.then((result) => {
if (result) {
setTranslation(result);
}
})
.catch((err) => toast.error(err.message))
.finally(() => {
setTranslating(false);
});
};
/*
* If the caption is changed, then reset the translation.
* Also, check if the translation is cached, then use it.
*/
useEffect(() => {
EnjoyApp.cacheObjects
.get(`translate-${md5(caption.text)}`)
.then((cached) => {
setTranslation(cached);
});
}, [caption.text]);
return (
<ScrollArea className="h-full relative">
<Tabs value={tab} onValueChange={(value) => setTab(value)} className="">
{children}
<TabsContent value="translation">
<SelectedWords {...props} />
<div className="px-4 pb-10 min-h-32">
<SelectedTabContent
caption={caption}
selectedIndices={selectedIndices}
toggleRegion={toggleRegion}
/>
<Separator />
<TranslationTabContent text={caption.text} />
{translation ? (
<div className="py-4">
<div className="text-sm italic text-muted-foreground mb-2">
{t("translateSetence")}
</div>
<Markdown className="select-text prose dark:prose-invert prose-sm prose-h3:text-base max-w-full mb-2">
{translation}
</Markdown>
<AnalysisTabContent text={caption.text} />
<TabsContent value="note">
<div className="text-muted-foreground text-center py-4">
Comming soon
</div>
</TabsContent>
<div className="flex items-center">
<Button
variant="secondary"
size="sm"
disabled={translating}
onClick={translateSetence}
>
{translating && (
<LoaderIcon className="animate-spin w-4 h-4 mr-2" />
)}
{t("reTranslate")}
</Button>
</div>
</div>
<TabsList className="grid grid-cols-4 gap-4 rounded-none absolute w-full bottom-0 px-4">
<TabsTrigger value="selected" className="block truncate px-1">
{t("captionTabs.selected")}
</TabsTrigger>
<TabsTrigger value="translation" className="block truncate px-1">
{t("captionTabs.translation")}
</TabsTrigger>
<TabsTrigger value="analysis" className="block truncate px-1">
{t("captionTabs.analysis")}
</TabsTrigger>
<TabsTrigger value="note" className="block truncate px-1">
{t("captionTabs.note")}
</TabsTrigger>
</TabsList>
</Tabs>
</ScrollArea>
) : (
<div className="flex items-center py-4">
<Button
size="sm"
disabled={translating}
onClick={() => translateSetence()}
>
{translating && (
<LoaderIcon className="animate-spin w-4 h-4 mr-2" />
)}
<span>{t("translateSetence")}</span>
</Button>
</div>
)}
</TabsContent>
);
};
}
const AIButton = (props: {
prompt: string;
onReply?: (replies: MessageType[]) => void;
tooltip: string;
}) => {
const { prompt, onReply, tooltip } = props;
return (
<ConversationShortcuts
prompt={prompt}
onReply={onReply}
title={tooltip}
trigger={
<Button
data-tooltip-id="media-player-tooltip"
data-tooltip-content={tooltip}
variant="outline"
size="sm"
className="p-0 w-8 h-8 rounded-full"
>
<BotIcon className="w-5 h-5" />
</Button>
}
/>
);
};
const SelectedTabContent = (props: {
const SelectedWords = (props: {
caption: TimelineEntry;
selectedIndices: number[];
toggleRegion: (index: number) => void;
}) => {
const { selectedIndices, caption, toggleRegion } = props;
const { selectedIndices, caption } = props;
const { transcription } = useContext(MediaPlayerProviderContext);
const { webApi } = useContext(AppSettingsProviderContext);
@@ -187,15 +179,13 @@ const SelectedTabContent = (props: {
if (selectedIndices.length === 0)
return (
<TabsContent value="selected">
<div className="text-sm text-muted-foreground py-4">
{t("clickAnyWordToSelect")}
</div>
</TabsContent>
<div className="text-sm text-muted-foreground py-4">
{t("clickAnyWordToSelect")}
</div>
);
return (
<TabsContent value="selected">
<>
<div className="flex flex-wrap items-center space-x-2 select-text mb-4">
{selectedIndices.map((index, i) => {
const word = caption.timeline[index];
@@ -330,191 +320,6 @@ const SelectedTabContent = (props: {
</Button>
</div>
)}
<div className="flex items-center justify-end py-2">
<Button
size="sm"
variant="secondary"
onClick={() => toggleRegion(selectedIndices[0])}
>
{t("cancel")}
</Button>
</div>
</TabsContent>
);
};
/*
* Translation tab content.
*/
const TranslationTabContent = (props: { text: string }) => {
const { text } = props;
const { EnjoyApp } = useContext(AppSettingsProviderContext);
const [translation, setTranslation] = useState<string>();
const [translating, setTranslating] = useState<boolean>(false);
const { translate } = useAiCommand();
const translateSetence = async () => {
if (translating) return;
setTranslating(true);
translate(text, `translate-${md5(text)}`)
.then((result) => {
if (result) {
setTranslation(result);
}
})
.catch((err) => toast.error(err.message))
.finally(() => {
setTranslating(false);
});
};
/*
* If the caption is changed, then reset the translation.
* Also, check if the translation is cached, then use it.
*/
useEffect(() => {
EnjoyApp.cacheObjects.get(`translate-${md5(text)}`).then((cached) => {
setTranslation(cached);
});
}, [text]);
return (
<TabsContent value="translation">
{translation ? (
<>
<Markdown className="select-text prose dark:prose-invert prose-sm prose-h3:text-base max-w-full mb-4">
{translation}
</Markdown>
<div className="flex items-center justify-end">
<Button
variant="secondary"
size="sm"
disabled={translating}
onClick={translateSetence}
>
{translating && (
<LoaderIcon className="animate-spin w-4 h-4 mr-2" />
)}
{t("reTranslate")}
</Button>
</div>
</>
) : (
<div className="flex items-center justify-center space-x-2 py-4">
<Button
size="sm"
disabled={translating}
onClick={() => translateSetence()}
>
{translating && (
<LoaderIcon className="animate-spin w-4 h-4 mr-2" />
)}
<span>{t("translateSetence")}</span>
</Button>
</div>
)}
</TabsContent>
);
};
const AnalysisTabContent = (props: { text: string }) => {
const { text } = props;
const { EnjoyApp } = useContext(AppSettingsProviderContext);
const [analyzing, setAnalyzing] = useState<boolean>(false);
const [analysisResult, setAnalysisResult] = useState<string>();
const { analyzeText } = useAiCommand();
const analyzeSetence = async () => {
if (analyzing) return;
setAnalyzing(true);
analyzeText(text, `analyze-${md5(text)}`)
.then((result) => {
if (result) {
setAnalysisResult(result);
}
})
.catch((err) => toast.error(err.message))
.finally(() => {
setAnalyzing(false);
});
};
/*
* If the caption is changed, then reset the analysis.
* Also, check if the translation is cached, then use it.
*/
useEffect(() => {
EnjoyApp.cacheObjects.get(`analyze-${md5(text)}`).then((cached) => {
setAnalysisResult(cached);
});
}, [text]);
return (
<TabsContent value="analysis">
{analysisResult ? (
<>
<Markdown
className="select-text prose dark:prose-invert prose-sm prose-h3:text-base max-w-full mb-4"
components={{
a({ node, children, ...props }) {
try {
new URL(props.href ?? "");
props.target = "_blank";
props.rel = "noopener noreferrer";
} catch (e) {}
return <a {...props}>{children}</a>;
},
}}
>
{analysisResult}
</Markdown>
<div className="flex items-center space-x-2 justify-end">
<Button
variant="secondary"
size="sm"
disabled={analyzing}
onClick={analyzeSetence}
>
{analyzing && (
<LoaderIcon className="animate-spin w-4 h-4 mr-2" />
)}
{t("reAnalyze")}
</Button>
<AIButton
prompt={text as string}
onReply={(replies) => {
const result = replies.map((m) => m.content).join("\n");
setAnalysisResult(result);
EnjoyApp.cacheObjects.set(`analyze-${md5(text)}`, result);
}}
tooltip={t("useAIAssistantToAnalyze")}
/>
</div>
</>
) : (
<div className="flex items-center justify-center space-x-2 py-4">
<Button size="sm" disabled={analyzing} onClick={analyzeSetence}>
{analyzing && <LoaderIcon className="animate-spin w-4 h-4 mr-2" />}
<span>{t("analyzeSetence")}</span>
</Button>
<AIButton
prompt={text as string}
onReply={(replies) => {
const result = replies.map((m) => m.content).join("\n");
setAnalysisResult(result);
EnjoyApp.cacheObjects.set(`analyze-${md5(text)}`, result);
}}
tooltip={t("useAIAssistantToAnalyze")}
/>
</div>
)}
</TabsContent>
</>
);
};

View File

@@ -57,7 +57,9 @@ export const MediaPlayerControls = () => {
setTranscriptionDraft,
} = useContext(MediaPlayerProviderContext);
const { EnjoyApp } = useContext(AppSettingsProviderContext);
const { currentHotkeys, enabled } = useContext(HotKeysSettingsProviderContext)
const { currentHotkeys, enabled } = useContext(
HotKeysSettingsProviderContext
);
const [playMode, setPlayMode] = useState<"loop" | "single" | "all">("single");
const [playbackRate, setPlaybackRate] = useState<number>(1);
const [grouping, setGrouping] = useState(false);
@@ -325,8 +327,7 @@ export const MediaPlayerControls = () => {
if (!decoded) return;
if (!wavesurfer) return;
setCurrentSegmentIndex(0);
const segment = transcription.result.timeline[0];
const segment = transcription.result.timeline[currentSegmentIndex];
wavesurfer.seekTo(
Math.floor((segment.startTime / wavesurfer.getDuration()) * 1e8) / 1e8
);
@@ -374,7 +375,13 @@ export const MediaPlayerControls = () => {
}, [wavesurfer, decoded, playMode, activeRegion, currentTime]);
useHotkeys(
[currentHotkeys.PlayOrPause, currentHotkeys.PlayPreviousSegment, currentHotkeys.PlayNextSegment, currentHotkeys.StartOrStopRecording, currentHotkeys.Compare],
[
currentHotkeys.PlayOrPause,
currentHotkeys.PlayPreviousSegment,
currentHotkeys.PlayNextSegment,
currentHotkeys.StartOrStopRecording,
currentHotkeys.Compare,
],
(keyboardEvent, hotkeyEvent) => {
if (!wavesurfer) return;
keyboardEvent.preventDefault();
@@ -396,8 +403,9 @@ export const MediaPlayerControls = () => {
document.getElementById("media-compare-button").click();
break;
}
},{
enabled
},
{
enabled,
},
[wavesurfer, currentHotkeys]
);

View File

@@ -18,7 +18,12 @@ import {
AlertDialogAction,
PingPoint,
} from "@renderer/components/ui";
import { LoaderIcon, CheckCircleIcon, MicIcon } from "lucide-react";
import {
LoaderIcon,
CheckCircleIcon,
MicIcon,
PencilLineIcon,
} from "lucide-react";
import { AlignmentResult } from "echogarden/dist/api/API.d.js";
import { formatDuration } from "@renderer/lib/utils";
@@ -40,6 +45,15 @@ export const MediaTranscription = () => {
const [recordingStats, setRecordingStats] =
useState<SegementRecordingStatsType>([]);
const [notesStats, setNotesStats] = useState<
{
targetId: string;
targetType: string;
count: number;
segment: SegmentType;
}[]
>([]);
const fetchSegmentStats = async () => {
if (!media) return;
@@ -48,6 +62,10 @@ export const MediaTranscription = () => {
.then((stats) => {
setRecordingStats(stats);
});
EnjoyApp.notes.groupBySegment(media.id, media.mediaType).then((stats) => {
setNotesStats(stats);
});
};
useEffect(() => {
@@ -134,8 +152,9 @@ export const MediaTranscription = () => {
<div
key={index}
id={`segment-${index}`}
className={`py-2 px-4 cursor-pointer hover:bg-yellow-400/10 ${currentSegmentIndex === index ? "bg-yellow-400/25" : ""
}`}
className={`py-2 px-4 cursor-pointer hover:bg-yellow-400/10 ${
currentSegmentIndex === index ? "bg-yellow-400/25" : ""
}`}
onClick={() => {
const duration = wavesurfer.getDuration();
wavesurfer.seekTo(
@@ -151,6 +170,9 @@ export const MediaTranscription = () => {
{(recordingStats || []).findIndex(
(s) => s.referenceId === index
) !== -1 && <MicIcon className="w-3 h-3 text-sky-500" />}
{(notesStats || []).findIndex(
(s) => s.segment?.segmentIndex === index
) !== -1 && <PencilLineIcon className="w-3 h-3 text-sky-500" />}
<span className="text-xs opacity-50">
{formatDuration(sentence.startTime, "s")}
</span>

View File

@@ -0,0 +1,4 @@
export * from './note-card';
export * from './note-form';
export * from './note-segment';
export * from './note-segment-group';

View File

@@ -0,0 +1,165 @@
import { AppSettingsProviderContext } from "@renderer/context";
import { useContext, useState } from "react";
import {
AlertDialog,
AlertDialogAction,
AlertDialogContent,
AlertDialogCancel,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
Button,
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuTrigger,
toast,
} from "@renderer/components/ui";
import { MoreHorizontalIcon } from "lucide-react";
import Markdown from "react-markdown";
import { t } from "i18next";
export const NoteCard = (props: {
note: NoteType;
onEdit?: (note: NoteType) => void;
}) => {
if (props.note.targetType === "Segment") {
return <SegmentNoteCard {...props} />;
}
};
export const SegmentNoteCard = (props: {
note: NoteType;
onEdit?: (note: NoteType) => void;
}) => {
const { note } = props;
return (
<div
id={`note-${note.id}`}
className="w-full rounded px-4 py-2 bg-muted/50"
>
<Markdown className="select-text prose prose-sm dark:prose-invert max-w-full mb-2">
{note.content}
</Markdown>
<div className="flex justify-between space-x-2">
{note.parameters?.quote ? (
<div className="flex">
<span className="text-muted-foreground text-sm px-1 border-b border-red-500 border-dashed">
{note.parameters.quote}
</span>
</div>
) : (
<div></div>
)}
<NoteActionsDropdownMenu {...props} />
</div>
</div>
);
};
const NoteActionsDropdownMenu = (props: {
note: NoteType;
onEdit?: (note: NoteType) => void;
}) => {
const { EnjoyApp, webApi } = useContext(AppSettingsProviderContext);
const { note, onEdit } = props;
const [deleting, setDeleting] = useState(false);
const [sharing, setSharing] = useState(false);
const handleDelete = () => {
EnjoyApp.notes.delete(note.id);
};
const handleShare = async () => {
try {
if (
note.segment &&
(!note.segment.syncedAt || !note.segment.uploadedAt)
) {
await EnjoyApp.segments.sync(note.segment.id);
}
if (!note.syncedAt) {
await EnjoyApp.notes.sync(note.id);
}
} catch (e) {
toast.error(t("shareFailed"), { description: e.message });
}
webApi
.createPost({
targetId: note.id,
targetType: "Note",
})
.then(() => {
toast.success(t("sharedSuccessfully"));
})
.catch((e) => {
toast.error(t("shareFailed"), { description: e.message });
});
};
return (
<>
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button variant="ghost" className="w-4 h-4 p-0">
<MoreHorizontalIcon className="w-4 h-4 text-muted-foreground" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent>
{onEdit && (
<DropdownMenuItem onClick={() => onEdit(note)}>
{t("edit")}
</DropdownMenuItem>
)}
<DropdownMenuItem onClick={() => setSharing(true)}>
{t("share")}
</DropdownMenuItem>
<DropdownMenuItem onClick={() => setDeleting(true)}>
{t("delete")}
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
<AlertDialog open={sharing} onOpenChange={(value) => setSharing(value)}>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>{t("shareNote")}</AlertDialogTitle>
<AlertDialogDescription>
{t("areYouSureToShareThisNoteToCommunity")}
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>{t("cancel")}</AlertDialogCancel>
<AlertDialogAction asChild>
<Button onClick={handleShare}>{t("share")}</Button>
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
<AlertDialog open={deleting} onOpenChange={(value) => setDeleting(value)}>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>{t("deleteNote")}</AlertDialogTitle>
<AlertDialogDescription>
{t("areYouSureToDeleteThisNote")}
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>{t("cancel")}</AlertDialogCancel>
<AlertDialogAction asChild>
<Button onClick={handleDelete}>{t("delete")}</Button>
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</>
);
};

View File

@@ -0,0 +1,110 @@
import { AppSettingsProviderContext } from "@renderer/context";
import { useContext, useEffect, useRef, useState } from "react";
import { Button, Textarea, toast } from "@renderer/components/ui";
import { t } from "i18next";
export const NoteForm = (props: {
segment: SegmentType;
note?: NoteType;
parameters: { quoteIndices: number[]; quote: string };
onParametersChange?: (parameters: any) => void;
onCancel?: () => void;
onSave?: (note: NoteType) => void;
}) => {
const { segment, note, parameters, onParametersChange, onCancel, onSave } =
props;
const [content, setContent] = useState<string>(note?.content ?? "");
const { EnjoyApp } = useContext(AppSettingsProviderContext);
const inputRef = useRef<HTMLTextAreaElement>(null);
const resizeTextarea = () => {
if (!inputRef.current) return;
inputRef.current.style.height = "auto";
inputRef.current.style.height = `${inputRef.current.scrollHeight}px`;
};
const handleSubmit = () => {
if (!content) return;
if (note) {
EnjoyApp.notes
.update(note.id, {
content,
parameters,
})
.then((note) => {
onSave && onSave(note);
})
.catch((err) => {
toast.error(err.message);
});
} else {
EnjoyApp.notes
.create({
targetId: segment.id,
targetType: "Segment",
parameters,
content,
})
.then((note) => {
onSave && onSave(note);
setContent("");
})
.catch((err) => {
toast.error(err.message);
});
}
};
useEffect(() => {
resizeTextarea();
}, [content]);
useEffect(() => {
if (!note) return;
if (note.parameters === parameters) return;
onParametersChange && onParametersChange(note.parameters);
}, [note]);
return (
<div className="w-full">
<div className="mb-2">
<Textarea
ref={inputRef}
className="w-full"
value={content}
placeholder={t("writeNoteHere")}
onChange={(e) => setContent(e.target.value)}
/>
</div>
<div className="flex items-center justify-between">
{parameters.quoteIndices?.length > 0 ? (
<div className="flex space-x-2">
<span className="text-sm px-1 rounded text-muted-foreground border-b border-red-500 border-dashed">
{parameters.quoteIndices
.map(
(index: number) => segment?.caption?.timeline?.[index]?.text
)
.join(" ")}
</span>
</div>
) : (
<div></div>
)}
<div className="flex space-x-2">
{note && (
<Button variant="secondary" size="sm" onClick={onCancel}>
{t("cancel")}
</Button>
)}
<Button disabled={!content} size="sm" onClick={handleSubmit}>
{t("save")}
</Button>
</div>
</div>
</div>
);
};

View File

@@ -0,0 +1,106 @@
import {
AudioLinesIcon,
ChevronDownIcon,
ChevronUpIcon,
VideoIcon,
} from "lucide-react";
import { Button, Separator } from "@renderer/components/ui";
import { t } from "i18next";
import { Link } from "react-router-dom";
import { useState } from "react";
import { useNotes } from "@/renderer/hooks";
import { NoteCard } from "./note-card";
import { NoteSemgent } from "./note-segment";
export const NoteSegmentGroup = (props: {
count: number;
segment: SegmentType;
}) => {
const { count, segment } = props;
const [collapsed, setCollapsed] = useState<boolean>(true);
const { notes, findNotes, hasMore } = useNotes({
targetId: segment.id,
targetType: "Segment",
});
return (
<div
className={`bg-background p-4 rounded-lg border transition-[shadow] ${
collapsed ? "" : "shadow-lg"
}`}
>
<div className="flex items-center space-x-4">
<div className="flex-1">
<div className="select-text line-clamp-3 text-muted-foreground font-serif pl-3 border-l-4 mb-4">
{segment.caption.text}
</div>
<div className="font-mono text-lg mb-4">
{t("notesCount", { count })}
</div>
<div className="flex justify-start text-sm text-muted-foreground">
<Link
to={`/${segment.targetType.toLowerCase()}s/${
segment.targetId
}?segmentIndex=${segment.segmentIndex}`}
>
{t("source")}: {t(segment.targetType.toLowerCase())}
</Link>
</div>
</div>
<div className="w-24 h-24 flex">
{segment.targetType === "Audio" && (
<AudioLinesIcon className="object-cover m-auto w-5/6 h-5/6 text-muted-foreground" />
)}
{segment.targetType === "Video" && (
<VideoIcon className="object-cover m-auto w-5/6 h-5/6 text-muted-foreground" />
)}
</div>
</div>
<div
className={`overflow-hidden transition-[height] ease-in-out duration-500 ${
collapsed ? "h-0" : "h-auto"
}`}
>
<Separator className="my-4" />
<div className="mb-4">
<NoteSemgent segment={segment} notes={notes} />
</div>
<div className="grid gap-2 mb-2">
{notes.map((note) => (
<NoteCard key={note.id} note={note} />
))}
</div>
{hasMore && (
<div className="flex justify-center mb-2">
<Button
onClick={() => findNotes({ offset: notes.length })}
variant="link"
size="sm"
>
{t("loadMore")}
</Button>
</div>
)}
</div>
<div className="flex items-center justify-center">
<Button
onClick={() => setCollapsed(!collapsed)}
variant="ghost"
size="icon"
className="p-0 w-6 h-6"
>
{collapsed ? (
<ChevronDownIcon className="w-5 h-5" />
) : (
<ChevronUpIcon className="w-5 h-5" />
)}
</Button>
</div>
</div>
);
};

View File

@@ -0,0 +1,80 @@
import { TimelineEntry } from "echogarden/dist/utilities/Timeline";
import { useState } from "react";
import { WavesurferPlayer } from "@renderer/components/widgets";
export const NoteSemgent = (props: {
segment: SegmentType;
notes: NoteType[];
}) => {
const { segment, notes } = props;
const caption: TimelineEntry = segment.caption;
const [notedquoteIndices, setNotedquoteIndices] = useState<number[]>([]);
let words = caption.text.split(" ");
const ipas = caption.timeline.map((w) =>
w.timeline.map((t) => t.timeline.map((s) => s.text))
);
if (words.length !== caption.timeline.length) {
words = caption.timeline.map((w) => w.text);
}
return (
<>
<div className="flex flex-wrap p-2 rounded-t-lg bg-muted/50 mb-4">
{/* use the words splitted by caption text if it is matched with the timeline length, otherwise use the timeline */}
{words.map((word, index) => (
<div
className=""
key={`note-segment-${segment.id}-${index}`}
id={`note-segment-${segment.id}-${index}`}
>
<div
className={`select-text font-serif text-lg xl:text-xl 2xl:text-2xl p-1 ${
notedquoteIndices.includes(index)
? "border-b border-red-500 border-dashed"
: ""
}
`}
>
{word}
</div>
<div
className={`select-text text-sm 2xl:text-base text-muted-foreground font-code mb-1 ${
index === 0 ? "before:content-['/']" : ""
} ${
index === caption.timeline.length - 1
? "after:content-['/']"
: ""
}`}
>
{ipas[index]}
</div>
{notes
.filter((note) => note.parameters.quoteIndices?.[0] === index)
.map((note) => (
<div
key={`note-${segment.id}-${note.id}`}
className="mb-1 text-xs 2xl:text-sm text-red-500 max-w-64 line-clamp-3 font-code cursor-pointer"
onMouseOver={() =>
setNotedquoteIndices(note.parameters.quoteIndices)
}
onMouseLeave={() => setNotedquoteIndices([])}
onClick={() =>
document.getElementById("note-" + note.id)?.scrollIntoView()
}
>
{note.parameters.quoteIndices[0] === index && note.content}
</div>
))}
</div>
))}
</div>
{segment.src && <WavesurferPlayer id={segment.id} src={segment.src} />}
</>
);
};

View File

@@ -1,9 +1,9 @@
export * from "./posts";
export * from "./post-actions";
export * from "./post-audio";
export * from "./post-card";
export * from "./post-medium";
export * from "./post-recording";
export * from "./post-story";
export * from "./post-note";
export * from "./post-options";
export * from "./post-actions";
export * from "./post-recording";
export * from "./post-story";

View File

@@ -1,12 +1,6 @@
import { useEffect, useState, useRef, useCallback, useContext } from "react";
import { useEffect, useState, useContext } from "react";
import { AppSettingsProviderContext } from "@renderer/context";
import { renderPitchContour } from "@renderer/lib/utils";
import { extractFrequencies } from "@/utils";
import WaveSurfer from "wavesurfer.js";
import { Button, Skeleton } from "@renderer/components/ui";
import { PlayIcon, PauseIcon } from "lucide-react";
import { useIntersectionObserver } from "@uidotdev/usehooks";
import { secondsToTimestamp } from "@renderer/lib/utils";
import { Button } from "@renderer/components/ui";
import { MediaPlayer, MediaProvider } from "@vidstack/react";
import {
DefaultAudioLayout,
@@ -16,6 +10,7 @@ import { STORAGE_WORKER_ENDPOINT } from "@/constants";
import { TimelineEntry } from "echogarden/dist/utilities/Timeline.d.js";
import { t } from "i18next";
import { XCircleIcon } from "lucide-react";
import { WavesurferPlayer } from "../widgets";
export const PostAudio = (props: {
audio: Partial<MediumType>;
@@ -44,6 +39,8 @@ export const PostAudio = (props: {
targetMd5: audio.md5,
})
.then((response) => {
const transcription = response?.transcriptions?.[0];
if (transcription.targetMd5 !== audio.md5) return;
setTranscription(response?.transcriptions?.[0]);
});
}, [audio.md5]);
@@ -70,7 +67,8 @@ export const PostAudio = (props: {
<WavesurferPlayer
currentTime={currentTime}
setCurrentTime={setCurrentTime}
audio={audio}
id={audio.id}
src={audio.sourceUrl}
height={height}
onError={(err) => setError(err.message)}
/>
@@ -103,136 +101,3 @@ export const PostAudio = (props: {
</div>
);
};
const WavesurferPlayer = (props: {
audio: Partial<MediumType>;
height?: number;
currentTime: number;
setCurrentTime: (currentTime: number) => void;
onError?: (error: Error) => void;
}) => {
const { audio, height = 80, onError, setCurrentTime } = props;
const [initialized, setInitialized] = useState(false);
const [isPlaying, setIsPlaying] = useState(false);
const [wavesurfer, setWavesurfer] = useState(null);
const containerRef = useRef();
const [ref, entry] = useIntersectionObserver({
threshold: 1,
});
const [duration, setDuration] = useState<number>(0);
const onPlayClick = useCallback(() => {
wavesurfer.isPlaying() ? wavesurfer.pause() : wavesurfer.play();
}, [wavesurfer]);
useEffect(() => {
// use the intersection observer to only create the wavesurfer instance
// when the player is visible
if (!entry?.isIntersecting) return;
if (!audio.sourceUrl) return;
if (wavesurfer) return;
const ws = WaveSurfer.create({
container: containerRef.current,
url: audio.sourceUrl,
height,
barWidth: 1,
cursorWidth: 0,
autoCenter: true,
autoScroll: true,
dragToSeek: true,
hideScrollbar: true,
minPxPerSec: 100,
waveColor: "#ddd",
progressColor: "rgba(0, 0, 0, 0.25)",
});
setWavesurfer(ws);
}, [audio.sourceUrl, entry]);
useEffect(() => {
if (!wavesurfer) return;
const subscriptions = [
wavesurfer.on("play", () => {
setIsPlaying(true);
}),
wavesurfer.on("pause", () => {
setIsPlaying(false);
}),
wavesurfer.on("timeupdate", (time: number) => {
setCurrentTime(time);
}),
wavesurfer.on("ready", () => {
setDuration(wavesurfer.getDuration());
const peaks = wavesurfer.getDecodedData().getChannelData(0);
const sampleRate = wavesurfer.options.sampleRate;
const data = extractFrequencies({ peaks, sampleRate });
setTimeout(() => {
renderPitchContour({
wrapper: wavesurfer.getWrapper(),
canvasId: `pitch-contour-${audio.id}-canvas`,
labels: new Array(data.length).fill(""),
datasets: [
{
data,
cubicInterpolationMode: "monotone",
pointRadius: 1,
},
],
});
}, 1000);
setInitialized(true);
}),
wavesurfer.on("error", (err: Error) => {
onError(err);
}),
];
return () => {
subscriptions.forEach((unsub) => unsub());
wavesurfer?.destroy();
};
}, [wavesurfer]);
return (
<>
<div className="flex justify-end">
<span className="text-xs text-muted-foreground">
{secondsToTimestamp(duration)}
</span>
</div>
<div
ref={ref}
className="bg-background rounded-lg grid grid-cols-9 items-center relative h-[80px]"
>
{!initialized && (
<div className="col-span-9 flex flex-col justify-around h-[80px]">
<Skeleton className="h-3 w-full rounded-full" />
<Skeleton className="h-3 w-full rounded-full" />
<Skeleton className="h-3 w-full rounded-full" />
</div>
)}
<div className={`flex justify-center ${initialized ? "" : "hidden"}`}>
<Button
onClick={onPlayClick}
className="aspect-square rounded-full p-2 w-12 h-12 bg-blue-600 hover:bg-blue-500"
>
{isPlaying ? (
<PauseIcon className="w-6 h-6 text-white" />
) : (
<PlayIcon className="w-6 h-6 text-white" />
)}
</Button>
</div>
<div
className={`col-span-8 ${initialized ? "" : "hidden"}`}
ref={containerRef}
></div>
</div>
</>
);
};

View File

@@ -6,6 +6,7 @@ import {
PostMedium,
PostStory,
PostOptions,
PostNote,
} from "@renderer/components";
import { Avatar, AvatarImage, AvatarFallback } from "@renderer/components/ui";
import { formatDateTime } from "@renderer/lib/utils";
@@ -97,6 +98,15 @@ export const PostCard = (props: {
</>
)}
{post.targetType == "Note" && (
<>
<div className="text-xs text-muted-foreground">
{t("sharedNote")}
</div>
<PostNote note={post.target as NoteType} />
</>
)}
<PostActions post={post} />
</div>
);

View File

@@ -0,0 +1,20 @@
import { NoteSemgent } from "@renderer/components";
export const PostNote = (props: { note: NoteType }) => {
const { note } = props;
return (
<div className="select-text">
<div className="select-text mb-2">{note.content}</div>
{note.parameters?.quote && (
<div className="mb-2">
<span className="text-sm text-muted-foreground select-text border-b border-red-500 border-dashed">
{note.parameters.quote}
</span>
</div>
)}
{note.segment && <NoteSemgent segment={note.segment} notes={[note]} />}
</div>
);
};

View File

@@ -18,7 +18,7 @@ export const Posts = (props: { userId?: string }) => {
const { webApi } = useContext(AppSettingsProviderContext);
const [loading, setLoading] = useState<boolean>(true);
const [type, setType] = useState<
"all" | "recording" | "medium" | "story" | "prompt" | "gpt"
"all" | "recording" | "medium" | "story" | "prompt" | "gpt" | "note"
>("all");
const [by, setBy] = useState<"all" | "following">("following");
const [posts, setPosts] = useState<PostType[]>([]);
@@ -109,6 +109,9 @@ export const Posts = (props: { userId?: string }) => {
<SelectItem key="recording" value="recording">
{t("recordingType")}
</SelectItem>
<SelectItem key="note" value="note">
{t("noteType")}
</SelectItem>
<SelectItem key="prompt" value="prompt">
{t("promptType")}
</SelectItem>
@@ -133,10 +136,10 @@ export const Posts = (props: { userId?: string }) => {
<div className="space-y-6">
{posts.map((post) => (
<>
<PostCard key={post.id} post={post} handleDelete={handleDelete} />
<div key={post.id}>
<PostCard post={post} handleDelete={handleDelete} />
<Separator />
</>
</div>
))}
</div>

View File

@@ -11,9 +11,15 @@ import {
} from "@renderer/components";
import { useVideo } from "@renderer/hooks";
export const VideoPlayer = (props: { id?: string; md5?: string }) => {
const { id, md5 } = props;
const { setMedia, layout } = useContext(MediaPlayerProviderContext);
export const VideoPlayer = (props: {
id?: string;
md5?: string;
segmentIndex?: number;
}) => {
const { id, md5, segmentIndex } = props;
const { setMedia, layout, setCurrentSegmentIndex } = useContext(
MediaPlayerProviderContext
);
const { video } = useVideo({ id, md5 });
useEffect(() => {
@@ -22,13 +28,20 @@ export const VideoPlayer = (props: { id?: string; md5?: string }) => {
setMedia(video);
}, [video]);
useEffect(() => {
if (!segmentIndex) return;
setCurrentSegmentIndex(segmentIndex);
}, []);
if (!layout) return <LoaderSpin />;
return (
<div data-testid="video-player" className={layout.wrapper}>
<div className={`${layout.upperWrapper} mb-4`}>
<div className="grid grid-cols-5 xl:grid-cols-3 gap-6 px-6 h-full">
<div className={`col-span-2 xl:col-span-1 rounded-lg border shadow-lg ${layout.upperWrapper}`}>
<div className="grid grid-cols-5 xl:grid-cols-3 gap-3 xl:gap-6 px-6 h-full">
<div
className={`col-span-2 xl:col-span-1 rounded-lg border shadow-lg ${layout.upperWrapper}`}
>
<MediaTabs />
</div>
<div className={`col-span-3 xl:col-span-2 ${layout.upperWrapper}`}>
@@ -38,11 +51,11 @@ export const VideoPlayer = (props: { id?: string; md5?: string }) => {
</div>
<div className={`${layout.lowerWrapper} flex flex-col`}>
<div className={`${layout.playerWrapper} py-2 px-6`}>
<div className={`${layout.playerWrapper} py-2 px-3 xl:px-6`}>
<MediaCurrentRecording />
</div>
<div className={`${layout.playerWrapper} py-2 px-6`}>
<div className={`${layout.playerWrapper} py-2 px-3 xl:px-6`}>
<MediaPlayer />
</div>

View File

@@ -1,13 +1,10 @@
export * from "./db-state";
export * from "./layout";
export * from "./sidebar";
export * from "./page-placeholder";
export * from "./login-form";
export * from "./loader-spin";
export * from "./lookup-result";
export * from "./login-form";
export * from "./no-records-found";
export * from "./page-placeholder";
export * from "./selection-menu";
export * from "./lookup-result";
export * from "./sidebar";
export * from "./wavesurfer-player";

View File

@@ -28,6 +28,7 @@ import {
LucideIcon,
HelpCircleIcon,
ExternalLinkIcon,
NotebookPenIcon,
} from "lucide-react";
import { useLocation, Link } from "react-router-dom";
import { t } from "i18next";
@@ -99,6 +100,14 @@ export const Sidebar = () => {
testid="sidebar-conversations"
/>
<SidebarItem
href="/notes"
label={t("sidebar.notes")}
tooltip={t("sidebar.notes")}
active={activeTab === "/notes"}
Icon={NotebookPenIcon}
/>
<SidebarItem
href="/vocabulary"
label={t("sidebar.vocabulary")}

View File

@@ -0,0 +1,141 @@
import { renderPitchContour, secondsToTimestamp } from "@renderer/lib/utils";
import { extractFrequencies } from "@/utils";
import { useIntersectionObserver } from "@uidotdev/usehooks";
import { useCallback, useEffect, useRef, useState } from "react";
import WaveSurfer from "wavesurfer.js";
import { Button, Skeleton } from "@renderer/components/ui";
import { PauseIcon, PlayIcon } from "lucide-react";
export const WavesurferPlayer = (props: {
id: string;
src: string;
height?: number;
currentTime?: number;
setCurrentTime?: (currentTime: number) => void;
onError?: (error: Error) => void;
}) => {
const { id, src, height = 80, onError, setCurrentTime } = props;
const [initialized, setInitialized] = useState(false);
const [isPlaying, setIsPlaying] = useState(false);
const [wavesurfer, setWavesurfer] = useState(null);
const containerRef = useRef();
const [ref, entry] = useIntersectionObserver({
threshold: 1,
});
const [duration, setDuration] = useState<number>(0);
const onPlayClick = useCallback(() => {
wavesurfer.isPlaying() ? wavesurfer.pause() : wavesurfer.play();
}, [wavesurfer]);
useEffect(() => {
// use the intersection observer to only create the wavesurfer instance
// when the player is visible
if (!entry?.isIntersecting) return;
if (!src) return;
if (wavesurfer) return;
const ws = WaveSurfer.create({
container: containerRef.current,
url: src,
height,
barWidth: 1,
cursorWidth: 0,
autoCenter: true,
autoScroll: true,
dragToSeek: true,
hideScrollbar: true,
minPxPerSec: 100,
waveColor: "#ddd",
progressColor: "rgba(0, 0, 0, 0.25)",
});
setWavesurfer(ws);
}, [src, entry]);
useEffect(() => {
if (!wavesurfer) return;
const subscriptions = [
wavesurfer.on("play", () => {
setIsPlaying(true);
}),
wavesurfer.on("pause", () => {
setIsPlaying(false);
}),
wavesurfer.on("timeupdate", (time: number) => {
setCurrentTime && setCurrentTime(time);
}),
wavesurfer.on("ready", () => {
setDuration(wavesurfer.getDuration());
const peaks = wavesurfer.getDecodedData().getChannelData(0);
const sampleRate = wavesurfer.options.sampleRate;
const data = extractFrequencies({ peaks, sampleRate });
setTimeout(() => {
renderPitchContour({
wrapper: wavesurfer.getWrapper(),
canvasId: `pitch-contour-${id}-canvas`,
labels: new Array(data.length).fill(""),
datasets: [
{
data,
cubicInterpolationMode: "monotone",
pointRadius: 1,
},
],
});
}, 1000);
setInitialized(true);
}),
wavesurfer.on("error", (err: Error) => {
onError(err);
}),
];
return () => {
subscriptions.forEach((unsub) => unsub());
wavesurfer?.destroy();
};
}, [wavesurfer]);
return (
<>
<div className="flex justify-end">
<span className="text-xs text-muted-foreground">
{secondsToTimestamp(duration)}
</span>
</div>
<div
ref={ref}
className="bg-background rounded-lg grid grid-cols-9 items-center relative h-[80px]"
>
{!initialized && (
<div className="col-span-9 flex flex-col justify-around h-[80px]">
<Skeleton className="h-3 w-full rounded-full" />
<Skeleton className="h-3 w-full rounded-full" />
<Skeleton className="h-3 w-full rounded-full" />
</div>
)}
<div className={`flex justify-center ${initialized ? "" : "hidden"}`}>
<Button
onClick={onPlayClick}
className="aspect-square rounded-full p-2 w-12 h-12 bg-blue-600 hover:bg-blue-500"
>
{isPlaying ? (
<PauseIcon className="w-6 h-6 text-white" />
) : (
<PlayIcon className="w-6 h-6 text-white" />
)}
</Button>
</div>
<div
className={`col-span-8 ${initialized ? "" : "hidden"}`}
ref={containerRef}
></div>
</div>
</>
);
};

View File

@@ -1,7 +1,12 @@
import { createContext, useEffect, useState, useContext } from "react";
import { extractFrequencies } from "@/utils";
import { AppSettingsProviderContext } from "@renderer/context";
import { useTranscriptions, useRecordings } from "@renderer/hooks";
import {
useTranscriptions,
useRecordings,
useSegments,
useNotes,
} from "@renderer/hooks";
import WaveSurfer from "wavesurfer.js";
import Regions, {
type Region as RegionType,
@@ -75,6 +80,12 @@ type MediaPlayerContextType = {
fetchRecordings: (offset: number) => void;
loadingRecordings: boolean;
hasMoreRecordings: boolean;
// Notes
currentNotes: NoteType[];
createNote: (params: any) => void;
// Segments
currentSegment: SegmentType;
createSegment: () => void;
};
export const MediaPlayerProviderContext =
@@ -163,6 +174,17 @@ export const MediaPlayerProvider = ({
hasMore: hasMoreRecordings,
} = useRecordings(media, currentSegmentIndex);
const { segment, createSegment } = useSegments({
targetId: media?.id,
targetType: media?.mediaType,
segmentIndex: currentSegmentIndex,
});
const { notes, createNote } = useNotes({
targetId: segment?.id,
targetType: "Segment",
});
const initializeWavesurfer = async () => {
if (!layout?.playerHeight) return;
if (!media) return;
@@ -507,7 +529,7 @@ export const MediaPlayerProvider = ({
useEffect(() => {
calculateHeight();
EnjoyApp.window.onResize((event, bounds) => {
EnjoyApp.window.onResize(() => {
deboundeCalculateHeight();
});
@@ -558,6 +580,10 @@ export const MediaPlayerProvider = ({
fetchRecordings,
loadingRecordings,
hasMoreRecordings,
currentNotes: notes,
createNote,
currentSegment: segment,
createSegment,
}}
>
{children}

View File

@@ -1,12 +1,10 @@
export * from './use-recordings';
export * from './use-transcribe';
export * from './use-transcriptions';
export * from './use-ai-command';
export * from './use-conversation';
export * from './use-audio';
export * from './use-video';
export * from './use-camdict';
export * from "./use-ai-command";
export * from "./use-audio";
export * from "./use-camdict";
export * from "./use-conversation";
export * from "./use-notes";
export * from "./use-recordings";
export * from "./use-segments";
export * from "./use-transcribe";
export * from "./use-transcriptions";
export * from "./use-video";

View File

@@ -0,0 +1,97 @@
import { useContext, useEffect, useState } from "react";
import { AppSettingsProviderContext, DbProviderContext } from "../context";
import { toast } from "@renderer/components/ui";
export const useNotes = (props: { targetId: string; targetType: string }) => {
const { targetId, targetType } = props;
const { EnjoyApp } = useContext(AppSettingsProviderContext);
const { addDblistener, removeDbListener } = useContext(DbProviderContext);
const [notes, setNotes] = useState<NoteType[]>([]);
const [hasMore, setHasMore] = useState<boolean>(true);
const findNotes = (params?: { offset: number; limit?: number }) => {
if (!targetId || !targetType) {
setNotes([]);
return;
}
const { offset = 0, limit = 100 } = params || {};
if (offset > 0 && !hasMore) return;
EnjoyApp.notes
.findAll({
targetId,
targetType,
limit,
offset,
})
.then((foundNotes) => {
if (offset === 0) {
setNotes(foundNotes);
} else {
setNotes([...notes, ...foundNotes]);
}
setHasMore(notes.length === limit);
})
.catch((err) => {
toast.error(err.message);
});
};
const createNote = (params: {
targetId: string;
targetType: string;
content: string;
}) => {
const { targetId, targetType, content } = params;
EnjoyApp.notes.create({
targetId,
targetType,
content,
});
};
const onNoteUpdate = (event: CustomEvent) => {
const { model, action, record } = event.detail || {};
if (model !== "Note") return;
if (action === "update") {
setNotes((notes) => {
const index = notes.findIndex((n) => n.id === record.id);
if (index > -1) {
notes[index] = record;
}
return [...notes];
});
} else if (action === "destroy") {
setNotes((notes) => {
return notes.filter((n) => n.id !== record.id);
});
} else if (action === "create") {
if (record.targetId === targetId && record.targetType === targetType) {
setNotes((notes) => {
return [record, ...notes];
});
}
}
};
useEffect(() => {
findNotes();
addDblistener(onNoteUpdate);
return () => {
removeDbListener(onNoteUpdate);
};
}, [targetId, targetType]);
return {
notes,
hasMore,
findNotes,
createNote,
};
};

View File

@@ -0,0 +1,54 @@
import { useContext, useEffect, useState } from "react";
import { AppSettingsProviderContext } from "../context";
import { toast } from "@renderer/components/ui";
export const useSegments = (props: {
targetId: string;
targetType: string;
segmentIndex: number;
}) => {
const { targetId, targetType, segmentIndex } = props;
const { EnjoyApp } = useContext(AppSettingsProviderContext);
const [segment, setSegment] = useState<SegmentType>();
const findSegment = () => {
if (!targetId || !targetType) return;
EnjoyApp.segments
.findAll({
targetId,
targetType,
segmentIndex,
})
.then((segments) => {
setSegment(segments[0]);
})
.catch((err) => {
toast.error(err.message);
});
};
const createSegment = () => {
EnjoyApp.segments
.create({
targetId,
targetType,
segmentIndex,
})
.then((segment) => {
setSegment(segment);
})
.catch((err) => {
toast.error(err.message);
});
};
useEffect(() => {
findSegment();
}, [targetId, targetType, segmentIndex]);
return {
segment,
createSegment,
};
};

View File

@@ -138,9 +138,10 @@ export function renderPitchContour(options: {
});
}
export function imgErrorToDefalut(e: React.SyntheticEvent<HTMLImageElement, Event>) {
export function imgErrorToDefalut(
e: React.SyntheticEvent<HTMLImageElement, Event>
) {
const target = e.target as HTMLImageElement;
target.onerror = null;
target.onerror = null;
target.src = "assets/default-img.jpg";
}

View File

@@ -1,4 +1,4 @@
import { useParams, useNavigate } from "react-router-dom";
import { useParams, useNavigate, useSearchParams } from "react-router-dom";
import { AudioPlayer } from "@renderer/components";
import { Button } from "@renderer/components/ui";
import { ChevronLeftIcon } from "lucide-react";
@@ -8,6 +8,8 @@ import { MediaPlayerProvider } from "@renderer/context";
export default () => {
const navigate = useNavigate();
const { id } = useParams<{ id: string }>();
const [searchParams] = useSearchParams();
const segmentIndex = searchParams.get("segmentIndex") || "0";
return (
<>
@@ -20,7 +22,7 @@ export default () => {
</div>
<MediaPlayerProvider>
<AudioPlayer id={id} />
<AudioPlayer id={id} segmentIndex={parseInt(segmentIndex)} />
</MediaPlayerProvider>
</div>
</>

View File

@@ -230,7 +230,7 @@ export default () => {
{conversations.map((conversation) => (
<Link key={conversation.id} to={`/conversations/${conversation.id}`}>
<div
className="bg-muted text-muted-foreground rounded-full w-full mb-2 p-4 hover:bg-primary hover:text-muted cursor-pointer flex items-center"
className="bg-background hover:bg-muted hover:text-muted-foreground border rounded-full w-full mb-2 p-4 cursor-pointer flex items-center"
style={{
borderLeftColor: `#${conversation.id
.replaceAll("-", "")

View File

@@ -0,0 +1,88 @@
import { ChevronLeftIcon } from "lucide-react";
import { Button, toast } from "../components/ui";
import { t } from "i18next";
import { useNavigate } from "react-router-dom";
import { useContext, useEffect, useState } from "react";
import { AppSettingsProviderContext } from "../context";
import { NoteSegmentGroup } from "../components";
export default function Notes() {
const navigate = useNavigate();
const [groups, setGroups] = useState<
{
targetId: string;
targetType: string;
count: number;
segment?: SegmentType;
}[]
>([]);
const [hasMore, setHasMore] = useState<boolean>(true);
const { EnjoyApp } = useContext(AppSettingsProviderContext);
const findNotesGroup = (params?: { offset: number; limit?: number }) => {
const { offset = 0, limit = 5 } = params || {};
if (offset > 0 && !hasMore) return;
EnjoyApp.notes
.groupByTarget({
limit,
offset,
})
.then((noteGroups) => {
if (offset === 0) {
setGroups(noteGroups);
} else {
setGroups([...groups, ...noteGroups]);
}
setHasMore(groups.length === limit);
})
.catch((err) => {
toast.error(err.message);
});
};
useEffect(() => {
findNotesGroup({ offset: 0 });
}, []);
return (
<div className="min-h-[100vh] w-full max-w-5xl mx-auto px-4 py-6 lg:px-8">
<div className="w-full max-w-screen-md mx-auto">
<div className="flex space-x-1 items-center mb-4">
<Button variant="ghost" size="icon" onClick={() => navigate(-1)}>
<ChevronLeftIcon className="w-5 h-5" />
</Button>
<span>{t("sidebar.notes")}</span>
</div>
{groups.length === 0 && (
<div className="flex justify-center">
<div className="my-4 text-muted-foreground text-sm">{t("noNotesYet")}</div>
</div>
)}
<div className="flex flex-col space-y-4">
{groups.map((group) => (
<NoteSegmentGroup
key={group.targetId}
count={group.count}
segment={group.segment}
/>
))}
</div>
{hasMore && (
<div className="flex justify-center mt-4">
<Button
variant="secondary"
onClick={() => findNotesGroup({ offset: groups.length })}
>
{t("loadMore")}
</Button>
</div>
)}
</div>
</div>
);
}

View File

@@ -1,4 +1,4 @@
import { useParams, useNavigate } from "react-router-dom";
import { useParams, useNavigate, useSearchParams } from "react-router-dom";
import { VideoPlayer } from "@renderer/components";
import { Button } from "@renderer/components/ui";
import { ChevronLeftIcon } from "lucide-react";
@@ -8,6 +8,8 @@ import { MediaPlayerProvider } from "@renderer/context";
export default () => {
const navigate = useNavigate();
const { id } = useParams<{ id: string }>();
const [searchParams] = useSearchParams();
const segmentIndex = searchParams.get("segmentIndex") || "0";
return (
<>
@@ -20,7 +22,7 @@ export default () => {
</div>
<MediaPlayerProvider>
<VideoPlayer id={id} />
<VideoPlayer id={id} segmentIndex={parseInt(segmentIndex)} />
</MediaPlayerProvider>
</div>
</>

View File

@@ -17,6 +17,7 @@ import User from "./pages/user";
import Home from "./pages/home";
import Community from "./pages/community";
import StoryPreview from "./pages/story-preview";
import Notes from "./pages/notes";
export default createHashRouter([
{
@@ -81,6 +82,10 @@ export default createHashRouter([
path: "/stories/preview/:uri",
element: <StoryPreview />,
},
{
path: "/notes",
element: <Notes />,
},
],
},
{ path: "/landing", element: <Landing /> },

View File

@@ -267,4 +267,35 @@ type EnjoyAppType = {
find: (id: string) => Promise<WaveFormDataType>;
save: (id: string, data: WaveFormDataType) => Promise<void>;
};
segments: {
findAll: (params: any) => Promise<SegmentType[]>;
find: (id: string) => Promise<SegmentType>;
create: (params: {
targetId: string;
targetType: string;
segmentIndex: number;
}) => Promise<SegmentType>;
sync: (id: string) => Promise<SegmentType>;
};
notes: {
groupByTarget: (params: any) => Promise<any>;
groupBySegment: (targetId: string, targetType: string) => Promise<any>;
findAll: (params: any) => Promise<NoteType[]>;
find: (id: string) => Promise<NoteType>;
update: (
id: string,
params: {
content: string;
parameters?: any;
}
) => Promise<NoteType>;
delete: (id: string) => Promise<void>;
create: (params: {
targetId: string;
targetType: string;
content: string;
parameters?: any;
}) => Promise<NoteType>;
sync: (id: string) => Promise<NoteType>;
};
};

14
enjoy/src/types/note.d.ts vendored Normal file
View File

@@ -0,0 +1,14 @@
type NoteType = {
id: string;
targetId: string;
targetType: string;
content: string;
parameters: any;
syncedAt: Date;
uploadedAt: Date;
updatedAt: Date;
createdAt: Date;
segment?: SegmentType;
isSynced?: boolean;
sync(): Promise<void>;
};

View File

@@ -1,7 +1,7 @@
type PostType = {
id: string;
metadata: {
type: "text" | "prompt" | "gpt";
type: "text" | "prompt" | "gpt" | "note";
content:
| string
| {
@@ -11,7 +11,7 @@ type PostType = {
user: UserType;
targetType?: string;
targetId?: string;
target?: MediumType | StoryType | RecordingType;
target?: MediumType | StoryType | RecordingType | NoteType;
createdAt: Date;
updatedAt: Date;
};

18
enjoy/src/types/segment.d.ts vendored Normal file
View File

@@ -0,0 +1,18 @@
type SegmentType = {
id: string;
targetId: string;
targetType: string;
caption: TimelineEntry;
audio?: AudioType;
video?: VideoType;
segmentIndex: number;
md5: string;
caption: TimeLIne;
startTime: number;
endTime: number;
src: string;
syncedAt?: Date;
uploadedAt?: Date
updatedAt: Date
createdAt: Date
};

View File

@@ -2,6 +2,7 @@ type TranscriptionType = {
id: string;
targetId: string;
targetType: string;
targetMd5?: string;
state: "pending" | "processing" | "finished";
engine: string;
model: string;