add enjoy app

enjoy/src/types/audio.d.ts

@@ -0,0 +1,17 @@
+type AudioType = {
+  id: string;
+  source: string;
+  name: string;
+  description?: string;
+  src?: string;
+  coverUrl?: string;
+  md5: string;
+  metadata?: Ffmpeg.FfprobeData;
+  transcribed?: boolean;
+  transcribing?: boolean;
+  recordingsCount?: number;
+  recordingsDuration?: number;
+  uploadedAt?: Date;
+  createdAt: Date;
+  updatedAt: Date;
+};

enjoy/src/types/conversation.d.ts

@@ -0,0 +1,9 @@
+type ConversationType = {
+  id: string;
+  engine: "openai" | "ollama" | "googleGenerativeAi";
+  name: string;
+  configuration: { [key: string]: any };
+  model: string;
+  messages?: MessageType[];
+  createdAt?: string;
+};

enjoy/src/types/enjoy-app.d.ts

@@ -0,0 +1,261 @@
+type EnjoyAppType = {
+  app: {
+    reset: () => Promise<void>;
+    relaunch: () => Promise<void>;
+    reload: () => Promise<void>;
+    isPackaged: () => Promise<boolean>;
+    apiUrl: () => Promise<string>;
+    version: string;
+  };
+  providers: {
+    audible: {
+      bestsellers: (params?: {
+        page?: number;
+        category?: number;
+        pageSize?: number;
+      }) => Promise<{
+        books: AudibleBookType[];
+        page: number;
+        nextPage: number | undefined;
+      }>;
+    };
+    ted: {
+      talks: () => Promise<TedTalkType[]>;
+      ideas: () => Promise<TedIdeaType[]>;
+      downloadTalk: (url: string) => Promise<{ audio: string; video: string }>;
+    };
+  };
+  view: {
+    load: (url: string, bounds?: object) => Promise<void>;
+    show: (bounds: object) => Promise<void>;
+    hide: () => Promise<void>;
+    remove: () => Promise<void>;
+    scrape: (url: string) => Promise<void>;
+    onViewState: (
+      callback: (
+        event,
+        state: { state: string; error?: string; url?: string; html?: string }
+      ) => void
+    ) => void;
+    removeViewStateListeners: () => void;
+  };
+  onNotification: (
+    callback: (event, notification: NotificationType) => void
+  ) => void;
+  shell: {
+    openExternal: (url: string) => Promise<void>;
+    openPath: (path: string) => Promise<void>;
+  };
+  dialog: {
+    showOpenDialog: (
+      options: Electron.OpenDialogOptions
+    ) => Promise<string[] | undefined>;
+    showSaveDialog: (
+      options: Electron.SaveDialogOptions
+    ) => Promise<Electron.SaveDialogReturnValue>;
+    showMessageBox: (
+      options: Electron.MessageBoxOptions
+    ) => Promise<Electron.MessageBoxReturnValue>;
+    showErrorBox: (title: string, content: string) => Promise<void>;
+  };
+  settings: {
+    getLibrary: () => Promise<string>;
+    setLibrary: (library: string) => Promise<void>;
+    getUser: () => Promise<UserType>;
+    setUser: (user: UserType) => Promise<void>;
+    getWhisperModel: () => Promise<string>;
+    setWhisperModel: (model: string) => Promise<void>;
+    getWhisperModelsPath: () => Promise<string>;
+    getUserDataPath: () => Promise<string>;
+    getLlm: (provider: SupportedLlmProviderType) => Promise<LlmProviderType>;
+    setLlm: (
+      provider: SupportedLlmProviderType,
+      LlmProviderType
+    ) => Promise<void>;
+    getFfmpegConfig: () => Promise<FfmpegConfigType>;
+    setFfmpegConfig: () => Promise<void>;
+  };
+  fs: {
+    ensureDir: (path: string) => Promise<boolean>;
+  };
+  path: {
+    join: (...paths: string[]) => Promise<string>;
+  };
+  db: {
+    init: () => Promise<DbState>;
+    onTransaction: (
+      callback: (event, state: TransactionStateType) => void
+    ) => Promise<void>;
+    removeListeners: () => Promise<void>;
+  };
+  audios: {
+    findAll: (params: object) => Promise<AudioType[]>;
+    findOne: (params: object) => Promise<AudioType>;
+    create: (source: string, params?: object) => Promise<AudioType>;
+    update: (id: string, params: object) => Promise<AudioType | undefined>;
+    destroy: (id: string) => Promise<undefined>;
+    transcribe: (id: string) => Promise<void>;
+    upload: (id: string) => Promise<void>;
+  };
+  videos: {
+    findAll: (params: object) => Promise<VideoType[]>;
+    findOne: (params: object) => Promise<VideoType>;
+    create: (source: string, params?: object) => Promise<VideoType>;
+    update: (id: string, params: object) => Promise<VideoType | undefined>;
+    destroy: (id: string) => Promise<undefined>;
+    transcribe: (id: string) => Promise<void>;
+    upload: (id: string) => Promise<void>;
+  };
+  recordings: {
+    findAll: (where: object) => Promise<RecordingType[]>;
+    findOne: (where: object) => Promise<RecordingType>;
+    create: (params: object) => Promise<RecordingType>;
+    update: (id: string, params: object) => Promise<RecordingType | undefined>;
+    destroy: (id: string) => Promise<void>;
+    upload: (id: string) => Promise<void>;
+    assess: (id: string) => Promise<void>;
+    stats: (params: { from: string; to: string }) => Promise<{
+      count: number;
+      duration: number;
+    }>;
+    groupByDate: (params: { from: string; to: string }) => Promise<
+      {
+        date: string;
+        count: number;
+        level?: number;
+      }[]
+    >;
+    groupByTarget: (params: { from: string; to: string }) => Promise<
+      {
+        date: string;
+        targetId: string;
+        targetType: string;
+        count: number;
+        duration: number;
+        target: AudioType | VideoType;
+      }[]
+    >;
+    groupBySegment: (
+      targetId: string,
+      targetType
+    ) => Promise<SegementRecordingStatsType>;
+  };
+  conversations: {
+    findAll: (params: object) => Promise<ConversationType[]>;
+    findOne: (params: object) => Promise<ConversationType>;
+    create: (params: object) => Promise<ConversationType>;
+    update: (id: string, params: object) => Promise<ConversationType>;
+    destroy: (id: string) => Promise<void>;
+    ask: (
+      id: string,
+      params: {
+        messageId?: string;
+        content?: string;
+        file?: string;
+        blob?: {
+          type: string;
+          arrayBuffer: ArrayBuffer;
+        };
+      }
+    ) => Promise<MessageType>;
+  };
+  messages: {
+    findAll: (params: object) => Promise<MessageType[]>;
+    findOne: (params: object) => Promise<MessageType>;
+    destroy: (id: string) => Promise<void>;
+    createSpeech: (id: string, configuration?: any) => Promise<SpeechType>;
+  };
+  whisper: {
+    availableModels: () => Promise<string[]>;
+    downloadModel: (name: string) => Promise<any>;
+    transcribe: (
+      blob: { type: string; arrayBuffer: ArrayBuffer },
+      prompt?: string
+    ) => Promise<{ file: string; content: string }>;
+  };
+  ffmpeg: {
+    download: () => Promise<FfmpegConfigType>;
+  };
+  download: {
+    onState: (callback: (event, state) => void) => void;
+    cancel: (filename: string) => void;
+    cancelAll: () => void;
+    dashboard: () => Promise<DownloadStateType[]>;
+    removeAllListeners: () => void;
+  };
+  webApi: {
+    auth: (params: { provider: string; code: string }) => Promise<UserType>;
+    me: () => Promise<UserType>;
+    lookup: (params: {
+      word: string;
+      context?: string;
+      sourceId?: string;
+      sourceType?: string;
+    }) => Promise<LookupType>;
+    lookupInBatch: (
+      params: {
+        word: string;
+        context?: string;
+        sourceId?: string;
+        sourceType?: string;
+      }[]
+    ) => Promise<{ successCount: number; errors: string[]; total: number }>;
+    mineMeanings: (params?: {
+      page?: number;
+      items?: number;
+      sourceId?: string;
+      sourceType?: string;
+    }) => Promise<
+      {
+        meanings: MeaningType[];
+      } & PagyResponseType
+    >;
+    createStory: (params: {
+      title: string;
+      content: string;
+      url: string;
+      metadata: {
+        [key: string]: any;
+      };
+    }) => Promise<StoryType>;
+    extractVocabularyFromStory: (id: string) => Promise<string[]>;
+    story: (id: string) => Promise<StoryType>;
+    stories: (params?: { page: number }) => Promise<{
+      stories: StoryType[];
+      page: number;
+      next: number | null;
+    }>;
+    mineStories: (params?: { page: number }) => Promise<{
+      stories: StoryType[];
+      page: number;
+      next: number | null;
+    }>;
+    storyMeanings: (
+      storyId: string,
+      params?: {
+        page?: number;
+        items?: number;
+        sourceId?: string;
+        sourceType?: string;
+      }
+    ) => Promise<
+      {
+        meanings: MeaningType[];
+        pendingLookups: LookupType[];
+      } & PagyResponseType
+    >;
+    starStory: (id: string) => Promise<{ starred: boolean }>;
+    unstarStory: (id: string) => Promise<{ starred: boolean }>;
+  };
+  cacheObjects: {
+    get: (key: string) => Promise<any>;
+    set: (key: string, value: any, ttl?: number) => Promise<void>;
+    delete: (key: string) => Promise<void>;
+    clear: () => Promise<void>;
+  };
+  transcriptions: {
+    findOrCreate: (params: any) => Promise<TranscriptionType>;
+    process: (params: any) => Promise<void>;
+    update: (id: string, params: any) => Promise<void>;
+  };
+};

enjoy/src/types/message.d.ts

@@ -0,0 +1,11 @@
+type MessageType = {
+  id: string;
+  role: "system" | "assistant" | "user";
+  content: string;
+  conversationId: string;
+  conversation?: ConversationType;
+  createdAt?: string;
+  status?: "pending" | "success" | "error";
+  speeches?: Partial<SpeechType>[];
+  recording?: RecordingType;
+};

enjoy/src/types/pronunciation-assessment.d.ts

@@ -0,0 +1,64 @@
+type PronunciationAssessmentType = {
+  id: string;
+  recordingId: string;
+  accuracyScore: number;
+  completenessScore: number;
+  fluencyScore: number;
+  pronunciationScore: number;
+  prosodyScore?: number;
+  grammarScore?: number;
+  vocabularyScore?: number;
+  topicScore?: number;
+  result: {
+    confidence: number;
+    display: string;
+    itn: string;
+    lexical: string;
+    markedItn: string;
+    pronunciationAssessment: {
+      accuracyScore: number;
+      completenessScore: number;
+      fluencyScore: number;
+      pronScore: number;
+    };
+    words: PronunciationAssessmentWordResultType[];
+  };
+  syncedAt: Date;
+  createdAt: Date;
+  updatedAt: Date;
+  isSynced: boolean;
+  recording?: RecodingType;
+};
+
+type PronunciationAssessmentWordResultType = {
+  duration: number;
+  offset: number;
+  word: string;
+  pronunciationAssessment: {
+    accuracyScore: number;
+    errorType?:
+      | "None"
+      | "Omission"
+      | "Insertion"
+      | "Mispronunciation"
+      | "UnexpectedBreak"
+      | "MissingBreak"
+      | "Monotone";
+  };
+  phonemes: {
+    duration: number;
+    offset: number;
+    phoneme: string;
+    pronunciationAssessment: {
+      accuracyScore: number;
+    };
+  }[];
+  syllables: {
+    duration: number;
+    offset: number;
+    syllable: string;
+    pronunciationAssessment: {
+      accuracyScore: number;
+    };
+  };
+};

enjoy/src/types/recording.d.ts

@@ -0,0 +1,25 @@
+type RecordingType = {
+  id: string;
+  filename: string;
+  target?: AudioType | (MessageType & any);
+  targetId: string;
+  targetType: string;
+  pronunciationAssessment?: PronunciationAssessmentType & any;
+  segmentIndex: number;
+  segmentText?: string;
+  duration?: number;
+  src?: string;
+  md5: string;
+  uploadedAt?: Date;
+  createdAt: Date;
+  updatedAt: Date;
+};
+
+type SegementRecordingStatsType = {
+  targetId: string;
+  targetType: string;
+  referenceId: number;
+  referenceText?: string;
+  count: number;
+  duration: number;
+}[];

enjoy/src/types/speech.d.ts

@@ -0,0 +1,16 @@
+type SpeechType = {
+  id: string;
+  sourceId: string;
+  sourceType: string;
+  source?: MessageType;
+  text: string;
+  engine: string;
+  model: string;
+  voice: string;
+  md5: string;
+  filename: string;
+  filePath: string;
+  src?: string;
+  createdAt: Date;
+  updatedAt: Date;
+};

enjoy/src/types/transcription.d.ts

@@ -0,0 +1,24 @@
+type TranscriptionType = {
+  id: string;
+  targetId: string;
+  targetType: string;
+  state: "pending" | "processing" | "finished";
+  model: string;
+  result: TranscriptionResultSegmentGroupType[];
+};
+
+type TranscriptionResultSegmentType = {
+  offsets: {
+    from: number;
+    to: number;
+  };
+  text: string;
+  timestamps: {
+    from: string;
+    to: string;
+  };
+};
+
+type TranscriptionResultSegmentGroupType = TranscriptionResultSegmentType & {
+  segments: TranscriptionResultSegmentType[];
+};

enjoy/src/types/user.d.ts

@@ -0,0 +1,6 @@
+type UserType = {
+  id: string;
+  name: string;
+  avatarUrl?: string;
+  accessToken?: string;
+};

enjoy/src/types/video.d.ts

@@ -0,0 +1,18 @@
+type VideoType = {
+  id: string;
+  source: string;
+  name: string;
+  description?: string;
+  filename?: string;
+  coverUrl?: string;
+  md5: string;
+  src?: string;
+  metadata?: Ffmpeg.FfprobeData;
+  transcribed: boolean;
+  transcribing: boolean;
+  recordingsCount?: number;
+  recordingsDuration?: number;
+  uploadedAt?: Date;
+  createdAt: Date;
+  updatedAt: Date;
+};