lostecho/everyone-can-use-english
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat: save waveform as file (#118)

Browse Source 
* package rpm

* cache waveform data as file in library

* clear waveform data in db

* fix some css
This commit is contained in:
an-lee
2024-01-15 16:57:44 +08:00
committed by GitHub
parent 187038c42e
commit b545ea2362
25 changed files with 133 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
enjoy/components.json
View File

@@ -7,7 +7,7 @@
"config": "tailwind.config.js",
"css": "src/index.css",
"baseColor": "zinc",
"cssVariables": true
"cssVariables": false
},
"aliases": {
"components": "src/renderer/components",

20
enjoy/src/main/db/index.ts
View File

@@ -51,9 +51,6 @@ db.connect = async () => {
db.connection = sequelize;
// vacuum the database
await sequelize.query("VACUUM");
const umzug = new Umzug({
migrations: { glob: __dirname + "/migrations/*.js" },
context: sequelize.getQueryInterface(),
@@ -68,6 +65,23 @@ db.connect = async () => {
await sequelize.sync();
await sequelize.authenticate();
// TODO:
// clear the large waveform data in DB.
// Remove this in next release
const caches = await CacheObject.findAll({
attributes: ["id", "key"],
});
const cacheIds: string[] = [];
caches.forEach((cache) => {
if (cache.key.startsWith("waveform")) {
cacheIds.push(cache.id);
}
});
await CacheObject.destroy({ where: { id: cacheIds } });
// vacuum the database
await sequelize.query("VACUUM");
// register handlers
audiosHandler.register();
cacheObjectsHandler.register();

38
enjoy/src/main/waveform.ts Normal file
View File

@@ -0,0 +1,38 @@
import { ipcMain } from "electron";
import settings from "@main/settings";
import path from "path";
import fs from "fs-extra";
export class Waveform {
public dir = path.join(settings.libraryPath(), "waveforms");
constructor() {
fs.ensureDirSync(this.dir);
}
find(id: string) {
const file = path.join(this.dir, id + ".waveform.json");
if (fs.existsSync(file)) {
return fs.readJsonSync(file);
} else {
return null;
}
}
save(id: string, data: WaveFormDataType) {
const file = path.join(this.dir, id + ".waveform.json");
fs.writeJsonSync(file, data);
}
registerIpcHandlers() {
ipcMain.handle("waveforms-find", async (_event, id) => {
return this.find(id);
});
ipcMain.handle("waveforms-save", (_event, id, data) => {
return this.save(id, data);
});
}
}

5
enjoy/src/main/window.ts
View File

@@ -18,6 +18,7 @@ import log from "electron-log/main";
import { WEB_API_URL } from "@/constants";
import { AudibleProvider, TedProvider } from "@main/providers";
import { FfmpegDownloader } from "@main/ffmpeg";
import { Waveform } from "./waveform";
log.initialize({ preload: true });
const logger = log.scope("window");
@@ -25,6 +26,7 @@ const logger = log.scope("window");
const audibleProvider = new AudibleProvider();
const tedProvider = new TedProvider();
const ffmpegDownloader = new FfmpegDownloader();
const waveform = new Waveform();
const main = {
win: null as BrowserWindow | null,
@@ -46,6 +48,9 @@ main.init = () => {
// Whisper
whisper.registerIpcHandlers();
// Waveform
waveform.registerIpcHandlers();
// Downloader
downloader.registerIpcHandlers();

8
enjoy/src/preload.ts
View File

@@ -399,4 +399,12 @@ contextBridge.exposeInMainWorld("__ENJOY_APP__", {
return ipcRenderer.invoke("transcriptions-update", id, params);
},
},
waveforms: {
find: (id: string) => {
return ipcRenderer.invoke("waveforms-find", id);
},
save: (id: string, data: WaveFormDataType) => {
return ipcRenderer.invoke("waveforms-save", id, data);
},
}
});

2
enjoy/src/renderer/app.tsx
View File

@@ -53,7 +53,7 @@ function App() {
<AISettingsProvider>
<DbProvider>
<RouterProvider router={router} />
<Toaster richColors position="top-center" />
<Toaster richColors closeButton position="top-center" />
<Tooltip id="global-tooltip" />
</DbProvider>
</AISettingsProvider>

4
enjoy/src/renderer/components/audios/audio-detail.tsx
View File

@@ -132,7 +132,7 @@ export const AudioDetail = (props: { id?: string; md5?: string }) => {
mediaId={audio.id}
mediaType="Audio"
mediaUrl={audio.src}
waveformCacheKey={`waveform-audio-${audio.md5}`}
mediaMd5={audio.md5}
transcription={transcription}
currentTime={currentTime}
setCurrentTime={setCurrentTime}
@@ -207,7 +207,7 @@ export const AudioDetail = (props: { id?: string; md5?: string }) => {
</AlertDialog>
{!initialized && (
<div className="top-0 w-full h-full absolute z-30 bg-white/10 flex items-center justify-center">
<div className="top-0 w-full h-full absolute z-30 bg-background/10 flex items-center justify-center">
<LoaderIcon className="text-muted-foreground animate-spin w-8 h-8" />
</div>
)}

2
enjoy/src/renderer/components/conversations/conversations-shortcut.tsx
View File

@@ -48,7 +48,7 @@ export const ConversationsShortcut = (props: {
<div
key={conversation.id}
onClick={() => ask(conversation)}
className="bg-white text-primary rounded-full w-full mb-2 py-2 px-4 hover:bg-primary hover:text-white cursor-pointer flex items-center border"
className="bg-background text-primary rounded-full w-full mb-2 py-2 px-4 hover:bg-primary hover:text-white cursor-pointer flex items-center border"
style={{
borderLeftColor: `#${conversation.id
.replaceAll("-", "")

2
enjoy/src/renderer/components/conversations/speech-player.tsx
View File

@@ -89,7 +89,7 @@ export const SpeechPlayer = (props: {
</div>
<div
ref={ref}
className="bg-white rounded-lg grid grid-cols-9 items-center relative pl-2 h-[100px]"
className="bg-background rounded-lg grid grid-cols-9 items-center relative pl-2 h-[100px]"
>
{!initialized && (
<div className="col-span-9 flex flex-col justify-around h-[80px]">

23
enjoy/src/renderer/components/medias/media-player.tsx
View File

@@ -34,7 +34,7 @@ export const MediaPlayer = (props: {
mediaId: string;
mediaType: "Audio" | "Video";
mediaUrl: string;
waveformCacheKey: string;
mediaMd5?: string;
transcription: TranscriptionType;
// player controls
currentTime: number;
@@ -67,7 +67,7 @@ export const MediaPlayer = (props: {
mediaId,
mediaType,
mediaUrl,
waveformCacheKey,
mediaMd5,
transcription,
height = 200,
currentTime,
@@ -94,12 +94,7 @@ export const MediaPlayer = (props: {
if (!mediaUrl) return;
const [wavesurfer, setWavesurfer] = useState(null);
const [waveform, setWaveForm] = useState<{
peaks: number[];
duration: number;
frequencies: number[];
sampleRate: number;
}>(null);
const [waveform, setWaveForm] = useState<WaveFormDataType>(null);
const containerRef = useRef<HTMLDivElement>();
const [mediaProvider, setMediaProvider] = useState<
HTMLAudioElement | HTMLVideoElement
@@ -181,7 +176,7 @@ export const MediaPlayer = (props: {
const renderPitchContour = (region: RegionType) => {
if (!region) return;
if (!waveform.frequencies.length) return;
if (!waveform?.frequencies?.length) return;
if (!wavesurfer) return;
const duration = wavesurfer.getDuration();
@@ -280,7 +275,6 @@ export const MediaPlayer = (props: {
const ws = WaveSurfer.create({
container: containerRef.current,
height,
url: mediaUrl,
waveColor: "#ddd",
progressColor: "rgba(0, 0, 0, 0.25)",
cursorColor: "#dc143c",
@@ -324,6 +318,7 @@ export const MediaPlayer = (props: {
const subscriptions = [
wavesurfer.on("play", () => setIsPlaying(true)),
wavesurfer.on("pause", () => setIsPlaying(false)),
wavesurfer.on("loading", (percent: number) => console.log(percent)),
wavesurfer.on("timeupdate", (time: number) => setCurrentTime(time)),
wavesurfer.on("decode", () => {
if (waveform?.frequencies) return;
@@ -340,7 +335,7 @@ export const MediaPlayer = (props: {
sampleRate,
frequencies: _frequencies,
};
EnjoyApp.cacheObjects.set(waveformCacheKey, _waveform);
EnjoyApp.waveforms.save(mediaMd5, _waveform);
setWaveForm(_waveform);
}),
wavesurfer.on("ready", () => {
@@ -479,10 +474,8 @@ export const MediaPlayer = (props: {
}, [wavesurfer, isPlaying]);
useEffect(() => {
EnjoyApp.cacheObjects.get(waveformCacheKey).then((cached) => {
if (!cached) return;
setWaveForm(cached);
EnjoyApp.waveforms.find(mediaMd5).then((waveform) => {
setWaveForm(waveform);
});
}, []);

6
enjoy/src/renderer/components/messages/assistant-message.tsx
View File

@@ -90,11 +90,11 @@ export const AssistantMessageComponent = (props: {
id={`message-${message.id}`}
className="flex items-end space-x-2 pr-10"
>
<Avatar className="w-8 h-8 bg-white avatar">
<Avatar className="w-8 h-8 bg-background avatar">
<AvatarImage></AvatarImage>
<AvatarFallback className="bg-white">AI</AvatarFallback>
<AvatarFallback className="bg-background">AI</AvatarFallback>
</Avatar>
<div className="flex flex-col gap-2 px-4 py-2 bg-white border rounded-lg shadow-sm w-full">
<div className="flex flex-col gap-2 px-4 py-2 bg-background border rounded-lg shadow-sm w-full">
{configuration?.autoSpeech && speeching ? (
<div className="p-4">
<LoaderIcon className="w-8 h-8 animate-spin" />

17
enjoy/src/renderer/components/messages/user-message.tsx
View File

@@ -32,6 +32,7 @@ import {
} from "lucide-react";
import { useCopyToClipboard } from "@uidotdev/usehooks";
import { t } from "i18next";
import { useNavigate } from "react-router-dom";
import Markdown from "react-markdown";
export const UserMessageComponent = (props: {
@@ -45,6 +46,7 @@ export const UserMessageComponent = (props: {
const { user, webApi } = useContext(AppSettingsProviderContext);
const [_, copyToClipboard] = useCopyToClipboard();
const [copied, setCopied] = useState<boolean>(false);
const navigate = useNavigate();
const handleShare = async () => {
if (message.role === "user") {
@@ -57,7 +59,18 @@ export const UserMessageComponent = (props: {
},
})
.then(() => {
toast(t("sharedSuccessfully"), { description: t("sharedPrompt") });
toast.success(t("sharedSuccessfully"), {
description: t("sharedPrompt"),
action: {
label: t("view"),
onClick: () => {
navigate("/community");
},
},
actionButtonStyle: {
backgroundColor: "var(--primary)",
},
});
})
.catch((err) => {
toast.error(t("shareFailed"), { description: err.message });
@@ -155,7 +168,7 @@ export const UserMessageComponent = (props: {
</DropdownMenuContent>
</DropdownMenu>
<Avatar className="w-8 h-8 bg-white">
<Avatar className="w-8 h-8 bg-background">
<AvatarImage src={user.avatarUrl} />
<AvatarFallback className="bg-primary text-white capitalize">
{user.name[0]}

2
enjoy/src/renderer/components/posts/post-audio.tsx
View File

@@ -165,7 +165,7 @@ const WavesurferPlayer = (props: {
<div
ref={ref}
className="bg-white rounded-lg grid grid-cols-9 items-center relative h-[80px]"
className="bg-background rounded-lg grid grid-cols-9 items-center relative h-[80px]"
>
{!initialized && (
<div className="col-span-9 flex flex-col justify-around h-[80px]">

2
enjoy/src/renderer/components/posts/post-card.tsx
View File

@@ -20,7 +20,7 @@ export const PostCard = (props: {
const { user } = useContext(AppSettingsProviderContext);
return (
<div className="rounded p-4 bg-white space-y-3">
<div className="rounded p-4 bg-background space-y-3">
<div className="flex items-center justify-between">
<div className="flex items-center space-x-2">
<Avatar>

2
enjoy/src/renderer/components/pronunciation-assessments/pronunciation-assessment-score-result.tsx
View File

@@ -96,7 +96,7 @@ export const PronunciationAssessmentScoreResult = (props: {
</div>
{!pronunciationScore && (
<div className="w-full h-full absolute z-30 bg-white/10 flex items-center justify-center">
<div className="w-full h-full absolute z-30 bg-background/10 flex items-center justify-center">
<Button size="lg" disabled={assessing} onClick={onAssess}>
{assessing && (
<LoaderIcon className="w-4 h-4 animate-spin inline mr-2" />

2
enjoy/src/renderer/components/recordings/recording-card.tsx
View File

@@ -70,7 +70,7 @@ export const RecordingCard = (props: {
return (
<div id={id} className="flex items-center justify-end px-4 transition-all">
<div className="w-full">
<div className="bg-white rounded-lg py-2 px-4 relative mb-1">
<div className="bg-background rounded-lg py-2 px-4 relative mb-1">
<div className="flex items-center justify-end space-x-2">
<span className="text-xs text-muted-foreground">
{secondsToTimestamp(recording.duration / 1000)}

8
enjoy/src/renderer/components/stories/story-viewer.tsx
View File

@@ -13,7 +13,7 @@ import { debounce , uniq } from "lodash";
import Mark from "mark.js";
export const StoryViewer = (props: {
story: StoryType & Partial<CreateStoryParamsType>;
story: Partial<StoryType> & Partial<CreateStoryParamsType>;
marked?: boolean;
meanings?: MeaningType[];
setMeanings: (meanings: MeaningType[]) => void;
@@ -96,7 +96,7 @@ export const StoryViewer = (props: {
return (
<>
<div className="w-full max-w-2xl xl:max-w-3xl mx-auto sticky bg-white top-0 z-30 px-4 py-2 border-b">
<div className="w-full max-w-2xl xl:max-w-3xl mx-auto sticky bg-background top-0 z-30 px-4 py-2 border-b">
<div className="w-full flex items-center space-x-4">
<Button
variant="ghost"
@@ -130,10 +130,10 @@ export const StoryViewer = (props: {
</div>
</div>
</div>
<div className="bg-white py-6 px-8 max-w-2xl xl:max-w-3xl mx-auto relative shadow-lg">
<div className="bg-background py-6 px-8 max-w-2xl xl:max-w-3xl mx-auto relative shadow-lg">
<article
ref={ref}
className="relative select-text prose prose-lg xl:prose-xl font-serif text-lg"
className="relative select-text prose dark:prose-invert prose-lg xl:prose-xl font-serif text-lg"
>
<h2>
{story.title.split(" ").map((word, i) => (

4
enjoy/src/renderer/components/ui/floating-toolbar.tsx
View File

@@ -41,8 +41,8 @@ export const ToolbarButton = (props: {
className={cn(
`rounded-full p-3 h-12 w-12 ${
toggled
? "bg-primary text-white"
: "bg-white text-muted-foreground hover:text-white "
? "bg-primary dark:bg-background text-background dark:text-foreground"
: "bg-background dark:bg-muted text-muted-foreground hover:text-background "
}`,
className
)}

2
enjoy/src/renderer/components/ui/sonner.tsx
View File

@@ -6,7 +6,7 @@ import { Toaster as Sonner, toast } from "sonner";
type ToasterProps = React.ComponentProps<typeof Sonner>;
const Toaster = ({ ...props }: ToasterProps) => {
const { theme = "system" } = useTheme();
const { theme = "light" } = useTheme();
return (
<Sonner

8
enjoy/src/renderer/components/videos/video-detail.tsx
View File

@@ -81,7 +81,9 @@ export const VideoDetail = (props: { id?: string; md5?: string }) => {
targetId: video.id,
})
.then(() => {
toast.success(t("sharedSuccessfully"), { description: t("sharedVideo") });
toast.success(t("sharedSuccessfully"), {
description: t("sharedVideo"),
});
})
.catch((err) => {
toast.error(t("shareFailed"), { description: err.message });
@@ -136,7 +138,7 @@ export const VideoDetail = (props: { id?: string; md5?: string }) => {
mediaId={video.id}
mediaType="Video"
mediaUrl={video.src}
waveformCacheKey={`waveform-video-${video.md5}`}
mediaMd5={video.md5}
transcription={transcription}
currentTime={currentTime}
setCurrentTime={setCurrentTime}
@@ -216,7 +218,7 @@ export const VideoDetail = (props: { id?: string; md5?: string }) => {
</AlertDialog>
{!initialized && (
<div className="top-0 w-full h-full absolute z-30 bg-white/10 flex items-center justify-center">
<div className="top-0 w-full h-full absolute z-30 bg-background/10 flex items-center justify-center">
<LoaderIcon className="text-muted-foreground animate-spin w-8 h-8" />
</div>
)}

4
enjoy/src/renderer/pages/conversation.tsx
View File

@@ -271,7 +271,7 @@ export default () => {
</ScrollArea>
<div className="px-4 absolute w-full bottom-0 left-0 h-14 bg-muted z-50">
<div className="focus-within:bg-white px-4 py-2 flex items-center space-x-4 rounded-lg border">
<div className="focus-within:bg-background px-4 py-2 flex items-center space-x-4 rounded-lg border">
<Textarea
rows={1}
ref={inputRef}
@@ -279,7 +279,7 @@ export default () => {
value={content}
onChange={(e) => setConent(e.target.value)}
placeholder={t("pressEnterToSend")}
className="px-0 py-0 shadow-none border-none focus-visible:outline-0 focus-visible:ring-0 border-none bg-muted focus:bg-white min-h-[1.25rem] max-h-[3.5rem] !overflow-x-hidden"
className="px-0 py-0 shadow-none border-none focus-visible:outline-0 focus-visible:ring-0 border-none bg-muted focus:bg-background min-h-[1.25rem] max-h-[3.5rem] !overflow-x-hidden"
/>
<Button
type="submit"

2
enjoy/src/renderer/pages/conversations.tsx
View File

@@ -86,7 +86,7 @@ export default () => {
{conversations.map((conversation) => (
<Link key={conversation.id} to={`/conversations/${conversation.id}`}>
<div
className="bg-white text-primary rounded-full w-full mb-2 p-4 hover:bg-primary hover:text-white cursor-pointer flex items-center"
className="bg-background text-muted-foreground rounded-full w-full mb-2 p-4 hover:bg-primary hover:text-muted cursor-pointer flex items-center"
style={{
borderLeftColor: `#${conversation.id
.replaceAll("-", "")

6
enjoy/src/renderer/pages/vocabulary.tsx
View File

@@ -38,7 +38,8 @@ export default () => {
}
return (
<div className="h-[100vh] max-w-screen-md mx-auto px-4 py-6">
<div className="h-[100vh] bg-muted">
<div className="max-w-screen-md mx-auto px-4 py-6">
<div className="flex space-x-1 items-center mb-4">
<Button variant="ghost" size="icon" onClick={() => navigate(-1)}>
<ChevronLeftIcon className="w-5 h-5" />
@@ -62,7 +63,7 @@ export default () => {
>
<ChevronLeftIcon className="w-5 h-5" />
</Button>
<div className="flex-1 h-5/6 border p-6 rounded-xl shadow-lg">
<div className="bg-background flex-1 h-5/6 border p-6 rounded-xl shadow-lg">
<MeaningMemorizingCard meaning={meanings[currentIndex]} />
</div>
<Button
@@ -83,5 +84,6 @@ export default () => {
</div>
)}
</div>
</div>
);
};

4
enjoy/src/types/enjoy-app.d.ts vendored
View File

@@ -205,4 +205,8 @@ type EnjoyAppType = {
process: (params: any) => Promise<void>;
update: (id: string, params: any) => Promise<void>;
};
waveforms: {
find: (id: string) => Promise<WaveFormDataType>;
save: (id: string, data: WaveFormDataType) => Promise<void>;
};
};

6
enjoy/src/types/waveform.d.ts vendored Normal file
View File

@@ -0,0 +1,6 @@
type WaveFormDataType = {
peaks: number[];
sampleRate: number;
duration: number;
frequencies: number[];
};