From 1243076bbb36d9332582c67f505bddac5b369d59 Mon Sep 17 00:00:00 2001 From: an-lee Date: Sat, 13 Jan 2024 01:09:40 +0800 Subject: [PATCH] add audio/video & send prompt from share zone --- enjoy/src/i18n/en.json | 13 +- enjoy/src/i18n/zh-CN.json | 13 +- enjoy/src/main/db/handlers/videos-handler.ts | 1 + enjoy/src/main/db/models/conversation.ts | 8 +- .../conversations/conversations-shortcut.tsx | 69 +++++ .../components/conversations/index.ts | 5 +- .../components/posts/post-actions.tsx | 242 ++++++++++++++---- .../renderer/components/posts/post-card.tsx | 8 +- enjoy/src/types/enjoy-app.d.ts | 12 +- enjoy/src/types/medium.d.ts | 1 + 10 files changed, 312 insertions(+), 60 deletions(-) create mode 100644 enjoy/src/renderer/components/conversations/conversations-shortcut.tsx diff --git a/enjoy/src/i18n/en.json b/enjoy/src/i18n/en.json index 8eae3080..05a4e1aa 100644 --- a/enjoy/src/i18n/en.json +++ b/enjoy/src/i18n/en.json @@ -333,9 +333,20 @@ "sharedSuccessfully": "Shared successfully", "shareFailed": "Share failed", "shareAudio": "Share audio", + "sharedAudio": "Shared an audio resource", "areYouSureToShareThisAudioToCommunity": "Are you sure to share this audio to community?", "shareVideo": "Share video", + "sharedVideo": "Shared a video resource", "areYouSureToShareThisVideoToCommunity": "Are you sure to share this video to community?", "sharePrompt": "Share prompt", - "areYouSureToShareThisPromptToCommunity": "Are you sure to share this prompt to community?" + "sharedPrompt": "Shared a prompt", + "areYouSureToShareThisPromptToCommunity": "Are you sure to share this prompt to community?", + "addToLibary": "Add to library", + "areYouSureToAddThisVideoToYourLibrary": "Are you sure to add this video to library?", + "areYouSureToAddThisAudioToYourLibrary": "Are you sure to add this audio to library?", + "audioAlreadyAddedToLibrary": "Audio already added to library", + "videoAlreadyAddedToLibrary": "Video already added to library", + "audioSuccessfullyAddedToLibrary": "Audio successfully added to library", + "videoSuccessfullyAddedToLibrary": "Video successfully added to library", + "sendToAIAssistant": "Send to AI assistant" } diff --git a/enjoy/src/i18n/zh-CN.json b/enjoy/src/i18n/zh-CN.json index 7083abc6..6d2252aa 100644 --- a/enjoy/src/i18n/zh-CN.json +++ b/enjoy/src/i18n/zh-CN.json @@ -333,9 +333,20 @@ "sharedSuccessfully": "分享成功", "sharedFailed": "分享失败", "shareAudio": "分享音频", + "sharedAudio": "分享了一个音频材料", "areYouSureToShareThisAudioToCommunity": "您确定要分享此音频到社区吗？", "shareVideo": "分享视频", + "sharedVideo": "分享了一个视频材料", "areYouSureToShareThisVideoToCommunity": "您确定要分享此视频到社区吗？", "sharePrompt": "分享提示语", - "areYouSureToShareThisPromptToCommunity": "您确定要分享此提示语到社区吗？" + "sharedPrompt": "分享了一条提示语", + "areYouSureToShareThisPromptToCommunity": "您确定要分享此提示语到社区吗？", + "addToLibary": "添加到资源库", + "areYouSureToAddThisVideoToYourLibrary": "您确定要添加此视频到资料库吗？", + "areYouSureToAddThisAudioToYourLibrary": "您确定要添加此音频到资料库吗？", + "audioAlreadyAddedToLibrary": "资料库已经存在此音频", + "videoAlreadyAddedToLibrary": "资料库已经存在此视频", + "audioSuccessfullyAddedToLibrary": "音频成功添加到资料库", + "videoSuccessfullyAddedToLibrary": "视频成功添加到资料库", + "sendToAIAssistant": "发送到智能助手" } diff --git a/enjoy/src/main/db/handlers/videos-handler.ts b/enjoy/src/main/db/handlers/videos-handler.ts index 48fac884..546e7a88 100644 --- a/enjoy/src/main/db/handlers/videos-handler.ts +++ b/enjoy/src/main/db/handlers/videos-handler.ts @@ -94,6 +94,7 @@ class VideosHandler { params: { name?: string; coverUrl?: string; + md5?: string; } = {} ) { let file = uri; diff --git a/enjoy/src/main/db/models/conversation.ts b/enjoy/src/main/db/models/conversation.ts index 3ae977c4..62a468a6 100644 --- a/enjoy/src/main/db/models/conversation.ts +++ b/enjoy/src/main/db/models/conversation.ts @@ -13,7 +13,7 @@ import { AllowNull, } from "sequelize-typescript"; import { Message, Speech } from "@main/db/models"; -import { ChatMessageHistory , BufferMemory } from "langchain/memory"; +import { ChatMessageHistory, BufferMemory } from "langchain/memory"; import { ConversationChain } from "langchain/chains"; import { ChatOpenAI } from "langchain/chat_models/openai"; import { ChatOllama } from "langchain/chat_models/ollama"; @@ -294,9 +294,9 @@ export class Conversation extends Model { } ); - await Promise.all( + const replies = await Promise.all( response.map(async (generation) => { - await Message.create( + return await Message.create( { conversationId: this.id, role: "assistant", @@ -330,5 +330,7 @@ export class Conversation extends Model { } await transaction.commit(); + + return replies.map((reply) => reply.toJSON()); } } diff --git a/enjoy/src/renderer/components/conversations/conversations-shortcut.tsx b/enjoy/src/renderer/components/conversations/conversations-shortcut.tsx new file mode 100644 index 00000000..da7da3d6 --- /dev/null +++ b/enjoy/src/renderer/components/conversations/conversations-shortcut.tsx @@ -0,0 +1,69 @@ +import { useContext, useEffect, useState } from "react"; +import { AppSettingsProviderContext } from "@renderer/context"; +import { ScrollArea } from "@renderer/components/ui"; +import { LoaderSpin } from "@renderer/components"; +import { MessageCircleIcon } from "lucide-react"; + +export const ConversationsShortcut = (props: { + prompt: string; + onReply?: (reply: MessageType[]) => void; +}) => { + const { EnjoyApp } = useContext(AppSettingsProviderContext); + const { prompt, onReply } = props; + const [conversations, setConversations] = useState([]); + const [loading, setLoading] = useState(false); + + const ask = (conversation: ConversationType) => { + setLoading(true); + EnjoyApp.conversations + .ask(conversation.id, { + content: prompt, + }) + .then((replies) => { + console.log(replies); + onReply(replies); + }) + .catch((error) => { + console.error(error); + }) + .finally(() => { + setLoading(false); + }); + }; + + useEffect(() => { + EnjoyApp.conversations.findAll({ limit: 10 }).then((conversations) => { + setConversations(conversations); + setLoading(false); + }); + }, []); + + if (loading) { + return ; + } + + return ( + + {conversations.map((conversation) => { + return ( +
ask(conversation)} + className="bg-white text-primary rounded-full w-full mb-2 py-2 px-4 hover:bg-primary hover:text-white cursor-pointer flex items-center border" + style={{ + borderLeftColor: `#${conversation.id + .replaceAll("-", "") + .substr(0, 6)}`, + borderLeftWidth: 3, + }} + > +
+ +
+
{conversation.name}
+
+ ); + })} + + ); +}; diff --git a/enjoy/src/renderer/components/conversations/index.ts b/enjoy/src/renderer/components/conversations/index.ts index 98c1abbc..477f078a 100644 --- a/enjoy/src/renderer/components/conversations/index.ts +++ b/enjoy/src/renderer/components/conversations/index.ts @@ -1,4 +1,5 @@ -export * from './conversation-form'; +export * from "./conversation-form"; +export * from "./conversations-shortcut"; -export * from './speech-form'; +export * from "./speech-form"; export * from "./speech-player"; diff --git a/enjoy/src/renderer/components/posts/post-actions.tsx b/enjoy/src/renderer/components/posts/post-actions.tsx index be0c94e8..44d7d914 100644 --- a/enjoy/src/renderer/components/posts/post-actions.tsx +++ b/enjoy/src/renderer/components/posts/post-actions.tsx @@ -1,59 +1,215 @@ import { useContext, useEffect, useState } from "react"; import { AppSettingsProviderContext } from "@renderer/context"; -import { PostAudioPlayer } from "@renderer/components"; -import { Button } from "@renderer/components/ui"; -import { formatDateTime } from "@renderer/lib/utils"; -import { t } from "i18next"; -import { MediaPlayer, MediaProvider } from "@vidstack/react"; +import { ConversationsShortcut } from "@renderer/components"; import { - DefaultVideoLayout, - defaultLayoutIcons, -} from "@vidstack/react/player/layouts/default"; + AlertDialog, + AlertDialogTrigger, + AlertDialogContent, + AlertDialogDescription, + AlertDialogHeader, + AlertDialogTitle, + AlertDialogAction, + AlertDialogCancel, + AlertDialogFooter, + Button, + Dialog, + DialogTrigger, + DialogContent, + DialogHeader, + DialogTitle, + ScrollArea, + useToast, +} from "@renderer/components/ui"; +import { t } from "i18next"; import Markdown from "react-markdown"; -import { BotIcon, CheckIcon, CopyPlusIcon, PlusCircleIcon } from "lucide-react"; +import { + BotIcon, + CheckIcon, + CopyPlusIcon, + PlusCircleIcon, + ChevronRightIcon, +} from "lucide-react"; import { useCopyToClipboard } from "@uidotdev/usehooks"; +import { Link } from "react-router-dom"; export const PostActions = (props: { post: PostType }) => { const { post } = props; const [_, copyToClipboard] = useCopyToClipboard(); const [copied, setCopied] = useState(false); + const { EnjoyApp } = useContext(AppSettingsProviderContext); + const { toast } = useToast(); + const [asking, setAsking] = useState(false); + const [aiReplies, setAiReplies] = useState([]); + + const handleAddMedium = async () => { + if (post.targetType !== "Medium") return; + const medium = post.target as MediumType; + if (!medium) return; + + if (medium.mediumType === "Video") { + try { + const video = await EnjoyApp.videos.findOne({ md5: medium.md5 }); + if (video) { + toast({ + description: t("videoAlreadyAddedToLibrary"), + }); + return; + } + } catch (error) { + console.error(error); + } + + EnjoyApp.videos + .create(medium.sourceUrl, { + coverUrl: medium.coverUrl, + md5: medium.md5, + }) + .then(() => { + toast({ + description: t("videoSuccessfullyAddedToLibrary"), + }); + }); + } else if (medium.mediumType === "Audio") { + try { + const audio = await EnjoyApp.audios.findOne({ md5: medium.md5 }); + if (audio) { + toast({ + description: t("audioAlreadyAddedToLibrary"), + }); + return; + } + } catch (error) { + console.error(error); + } + + EnjoyApp.audios + .create(medium.sourceUrl, { + coverUrl: medium.coverUrl, + md5: medium.md5, + }) + .then(() => { + toast({ + description: t("audioSuccessfullyAddedToLibrary"), + }); + }); + } + }; return ( -
- {post.target && post.targetType === "Medium" && ( - + + + + {t("addRecourse")} + + {(post.target as MediumType).mediumType === "Video" && + t("areYouSureToAddThisVideoToYourLibrary")} + + {(post.target as MediumType).mediumType === "Audio" && + t("areYouSureToAddThisAudioToYourLibrary")} + + + + {t("cancel")} + + {t("confirm")} + + + + + )} + + {typeof post.metadata?.content === "string" && ( + - )} - {typeof post.metadata?.content === "string" && ( - - )} - {post.metadata?.type === "prompt" && ( - - )} + data-tooltip-content={t("copy")} + data-tooltip-place="bottom" + variant="ghost" + size="sm" + className="px-1.5 rounded-full" + > + {copied ? ( + + ) : ( + { + copyToClipboard(post.metadata.content as string); + setCopied(true); + setTimeout(() => { + setCopied(false); + }, 3000); + }} + /> + )} + + )} + {post.metadata?.type === "prompt" && ( + + + + + + + {t("sendToAIAssistant")} + + { + setAiReplies([...aiReplies, ...replies]); + setAsking(false); + }} + /> + + + + )} +
+ + {aiReplies.length > 0 && } + + ); +}; + +const AIReplies = (props: { replies: MessageType[] }) => { + return ( +
+
+ {props.replies.map((reply) => ( +
+
+ + + + +
+ {reply.content} +
+ ))} +
); }; diff --git a/enjoy/src/renderer/components/posts/post-card.tsx b/enjoy/src/renderer/components/posts/post-card.tsx index 03e994d8..d39e551c 100644 --- a/enjoy/src/renderer/components/posts/post-card.tsx +++ b/enjoy/src/renderer/components/posts/post-card.tsx @@ -20,7 +20,7 @@ export const PostCard = (props: { post: PostType }) => { const { post } = props; return ( -
+
@@ -40,7 +40,7 @@ export const PostCard = (props: { post: PostType }) => { {post.metadata?.type === "prompt" && ( <> -
+
{t("sharedPrompt")}
@@ -66,7 +66,7 @@ const PostMedium = (props: { medium: MediumType }) => {
{medium.mediumType == "Video" && ( <> -
+
{t("sharedAudio")}
{ {medium.mediumType == "Audio" && ( <> -
+
{t("sharedAudio")}
diff --git a/enjoy/src/types/enjoy-app.d.ts b/enjoy/src/types/enjoy-app.d.ts index 14588882..55139ea2 100644 --- a/enjoy/src/types/enjoy-app.d.ts +++ b/enjoy/src/types/enjoy-app.d.ts @@ -102,8 +102,8 @@ type EnjoyAppType = { videos: { findAll: (params: object) => Promise; findOne: (params: object) => Promise; - create: (uri: string, params?: object) => Promise; - update: (id: string, params: object) => Promise; + create: (uri: string, params?: any) => Promise; + update: (id: string, params: any) => Promise; destroy: (id: string) => Promise; transcribe: (id: string) => Promise; upload: (id: string) => Promise; @@ -143,9 +143,9 @@ type EnjoyAppType = { ) => Promise; }; conversations: { - findAll: (params: object) => Promise; - findOne: (params: object) => Promise; - create: (params: object) => Promise; + findAll: (params: any) => Promise; + findOne: (params: any) => Promise; + create: (params: any) => Promise; update: (id: string, params: object) => Promise; destroy: (id: string) => Promise; ask: ( @@ -159,7 +159,7 @@ type EnjoyAppType = { arrayBuffer: ArrayBuffer; }; } - ) => Promise; + ) => Promise; }; messages: { findAll: (params: object) => Promise; diff --git a/enjoy/src/types/medium.d.ts b/enjoy/src/types/medium.d.ts index 8c742639..94777951 100644 --- a/enjoy/src/types/medium.d.ts +++ b/enjoy/src/types/medium.d.ts @@ -1,5 +1,6 @@ type MediumType = { id: string; + md5: string; mediumType: string; coverUrl?: string; sourceUrl?: string;