add audio/video & send prompt from share zone

2024-01-13 01:09:40 +08:00
parent 3dff4330a1
commit 1243076bbb
10 changed files with 312 additions and 60 deletions
--- a/enjoy/src/i18n/en.json
+++ b/enjoy/src/i18n/en.json
@@ -333,9 +333,20 @@
  "sharedSuccessfully": "Shared successfully",
  "shareFailed": "Share failed",
  "shareAudio": "Share audio",
+  "sharedAudio": "Shared an audio resource",
  "areYouSureToShareThisAudioToCommunity": "Are you sure to share this audio to community?",
  "shareVideo": "Share video",
+  "sharedVideo": "Shared a video resource",
  "areYouSureToShareThisVideoToCommunity": "Are you sure to share this video to community?",
  "sharePrompt": "Share prompt",
-  "areYouSureToShareThisPromptToCommunity": "Are you sure to share this prompt to community?"
+  "sharedPrompt": "Shared a prompt",
+  "areYouSureToShareThisPromptToCommunity": "Are you sure to share this prompt to community?",
+  "addToLibary": "Add to library",
+  "areYouSureToAddThisVideoToYourLibrary": "Are you sure to add this video to library?",
+  "areYouSureToAddThisAudioToYourLibrary": "Are you sure to add this audio to library?",
+  "audioAlreadyAddedToLibrary": "Audio already added to library",
+  "videoAlreadyAddedToLibrary": "Video already added to library",
+  "audioSuccessfullyAddedToLibrary": "Audio successfully added to library",
+  "videoSuccessfullyAddedToLibrary": "Video successfully added to library",
+  "sendToAIAssistant": "Send to AI assistant"
 }
--- a/enjoy/src/i18n/zh-CN.json
+++ b/enjoy/src/i18n/zh-CN.json
@@ -333,9 +333,20 @@
  "sharedSuccessfully": "分享成功",
  "sharedFailed": "分享失败",
  "shareAudio": "分享音频",
+  "sharedAudio": "分享了一个音频材料",
  "areYouSureToShareThisAudioToCommunity": "您确定要分享此音频到社区吗？",
  "shareVideo": "分享视频",
+  "sharedVideo": "分享了一个视频材料",
  "areYouSureToShareThisVideoToCommunity": "您确定要分享此视频到社区吗？",
  "sharePrompt": "分享提示语",
-  "areYouSureToShareThisPromptToCommunity": "您确定要分享此提示语到社区吗？"
+  "sharedPrompt": "分享了一条提示语",
+  "areYouSureToShareThisPromptToCommunity": "您确定要分享此提示语到社区吗？",
+  "addToLibary": "添加到资源库",
+  "areYouSureToAddThisVideoToYourLibrary": "您确定要添加此视频到资料库吗？",
+  "areYouSureToAddThisAudioToYourLibrary": "您确定要添加此音频到资料库吗？",
+  "audioAlreadyAddedToLibrary": "资料库已经存在此音频",
+  "videoAlreadyAddedToLibrary": "资料库已经存在此视频",
+  "audioSuccessfullyAddedToLibrary": "音频成功添加到资料库",
+  "videoSuccessfullyAddedToLibrary": "视频成功添加到资料库",
+  "sendToAIAssistant": "发送到智能助手"
 }
--- a/enjoy/src/main/db/handlers/videos-handler.ts
+++ b/enjoy/src/main/db/handlers/videos-handler.ts
@@ -94,6 +94,7 @@ class VideosHandler {
    params: {
      name?: string;
      coverUrl?: string;
+      md5?: string;
    } = {}
  ) {
    let file = uri;
--- a/enjoy/src/main/db/models/conversation.ts
+++ b/enjoy/src/main/db/models/conversation.ts
@@ -13,7 +13,7 @@ import {
  AllowNull,
 } from "sequelize-typescript";
 import { Message, Speech } from "@main/db/models";
-import { ChatMessageHistory , BufferMemory } from "langchain/memory";
+import { ChatMessageHistory, BufferMemory } from "langchain/memory";
 import { ConversationChain } from "langchain/chains";
 import { ChatOpenAI } from "langchain/chat_models/openai";
 import { ChatOllama } from "langchain/chat_models/ollama";
@@ -294,9 +294,9 @@ export class Conversation extends Model<Conversation> {
      }
    );

-    await Promise.all(
+    const replies = await Promise.all(
      response.map(async (generation) => {
-        await Message.create(
+        return await Message.create(
          {
            conversationId: this.id,
            role: "assistant",
@@ -330,5 +330,7 @@ export class Conversation extends Model<Conversation> {
    }

    await transaction.commit();
+
+    return replies.map((reply) => reply.toJSON());
  }
 }
--- a/enjoy/src/renderer/components/conversations/conversations-shortcut.tsx
+++ b/enjoy/src/renderer/components/conversations/conversations-shortcut.tsx
@@ -0,0 +1,69 @@
+import { useContext, useEffect, useState } from "react";
+import { AppSettingsProviderContext } from "@renderer/context";
+import { ScrollArea } from "@renderer/components/ui";
+import { LoaderSpin } from "@renderer/components";
+import { MessageCircleIcon } from "lucide-react";
+
+export const ConversationsShortcut = (props: {
+  prompt: string;
+  onReply?: (reply: MessageType[]) => void;
+}) => {
+  const { EnjoyApp } = useContext(AppSettingsProviderContext);
+  const { prompt, onReply } = props;
+  const [conversations, setConversations] = useState<ConversationType[]>([]);
+  const [loading, setLoading] = useState<boolean>(false);
+
+  const ask = (conversation: ConversationType) => {
+    setLoading(true);
+    EnjoyApp.conversations
+      .ask(conversation.id, {
+        content: prompt,
+      })
+      .then((replies) => {
+        console.log(replies);
+        onReply(replies);
+      })
+      .catch((error) => {
+        console.error(error);
+      })
+      .finally(() => {
+        setLoading(false);
+      });
+  };
+
+  useEffect(() => {
+    EnjoyApp.conversations.findAll({ limit: 10 }).then((conversations) => {
+      setConversations(conversations);
+      setLoading(false);
+    });
+  }, []);
+
+  if (loading) {
+    return <LoaderSpin />;
+  }
+
+  return (
+    <ScrollArea>
+      {conversations.map((conversation) => {
+        return (
+          <div
+            key={conversation.id}
+            onClick={() => ask(conversation)}
+            className="bg-white text-primary rounded-full w-full mb-2 py-2 px-4 hover:bg-primary hover:text-white cursor-pointer flex items-center border"
+            style={{
+              borderLeftColor: `#${conversation.id
+                .replaceAll("-", "")
+                .substr(0, 6)}`,
+              borderLeftWidth: 3,
+            }}
+          >
+            <div className="">
+              <MessageCircleIcon className="mr-2" />
+            </div>
+            <div className="flex-1 truncated">{conversation.name}</div>
+          </div>
+        );
+      })}
+    </ScrollArea>
+  );
+};
--- a/enjoy/src/renderer/components/conversations/index.ts
+++ b/enjoy/src/renderer/components/conversations/index.ts
@@ -1,4 +1,5 @@
-export * from './conversation-form';
+export * from "./conversation-form";
+export * from "./conversations-shortcut";

-export * from './speech-form';
+export * from "./speech-form";
 export * from "./speech-player";
--- a/enjoy/src/renderer/components/posts/post-actions.tsx
+++ b/enjoy/src/renderer/components/posts/post-actions.tsx
@@ -1,59 +1,215 @@
 import { useContext, useEffect, useState } from "react";
 import { AppSettingsProviderContext } from "@renderer/context";
-import { PostAudioPlayer } from "@renderer/components";
-import { Button } from "@renderer/components/ui";
-import { formatDateTime } from "@renderer/lib/utils";
-import { t } from "i18next";
-import { MediaPlayer, MediaProvider } from "@vidstack/react";
+import { ConversationsShortcut } from "@renderer/components";
 import {
-  DefaultVideoLayout,
-  defaultLayoutIcons,
-} from "@vidstack/react/player/layouts/default";
+  AlertDialog,
+  AlertDialogTrigger,
+  AlertDialogContent,
+  AlertDialogDescription,
+  AlertDialogHeader,
+  AlertDialogTitle,
+  AlertDialogAction,
+  AlertDialogCancel,
+  AlertDialogFooter,
+  Button,
+  Dialog,
+  DialogTrigger,
+  DialogContent,
+  DialogHeader,
+  DialogTitle,
+  ScrollArea,
+  useToast,
+} from "@renderer/components/ui";
+import { t } from "i18next";
 import Markdown from "react-markdown";
-import { BotIcon, CheckIcon, CopyPlusIcon, PlusCircleIcon } from "lucide-react";
+import {
+  BotIcon,
+  CheckIcon,
+  CopyPlusIcon,
+  PlusCircleIcon,
+  ChevronRightIcon,
+} from "lucide-react";
 import { useCopyToClipboard } from "@uidotdev/usehooks";
+import { Link } from "react-router-dom";

 export const PostActions = (props: { post: PostType }) => {
  const { post } = props;
  const [_, copyToClipboard] = useCopyToClipboard();
  const [copied, setCopied] = useState<boolean>(false);
+  const { EnjoyApp } = useContext(AppSettingsProviderContext);
+  const { toast } = useToast();
+  const [asking, setAsking] = useState<boolean>(false);
+  const [aiReplies, setAiReplies] = useState<MessageType[]>([]);
+
+  const handleAddMedium = async () => {
+    if (post.targetType !== "Medium") return;
+    const medium = post.target as MediumType;
+    if (!medium) return;
+
+    if (medium.mediumType === "Video") {
+      try {
+        const video = await EnjoyApp.videos.findOne({ md5: medium.md5 });
+        if (video) {
+          toast({
+            description: t("videoAlreadyAddedToLibrary"),
+          });
+          return;
+        }
+      } catch (error) {
+        console.error(error);
+      }
+
+      EnjoyApp.videos
+        .create(medium.sourceUrl, {
+          coverUrl: medium.coverUrl,
+          md5: medium.md5,
+        })
+        .then(() => {
+          toast({
+            description: t("videoSuccessfullyAddedToLibrary"),
+          });
+        });
+    } else if (medium.mediumType === "Audio") {
+      try {
+        const audio = await EnjoyApp.audios.findOne({ md5: medium.md5 });
+        if (audio) {
+          toast({
+            description: t("audioAlreadyAddedToLibrary"),
+          });
+          return;
+        }
+      } catch (error) {
+        console.error(error);
+      }
+
+      EnjoyApp.audios
+        .create(medium.sourceUrl, {
+          coverUrl: medium.coverUrl,
+          md5: medium.md5,
+        })
+        .then(() => {
+          toast({
+            description: t("audioSuccessfullyAddedToLibrary"),
+          });
+        });
+    }
+  };

  return (
-    <div className="flex items-center space-x-2 justify-end">
-      {post.target && post.targetType === "Medium" && (
-        <Button variant="ghost" size="sm" className="px-1.5 rounded-full">
-          <PlusCircleIcon
+    <>
+      <div className="flex items-center space-x-2 justify-end">
+        {post.target && post.targetType === "Medium" && (
+          <AlertDialog>
+            <AlertDialogTrigger asChild>
+              <Button
+                data-tooltip-id="global-tooltip"
+                data-tooltip-content={t("addToLibary")}
+                data-tooltip-place="bottom"
+                variant="ghost"
+                size="sm"
+                className="px-1.5 rounded-full"
+              >
+                <PlusCircleIcon className="w-5 h-5 text-muted-foreground hover:text-primary" />
+              </Button>
+            </AlertDialogTrigger>
+            <AlertDialogContent>
+              <AlertDialogHeader>
+                <AlertDialogTitle>{t("addRecourse")}</AlertDialogTitle>
+                <AlertDialogDescription>
+                  {(post.target as MediumType).mediumType === "Video" &&
+                    t("areYouSureToAddThisVideoToYourLibrary")}
+
+                  {(post.target as MediumType).mediumType === "Audio" &&
+                    t("areYouSureToAddThisAudioToYourLibrary")}
+                </AlertDialogDescription>
+              </AlertDialogHeader>
+              <AlertDialogFooter>
+                <AlertDialogCancel>{t("cancel")}</AlertDialogCancel>
+                <AlertDialogAction onClick={handleAddMedium}>
+                  {t("confirm")}
+                </AlertDialogAction>
+              </AlertDialogFooter>
+            </AlertDialogContent>
+          </AlertDialog>
+        )}
+
+        {typeof post.metadata?.content === "string" && (
+          <Button
            data-tooltip-id="global-tooltip"
-            data-tooltip-content={t("addToLibary")}
-            className="w-5 h-5 text-muted-foreground hover:text-primary"
-          />
-        </Button>
-      )}
-      {typeof post.metadata?.content === "string" && (
-        <Button variant="ghost" size="sm" className="px-1.5 rounded-full">
-          {copied ? (
-            <CheckIcon className="w-5 h-5 text-green-500" />
-          ) : (
-            <CopyPlusIcon
-              data-tooltip-id="global-tooltip"
-              data-tooltip-content={t("copy")}
-              className="w-5 h-5 text-muted-foreground hover:text-primary"
-              onClick={() => {
-                copyToClipboard(post.metadata.content as string);
-                setCopied(true);
-                setTimeout(() => {
-                  setCopied(false);
-                }, 3000);
-              }}
-            />
-          )}
-        </Button>
-      )}
-      {post.metadata?.type === "prompt" && (
-        <Button variant="ghost" size="sm" className="px-1.5 rounded-full">
-          <BotIcon className="w-5 h-5 text-muted-foreground hover:text-primary" />
-        </Button>
-      )}
+            data-tooltip-content={t("copy")}
+            data-tooltip-place="bottom"
+            variant="ghost"
+            size="sm"
+            className="px-1.5 rounded-full"
+          >
+            {copied ? (
+              <CheckIcon className="w-5 h-5 text-green-500" />
+            ) : (
+              <CopyPlusIcon
+                className="w-5 h-5 text-muted-foreground hover:text-primary"
+                onClick={() => {
+                  copyToClipboard(post.metadata.content as string);
+                  setCopied(true);
+                  setTimeout(() => {
+                    setCopied(false);
+                  }, 3000);
+                }}
+              />
+            )}
+          </Button>
+        )}
+        {post.metadata?.type === "prompt" && (
+          <Dialog open={asking} onOpenChange={setAsking}>
+            <DialogTrigger asChild>
+              <Button
+                data-tooltip-id="global-tooltip"
+                data-tooltip-content={t("sendToAIAssistant")}
+                data-tooltip-place="bottom"
+                variant="ghost"
+                size="sm"
+                className="px-1.5 rounded-full"
+              >
+                <BotIcon className="w-5 h-5 text-muted-foreground hover:text-primary" />
+              </Button>
+            </DialogTrigger>
+            <DialogContent>
+              <DialogHeader>
+                <DialogTitle>{t("sendToAIAssistant")}</DialogTitle>
+              </DialogHeader>
+              <ConversationsShortcut
+                prompt={post.metadata.content as string}
+                onReply={(replies) => {
+                  setAiReplies([...aiReplies, ...replies]);
+                  setAsking(false);
+                }}
+              />
+            </DialogContent>
+            <ScrollArea></ScrollArea>
+          </Dialog>
+        )}
+      </div>
+
+      {aiReplies.length > 0 && <AIReplies replies={aiReplies} />}
+    </>
+  );
+};
+
+const AIReplies = (props: { replies: MessageType[] }) => {
+  return (
+    <div>
+      <div className="space-y-2">
+        {props.replies.map((reply) => (
+          <div key={reply.id} className="bg-muted py-2 px-4 rounded">
+            <div className="mb-2 flex items-center justify-between">
+              <BotIcon className="w-5 h-5 text-blue-500" />
+              <Link to={`/conversations/${reply.conversationId}`}>
+                <ChevronRightIcon className="w-5 h-5 text-muted-foreground" />
+              </Link>
+            </div>
+            <Markdown className="prose select-text">{reply.content}</Markdown>
+          </div>
+        ))}
+      </div>
    </div>
  );
 };
--- a/enjoy/src/renderer/components/posts/post-card.tsx
+++ b/enjoy/src/renderer/components/posts/post-card.tsx
@@ -20,7 +20,7 @@ export const PostCard = (props: { post: PostType }) => {
  const { post } = props;

  return (
-    <div className="rounded p-4 bg-white space-y-2">
+    <div className="rounded p-4 bg-white space-y-3">
      <div className="flex items-center justify-between">
        <div className="flex items-center space-x-2">
          <Avatar>
@@ -40,7 +40,7 @@ export const PostCard = (props: { post: PostType }) => {

      {post.metadata?.type === "prompt" && (
        <>
-          <div className="text-sm text-muted-foreground">
+          <div className="text-xs text-muted-foreground">
            {t("sharedPrompt")}
          </div>
          <Markdown className="prose prose-slate prose-pre:whitespace-normal select-text">
@@ -66,7 +66,7 @@ const PostMedium = (props: { medium: MediumType }) => {
    <div className="space-y-2">
      {medium.mediumType == "Video" && (
        <>
-          <div className="text-sm text-muted-foreground">
+          <div className="text-xs text-muted-foreground">
            {t("sharedAudio")}
          </div>
          <MediaPlayer
@@ -86,7 +86,7 @@ const PostMedium = (props: { medium: MediumType }) => {

      {medium.mediumType == "Audio" && (
        <>
-          <div className="text-sm text-muted-foreground">
+          <div className="text-xs text-muted-foreground">
            {t("sharedAudio")}
          </div>
          <PostAudioPlayer src={medium.sourceUrl} />
--- a/enjoy/src/types/enjoy-app.d.ts
+++ b/enjoy/src/types/enjoy-app.d.ts
@@ -102,8 +102,8 @@ type EnjoyAppType = {
  videos: {
    findAll: (params: object) => Promise<VideoType[]>;
    findOne: (params: object) => Promise<VideoType>;
-    create: (uri: string, params?: object) => Promise<VideoType>;
-    update: (id: string, params: object) => Promise<VideoType | undefined>;
+    create: (uri: string, params?: any) => Promise<VideoType>;
+    update: (id: string, params: any) => Promise<VideoType | undefined>;
    destroy: (id: string) => Promise<undefined>;
    transcribe: (id: string) => Promise<void>;
    upload: (id: string) => Promise<void>;
@@ -143,9 +143,9 @@ type EnjoyAppType = {
    ) => Promise<SegementRecordingStatsType>;
  };
  conversations: {
-    findAll: (params: object) => Promise<ConversationType[]>;
-    findOne: (params: object) => Promise<ConversationType>;
-    create: (params: object) => Promise<ConversationType>;
+    findAll: (params: any) => Promise<ConversationType[]>;
+    findOne: (params: any) => Promise<ConversationType>;
+    create: (params: any) => Promise<ConversationType>;
    update: (id: string, params: object) => Promise<ConversationType>;
    destroy: (id: string) => Promise<void>;
    ask: (
@@ -159,7 +159,7 @@ type EnjoyAppType = {
          arrayBuffer: ArrayBuffer;
        };
      }
-    ) => Promise<MessageType>;
+    ) => Promise<MessageType[]>;
  };
  messages: {
    findAll: (params: object) => Promise<MessageType[]>;
--- a/enjoy/src/types/medium.d.ts
+++ b/enjoy/src/types/medium.d.ts
@@ -1,5 +1,6 @@
 type MediumType = { 
  id: string;
+  md5: string;
  mediumType: string;
  coverUrl?: string;
  sourceUrl?: string;