Feat: Mention agent in chat (#1118)

* mention UI * fix agent update * create message with mentions * mention agent to reply * fix chat hooks * refactor group prompt * improve UI * refactor Sentence widget
2024-10-13 10:10:25 +08:00
parent d32a51ef23
commit 8589650f55
21 changed files with 405 additions and 520 deletions
--- a/enjoy/src/constants/index.ts
+++ b/enjoy/src/constants/index.ts
@@ -60,11 +60,8 @@ export const NOT_SUPPORT_JSON_FORMAT_MODELS = [

 export const CHAT_GROUP_PROMPT_TEMPLATE = `You are {name} in this chat. You should reply to everyone in this chat and always stay in character.

-[Chat History]
-{history}
-
-Return reply as {name}.
-`;
+[Recent Chat History]
+{history}`;

 export const DEFAULT_GPT_CONFIG = {
  model: "gpt-4o",
--- a/enjoy/src/i18n/en.json
+++ b/enjoy/src/i18n/en.json
@@ -866,5 +866,10 @@
  "migrateToChat": "Migrate to chat",
  "memberJoined": "{{name}} has joined the chat.",
  "memberLeft": "{{name}} has left the chat.",
-  "templates": "templates"
+  "templates": "templates",
+  "askAgentToReply": "Ask @{{name}} to reply",
+  "inviteAgentInChatAndReply": "Invite @{{name}} into the chat and reply",
+  "chatInputPlaceholder": "Type @ to mention agents, press Enter to send",
+  "replyOnlyWhenMentioned": "Reply only when mentioned",
+  "replyOnlyWhenMentionedDescription": "Will only reply when you mention this agent using @"
 }
--- a/enjoy/src/i18n/zh-CN.json
+++ b/enjoy/src/i18n/zh-CN.json
@@ -866,5 +866,10 @@
  "migrateToChat": "迁移到聊天",
  "memberJoined": "{{name}} 已加入聊天",
  "memberLeft": "{{name}} 已离开聊天",
-  "templates": "模板"
+  "templates": "模板",
+  "askAgentToReply": "请 @{{name}} 回复",
+  "inviteAgentInChatAndReply": "邀请 @{{name}} 加入聊天并回复",
+  "chatInputPlaceholder": "输入 @ 指定智能体回答，按回车发送",
+  "replyOnlyWhenMentioned": "仅在提及时回复",
+  "replyOnlyWhenMentionedDescription": "通过 @ 指定智能体时才会回复"
 }
--- a/enjoy/src/main/db/handlers/chat-messages-handler.ts
+++ b/enjoy/src/main/db/handlers/chat-messages-handler.ts
@@ -23,7 +23,6 @@ class ChatMessagesHandler {
      };
    }
    const chatMessages = await ChatMessage.findAll({
-      order: [["createdAt", "DESC"]],
      where,
      ...options,
    });
@@ -47,7 +46,9 @@ class ChatMessagesHandler {

  private async create(
    _event: IpcMainEvent,
-    data: Partial<Attributes<ChatMessage>> & { recordingUrl?: string }
+    data: Partial<Attributes<ChatMessage>> & {
+      recordingUrl?: string;
+    }
  ) {
    const { recordingUrl } = data;
    delete data.recordingUrl;
--- a/enjoy/src/main/db/models/chat-member.ts
+++ b/enjoy/src/main/db/models/chat-member.ts
@@ -1,7 +1,6 @@
 import {
  AfterUpdate,
  AfterDestroy,
-  BeforeDestroy,
  BelongsTo,
  Table,
  Column,
@@ -74,22 +73,20 @@ export class ChatMember extends Model<ChatMember> {

  @AfterCreate
  static async updateChats(member: ChatMember) {
-    const agent = await ChatAgent.findByPk(member.userId);
-    if (agent) {
-      agent.changed("updatedAt", true);
-      agent.update({ updatedAt: new Date() }, { hooks: false });
-    }
-
    const chat = await Chat.findByPk(member.chatId);
    if (chat) {
      chat.changed("updatedAt", true);
-      chat.update({ updatedAt: new Date() }, { hooks: false });
+      chat.update({ updatedAt: new Date() });
    }
  }

  @AfterCreate
  static async chatSystemAddedMessage(member: ChatMember) {
    const chatAgent = await ChatAgent.findByPk(member.userId);
+    if (!chatAgent) return;
+    chatAgent.changed("updatedAt", true);
+    chatAgent.update({ updatedAt: new Date() });
+
    ChatMessage.create({
      chatId: member.chatId,
      content: `${chatAgent.name} has joined the chat.`,
--- a/enjoy/src/main/db/models/chat-message.ts
+++ b/enjoy/src/main/db/models/chat-message.ts
@@ -88,6 +88,7 @@ export class ChatMessage extends Model<ChatMessage> {
  @Column(DataType.UUID)
  agentId: string | null;

+  @Default([])
  @Column(DataType.JSON)
  mentions: string[];

--- a/enjoy/src/renderer/components/chats/chat-agent-card.tsx
+++ b/enjoy/src/renderer/components/chats/chat-agent-card.tsx
@@ -13,7 +13,7 @@ import { EllipsisIcon } from "lucide-react";

 export const ChatAgentCard = (props: {
  chatAgent: ChatAgentType;
-  selected: boolean;
+  selected?: boolean;
  onSelect: (chatAgent: ChatAgentType) => void;
  onEdit?: (chatAgent: ChatAgentType) => void;
  onDelete?: (chatAgent: ChatAgentType) => void;
--- a/enjoy/src/renderer/components/chats/chat-agent-form.tsx
+++ b/enjoy/src/renderer/components/chats/chat-agent-form.tsx
@@ -86,11 +86,18 @@ export const ChatAgentForm = (props: {

  const form = useForm<z.infer<typeof agentFormSchema>>({
    resolver: zodResolver(agentFormSchema),
-    values: agent || {
-      type: ChatAgentTypeEnum.GPT,
-      name: "",
-      description: "",
-    },
+    values: agent
+      ? {
+          type: agent.type,
+          name: agent.name,
+          description: agent.description,
+          config: agent.config,
+        }
+      : {
+          type: ChatAgentTypeEnum.GPT,
+          name: "",
+          description: "",
+        },
  });

  const onSubmit = form.handleSubmit((data) => {
@@ -155,7 +162,7 @@ export const ChatAgentForm = (props: {
    ...templates,
  ];

-  useEffect(() => {
+  const applyTemplate = () => {
    if (form.watch("type") !== ChatAgentTypeEnum.GPT) {
      form.setValue("name", "");
      form.setValue("config.prompt", "");
@@ -172,6 +179,12 @@ export const ChatAgentForm = (props: {
      }
      form.setValue("config.prompt", template.prompt || "");
    }
+  };
+
+  useEffect(() => {
+    if (agent && selectedTemplate === "custom") return;
+
+    applyTemplate();
  }, [selectedTemplate, form.watch("type")]);

  useEffect(() => {
@@ -257,7 +270,7 @@ export const ChatAgentForm = (props: {
                    <CaretSortIcon className="ml-2 h-4 w-4 shrink-0 opacity-50" />
                  </Button>
                </PopoverTrigger>
-                <PopoverContent className="w-full p-0">
+                <PopoverContent className="w-[var(--radix-popper-anchor-width)] p-0">
                  <Command>
                    <CommandInput
                      placeholder={t("templates")}
--- a/enjoy/src/renderer/components/chats/chat-input.tsx
+++ b/enjoy/src/renderer/components/chats/chat-input.tsx
@@ -20,7 +20,7 @@ import {
 } from "@renderer/context";
 import { t } from "i18next";
 import autosize from "autosize";
-import { ChatSuggestionButton } from "@renderer/components";
+import { ChatMentioning, ChatSuggestionButton } from "@renderer/components";
 import { useHotkeys } from "react-hotkeys-hook";
 import { ChatTypeEnum } from "@/types/enums";

@@ -39,6 +39,7 @@ export const ChatInput = () => {
    askAgent,
    createMessage,
    shadowing,
+    chatAgents,
  } = useContext(ChatSessionProviderContext);
  const { EnjoyApp } = useContext(AppSettingsProviderContext);
  const inputRef = useRef<HTMLTextAreaElement>(null);
@@ -46,6 +47,7 @@ export const ChatInput = () => {
  const [inputMode, setInputMode] = useState<"text" | "audio">("text");
  const [content, setContent] = useState("");
  const { currentHotkeys } = useContext(HotKeysSettingsProviderContext);
+  const [mentioned, setMentioned] = useState<ChatAgentType[]>([]);

  useEffect(() => {
    if (!inputRef.current) return;
@@ -183,75 +185,121 @@ export const ChatInput = () => {

  if (inputMode === "text") {
    return (
-      <div className="z-10 w-full flex items-end gap-2 px-2 py-2 bg-muted mx-4 rounded-3xl shadow-lg">
-        <Button
-          data-tooltip-id={`${chat.id}-tooltip`}
-          data-tooltip-content={t("audioInput")}
-          disabled={submitting}
-          onClick={() => setInputMode("audio")}
-          variant="ghost"
-          className=""
-          size="icon"
-        >
-          <MicIcon className="w-6 h-6" />
-        </Button>
-        <Textarea
-          ref={inputRef}
-          value={content}
-          onChange={(e) => setContent(e.target.value)}
-          disabled={submitting}
-          placeholder={t("pressEnterToSend")}
-          data-testid="chat-input"
-          className="flex-1 h-8 text-muted-foreground rounded-lg text-sm leading-7 px-0 py-1 shadow-none focus-visible:outline-0 focus-visible:ring-0 border-none min-h-[2.25rem] max-h-[70vh] scrollbar-thin !overflow-x-hidden"
-        />
-        <Button
-          ref={submitRef}
-          data-tooltip-id={`${chat.id}-tooltip`}
-          data-tooltip-content={t("send")}
-          onClick={() =>
-            createMessage(content, {
-              onSuccess: () => setContent(""),
-            })
-          }
-          disabled={submitting || !content}
-          className="rounded-full shadow w-8 h-8"
-          variant="default"
-          size="icon"
-        >
-          {submitting ? (
-            <LoaderIcon className="w-6 h-6 animate-spin" />
-          ) : (
-            <ArrowUpIcon className="w-6 h-6" />
+      <ChatMentioning
+        input={content}
+        members={chatAgents}
+        mentioned={mentioned.map((chatAgent) => chatAgent.id)}
+        onMention={(chatAgent) => {
+          setMentioned([...mentioned, chatAgent]);
+        }}
+        onRemove={(chatAgent) => {
+          setMentioned(mentioned.filter((ca) => ca.id !== chatAgent.id));
+        }}
+        onCancel={() => setContent("")}
+      >
+        <div className="z-10 w-full mx-4">
+          {mentioned.length > 0 && (
+            <div className="w-full bg-purple-300 rounded px-4 py-2 mb-1 opacity-80 hover:opacity-100">
+              {mentioned.map((chatAgent) => (
+                <div
+                  className="flex items-center justify-between"
+                  key={chatAgent.id}
+                >
+                  <div className="text-sm text-purple-700">
+                    {chatAgents.findIndex((ca) => ca.id === chatAgent.id) > -1
+                      ? t("askAgentToReply", { name: chatAgent.name })
+                      : t("inviteAgentInChatAndReply", {
+                          name: chatAgent.name,
+                        })}
+                  </div>
+                  <Button
+                    variant="ghost"
+                    size="icon"
+                    className="w-6 h-6 text-purple-700 hover:bg-transparent hover:text-purple-900"
+                    onClick={() =>
+                      setMentioned(
+                        mentioned.filter((ca) => ca.id !== chatAgent.id)
+                      )
+                    }
+                  >
+                    <XIcon className="w-4 h-4" />
+                  </Button>
+                </div>
+              ))}
+            </div>
          )}
-        </Button>
-        {chat.config.enableChatAssistant && (
-          <ChatSuggestionButton chat={chat} asChild>
+          <div className="w-full flex items-end gap-2 px-2 py-2 bg-muted rounded-3xl shadow-lg">
            <Button
              data-tooltip-id={`${chat.id}-tooltip`}
-              data-tooltip-content={t("suggestion")}
-              className="rounded-full w-8 h-8"
+              data-tooltip-content={t("audioInput")}
+              disabled={submitting}
+              onClick={() => setInputMode("audio")}
              variant="ghost"
+              className=""
              size="icon"
            >
-              <WandIcon className="w-6 h-6" />
+              <MicIcon className="w-6 h-6" />
            </Button>
-          </ChatSuggestionButton>
-        )}
+            <Textarea
+              ref={inputRef}
+              value={content}
+              onChange={(e) => setContent(e.target.value)}
+              disabled={submitting}
+              placeholder={t("pressEnterToSend")}
+              data-testid="chat-input"
+              className="flex-1 h-8 text-muted-foreground rounded-lg text-sm leading-7 px-0 py-1 shadow-none focus-visible:outline-0 focus-visible:ring-0 border-none min-h-[2.25rem] max-h-[70vh] scrollbar-thin !overflow-x-hidden"
+            />
+            <Button
+              ref={submitRef}
+              data-tooltip-id={`${chat.id}-tooltip`}
+              data-tooltip-content={t("send")}
+              onClick={() =>
+                createMessage(content, {
+                  mentions: mentioned.map((m) => m.id),
+                  onSuccess: () => setContent(""),
+                })
+              }
+              disabled={submitting || !content.trim() || content === "@"}
+              className="rounded-full shadow w-8 h-8"
+              variant="default"
+              size="icon"
+            >
+              {submitting ? (
+                <LoaderIcon className="w-6 h-6 animate-spin" />
+              ) : (
+                <ArrowUpIcon className="w-6 h-6" />
+              )}
+            </Button>
+            {chat.config.enableChatAssistant && (
+              <ChatSuggestionButton chat={chat} asChild>
+                <Button
+                  data-tooltip-id={`${chat.id}-tooltip`}
+                  data-tooltip-content={t("suggestion")}
+                  className="rounded-full w-8 h-8"
+                  variant="ghost"
+                  size="icon"
+                >
+                  <WandIcon className="w-6 h-6" />
+                </Button>
+              </ChatSuggestionButton>
+            )}

-        {chat.type === ChatTypeEnum.GROUP && (
-          <Button
-            data-tooltip-id={`${chat.id}-tooltip`}
-            data-tooltip-content={t("continue")}
-            disabled={submitting}
-            onClick={() => askAgent({ force: true })}
-            className=""
-            variant="ghost"
-            size="icon"
-          >
-            <StepForwardIcon className="w-6 h-6" />
-          </Button>
-        )}
-      </div>
+            {chat.type === ChatTypeEnum.GROUP && (
+              <Button
+                data-tooltip-id={`${chat.id}-tooltip`}
+                data-tooltip-content={t("continue")}
+                disabled={submitting}
+                onClick={() => askAgent({ force: true })}
+                className=""
+                variant="ghost"
+                size="icon"
+              >
+                <StepForwardIcon className="w-6 h-6" />
+              </Button>
+            )}
+          </div>
+        </div>
+      </ChatMentioning>
    );
  }

--- a/enjoy/src/renderer/components/chats/chat-member-form.tsx
+++ b/enjoy/src/renderer/components/chats/chat-member-form.tsx
@@ -22,6 +22,7 @@ import {
  FormItem,
  FormLabel,
  FormMessage,
+  Switch,
  Textarea,
  toast,
 } from "@renderer/components/ui";
@@ -43,8 +44,10 @@ export const ChatMemberForm = (props: {
  const buildFullPrompt = (prompt: string) => {
    return Mustache.render(
      `{{{agent_prompt}}}
-      {{{chat_prompt}}}
-      {{{member_prompt}}}`,
+
+{{{chat_prompt}}}
+
+{{{member_prompt}}}`,
      {
        agent_prompt: member.agent.prompt,
        chat_prompt: chat.config.prompt,
@@ -59,7 +62,7 @@ export const ChatMemberForm = (props: {
    userType: z.enum(["User", "ChatAgent"]).default("ChatAgent"),
    config: z.object({
      prompt: z.string().optional(),
-      description: z.string().optional(),
+      replyOnlyWhenMentioned: z.boolean().default(false),
      gpt: z.object({
        engine: z.string(),
        model: z.string(),
@@ -132,6 +135,24 @@ export const ChatMemberForm = (props: {
  return (
    <Form {...form}>
      <form onSubmit={onSubmit}>
+        <FormField
+          control={form.control}
+          name="config.replyOnlyWhenMentioned"
+          render={({ field }) => (
+            <FormItem>
+              <div className="flex items-center space-x-2">
+                <FormLabel>{t("replyOnlyWhenMentioned")}</FormLabel>
+                <Switch
+                  checked={field.value}
+                  onCheckedChange={field.onChange}
+                />
+              </div>
+              <FormDescription>
+                {t("replyOnlyWhenMentionedDescription")}
+              </FormDescription>
+            </FormItem>
+          )}
+        />
        <Accordion
          defaultValue="gpt"
          type="single"
--- a/enjoy/src/renderer/components/chats/chat-mentioning.tsx
+++ b/enjoy/src/renderer/components/chats/chat-mentioning.tsx
@@ -0,0 +1,88 @@
+import { AppSettingsProviderContext } from "@/renderer/context";
+import {
+  Button,
+  Popover,
+  PopoverAnchor,
+  PopoverContent,
+  PopoverTrigger,
+} from "@renderer/components/ui";
+import { useContext, useEffect, useState } from "react";
+import { ChatAgentCard } from "@renderer/components";
+
+export const ChatMentioning = (props: {
+  input: string;
+  members: ChatAgentType[];
+  mentioned?: string[];
+  onMention: (chatAgent: ChatAgentType) => void;
+  onRemove: (chatAgent: ChatAgentType) => void;
+  onCancel: () => void;
+  children?: React.ReactNode;
+}) => {
+  const {
+    input,
+    members,
+    mentioned = [],
+    onMention,
+    onRemove,
+    onCancel,
+    children,
+  } = props;
+  const [open, setOpen] = useState(false);
+  const [chatAgents, setChatAgents] = useState<ChatAgentType[]>([]);
+  const { EnjoyApp } = useContext(AppSettingsProviderContext);
+
+  const fetchChatAgents = async () => {
+    EnjoyApp.chatAgents.findAll({}).then((chatAgents) => {
+      // sort members to the front
+      const sortedChatAgents = [
+        ...members,
+        ...chatAgents.filter((ca) => !members.some((m) => m.id === ca.id)),
+      ];
+      setChatAgents(sortedChatAgents);
+    });
+  };
+
+  useEffect(() => {
+    // if input starts with @ and contains non-space characters, set open to true
+    if (input === "@") {
+      setOpen(true);
+    } else {
+      setOpen(false);
+    }
+  }, [input]);
+
+  useEffect(() => {
+    if (open) {
+      fetchChatAgents();
+    } else {
+      onCancel();
+    }
+  }, [open]);
+
+  return (
+    <Popover open={open} onOpenChange={setOpen}>
+      <PopoverAnchor asChild>{children}</PopoverAnchor>
+      <PopoverContent
+        side="top"
+        className="p-0 w-[var(--radix-popper-anchor-width)]"
+      >
+        <div className="w-full max-h-72 overflow-y-auto">
+          {chatAgents.map((chatAgent) => (
+            <ChatAgentCard
+              key={chatAgent.id}
+              chatAgent={chatAgent}
+              onSelect={() => {
+                if (mentioned.includes(chatAgent.id)) {
+                  onRemove(chatAgent);
+                } else {
+                  onMention(chatAgent);
+                }
+              }}
+              selected={mentioned.includes(chatAgent.id)}
+            />
+          ))}
+        </div>
+      </PopoverContent>
+    </Popover>
+  );
+};
--- a/enjoy/src/renderer/components/chats/chat-session.tsx
+++ b/enjoy/src/renderer/components/chats/chat-session.tsx
@@ -27,7 +27,7 @@ export const ChatSession = (props: {
        />
        <div className="w-full max-w-screen-md mx-auto">
          <ChatMessages />
-          <div className="h-16" />
+          <div className="h-96" />
          <div className="absolute w-full max-w-screen-md bottom-0 min-h-16 pb-3 flex items-center">
            <ChatInput />
          </div>
--- a/enjoy/src/renderer/components/chats/chat-suggestion-button.tsx
+++ b/enjoy/src/renderer/components/chats/chat-suggestion-button.tsx
@@ -2,6 +2,7 @@ import { ArrowUpIcon, WandIcon } from "lucide-react";
 import {
  Button,
  Popover,
+  PopoverAnchor,
  PopoverArrow,
  PopoverContent,
  PopoverTrigger,
@@ -25,8 +26,9 @@ export const ChatSuggestionButton = (props: {
  chat: ChatType;
  asChild?: boolean;
  children?: ReactElement;
+  anchorRef?: React.RefObject<HTMLDivElement>;
 }) => {
-  const { chat } = props;
+  const { chat, anchorRef } = props;
  const { chatMessages, createMessage } = useContext(
    ChatSessionProviderContext
  );
--- a/enjoy/src/renderer/components/chats/index.ts
+++ b/enjoy/src/renderer/components/chats/index.ts
@@ -16,3 +16,4 @@ export * from "./chat-header";
 export * from "./chat-tts-form";
 export * from "./chat-gpt-form";
 export * from "./chat-suggestion-button";
+export * from "./chat-mentioning";
--- a/enjoy/src/renderer/components/widgets/sentence.tsx
+++ b/enjoy/src/renderer/components/widgets/sentence.tsx
@@ -15,7 +15,11 @@ export const Sentence = ({
      {words.map((word, index) => {
        return (
          <span key={index}>
-            <Vocabulary key={index} word={word} context={sentence} />
+            {word.match(/[a-zA-Z]+/) ? (
+              <Vocabulary word={word} context={sentence} />
+            ) : (
+              word
+            )}
            {index === words.length - 1 ? " " : " "}
          </span>
        );
--- a/enjoy/src/renderer/context/app-settings-provider.tsx
+++ b/enjoy/src/renderer/context/app-settings-provider.tsx
@@ -19,7 +19,7 @@ type AppSettingsProviderState = {
  login?: (user: UserType) => void;
  logout?: () => void;
  setLibraryPath?: (path: string) => Promise<void>;
-  EnjoyApp?: EnjoyAppType;
+  EnjoyApp: EnjoyAppType;
  language?: "en" | "zh-CN";
  switchLanguage?: (language: "en" | "zh-CN") => void;
  nativeLanguage?: string;
@@ -38,14 +38,15 @@ type AppSettingsProviderState = {
  ipaMappings?: { [key: string]: string };
 };

+const EnjoyApp = window.__ENJOY_APP__;
+
 const initialState: AppSettingsProviderState = {
  webApi: null,
  user: null,
  initialized: false,
+  EnjoyApp: EnjoyApp,
 };

-const EnjoyApp = window.__ENJOY_APP__;
-
 export const AppSettingsProviderContext =
  createContext<AppSettingsProviderState>(initialState);

--- a/enjoy/src/renderer/context/chat-session-provider.tsx
+++ b/enjoy/src/renderer/context/chat-session-provider.tsx
@@ -57,6 +57,7 @@ type ChatSessionProviderState = {
  createMessage?: (
    content: string,
    options: {
+      mentions?: string[];
      onSuccess?: (message: ChatMessageType) => void;
      onError?: (error: Error) => void;
    }
@@ -116,10 +117,10 @@ export const ChatSessionProvider = ({
    chatMembers,
    chatMessages,
    dispatchChatMessages,
-    createUserMessage,
    updateMessage,
    deleteMessage,
    invokeAgent,
+    buildAgentMember,
  } = useChatSession(chatId);

  const [deletingMessage, setDeletingMessage] = useState<string>(null);
@@ -156,15 +157,28 @@ export const ChatSessionProvider = ({
  const createMessage = async (
    content: string,
    options: {
+      mentions?: string[];
      onSuccess?: (message: ChatMessageType) => void;
      onError?: (error: Error) => void;
    } = {}
  ) => {
-    const { onSuccess, onError } = options;
+    const { onSuccess, onError, mentions } = options;
    if (submitting) return;

    setSubmitting(true);
-    createUserMessage(content)
+    const joined = await ensureAllMentionsJoined(mentions);
+    if (!joined) {
+      setSubmitting(false);
+      return;
+    }
+    EnjoyApp.chatMessages
+      .create({
+        chatId,
+        content,
+        role: ChatMessageRoleEnum.USER,
+        state: ChatMessageStateEnum.PENDING,
+        mentions,
+      })
      .then((message) => {
        if (message) {
          onSuccess?.(message);
@@ -179,6 +193,26 @@ export const ChatSessionProvider = ({
      });
  };

+  const ensureAllMentionsJoined = async (mentions: string[]) => {
+    if (!mentions || mentions.length === 0) return true;
+
+    const unJoinedMentions = mentions.filter((userId) => {
+      return chatMembers.findIndex((m) => m.userId === userId) === -1;
+    });
+
+    try {
+      for (const userId of unJoinedMentions) {
+        const memberDto = await buildAgentMember(userId);
+        memberDto.config.replyOnlyWhenMentioned = true;
+        await EnjoyApp.chatMembers.create(memberDto);
+      }
+      return true;
+    } catch (error) {
+      toast.error(error.message);
+      return false;
+    }
+  };
+
  const onRecorded = async (blob: Blob) => {
    if (cancelingRecording) {
      setCancelingRecording(false);
@@ -201,12 +235,18 @@ export const ChatSessionProvider = ({
      });

      if (pendingMessage) {
-        await updateMessage(pendingMessage.id, {
+        await EnjoyApp.chatMessages.update(pendingMessage.id, {
          content: transcript,
          recordingUrl: url,
        });
      } else {
-        await createUserMessage(transcript, url);
+        await EnjoyApp.chatMessages.create({
+          chatId,
+          content: transcript,
+          role: ChatMessageRoleEnum.USER,
+          state: ChatMessageStateEnum.PENDING,
+          recordingUrl: url,
+        });
      }
    } catch (error) {
      toast.error(error.message);
@@ -261,6 +301,7 @@ export const ChatSessionProvider = ({
    );
    let currentIndex = messages.length - 1;
    const spokeMembers = new Set();
+    let lastUserMessage: ChatMessageType = null;

    while (currentIndex >= 0) {
      const message = messages[currentIndex];
@@ -271,6 +312,7 @@ export const ChatSessionProvider = ({
        break;
      }
      if (message.role === ChatMessageRoleEnum.USER) {
+        lastUserMessage = message;
        break;
      }
      if (!message.member) break;
@@ -279,10 +321,20 @@ export const ChatSessionProvider = ({
      currentIndex--;
    }

-    // pick a member that has not spoken yet
-    const nextMember = members.find((member) => !spokeMembers.has(member.id));
+    // If the last user message mentions some members, pick one of them that has not spoken yet
+    if (lastUserMessage && lastUserMessage.mentions.length > 0) {
+      return members.find(
+        (member) =>
+          lastUserMessage.mentions.includes(member.userId) &&
+          !spokeMembers.has(member.id)
+      );
+    }

-    return nextMember;
+    // pick a member that has not spoken yet
+    return members.find(
+      (member) =>
+        !member.config.replyOnlyWhenMentioned && !spokeMembers.has(member.id)
+    );
  };

  const onAssess = (assessment: PronunciationAssessmentType) => {
--- a/enjoy/src/renderer/context/copilot-provider.tsx
+++ b/enjoy/src/renderer/context/copilot-provider.tsx
@@ -42,8 +42,7 @@ export const CopilotProvider = ({
  const [active, setActive] = useState(false);
  const [currentChat, setCurrentChat] = useState<ChatType>(null);
  const [occupiedChat, setOccupiedChat] = useState<ChatType | null>(null);
-  const { EnjoyApp } = useContext(AppSettingsProviderContext);
-  const { learningLanguage } = useContext(AppSettingsProviderContext);
+  const { EnjoyApp, learningLanguage } = useContext(AppSettingsProviderContext);
  const { sttEngine, currentGptEngine, currentTtsEngine } = useContext(
    AISettingsProviderContext
  );
--- a/enjoy/src/renderer/hooks/use-chat-message.tsx
+++ b/enjoy/src/renderer/hooks/use-chat-message.tsx
@@ -1,386 +0,0 @@
-import { useEffect, useContext, useReducer, useState } from "react";
-import {
-  AISettingsProviderContext,
-  AppSettingsProviderContext,
-  DbProviderContext,
-} from "@renderer/context";
-import { toast } from "@renderer/components/ui";
-import { chatMessagesReducer } from "@renderer/reducers";
-import { ChatOpenAI } from "@langchain/openai";
-import {
-  ChatPromptTemplate,
-  MessagesPlaceholder,
-} from "@langchain/core/prompts";
-import { BufferMemory, ChatMessageHistory } from "langchain/memory";
-import { ConversationChain } from "langchain/chains";
-import { LLMResult } from "@langchain/core/outputs";
-import { CHAT_GROUP_PROMPT_TEMPLATE } from "@/constants";
-import dayjs from "@renderer/lib/dayjs";
-import Mustache from "mustache";
-import { t } from "i18next";
-import {
-  ChatMessageRoleEnum,
-  ChatMessageStateEnum,
-  ChatTypeEnum,
-} from "@/types/enums";
-
-export const useChatMessage = (chatId: string) => {
-  const { EnjoyApp, user, apiUrl } = useContext(AppSettingsProviderContext);
-  const { openai } = useContext(AISettingsProviderContext);
-  const { addDblistener, removeDbListener } = useContext(DbProviderContext);
-  const [chatMessages, dispatchChatMessages] = useReducer(
-    chatMessagesReducer,
-    []
-  );
-  const [chat, setChat] = useState<ChatType>(null);
-
-  const fetchChat = async () => {
-    if (!chatId) return;
-
-    EnjoyApp.chats.findOne({ where: { id: chatId } }).then((c) => {
-      setChat(c);
-    });
-  };
-
-  const fetchChatMessages = async (query?: string) => {
-    if (!chatId) return;
-
-    return EnjoyApp.chatMessages
-      .findAll({ where: { chatId }, query })
-      .then((data) => {
-        dispatchChatMessages({ type: "set", records: data });
-        return data;
-      })
-      .catch((error) => {
-        toast.error(error.message);
-      });
-  };
-
-  const createUserMessage = (content: string, recordingUrl?: string) => {
-    if (!content) return;
-
-    return EnjoyApp.chatMessages
-      .create({
-        chatId,
-        content,
-        role: ChatMessageRoleEnum.USER,
-        state: ChatMessageStateEnum.PENDING,
-        recordingUrl,
-      })
-      .catch((error) => {
-        toast.error(error.message);
-      });
-  };
-
-  const updateMessage = (id: string, data: ChatMessageDtoType) => {
-    return EnjoyApp.chatMessages.update(id, data);
-  };
-
-  const deleteMessage = async (chatMessageId: string) => {
-    return EnjoyApp.chatMessages
-      .destroy(chatMessageId)
-      .then(() =>
-        dispatchChatMessages({
-          type: "remove",
-          record: { id: chatMessageId } as ChatMessageType,
-        })
-      )
-      .catch((error) => {
-        toast.error(error.message);
-      });
-  };
-
-  const onChatMessageRecordUpdate = (event: CustomEvent) => {
-    const { model, action, record } = event.detail;
-    if (model === "ChatMessage") {
-      if (record.chatId !== chatId) return;
-
-      switch (action) {
-        case "create":
-          dispatchChatMessages({ type: "append", record });
-          break;
-        case "update":
-          dispatchChatMessages({ type: "update", record });
-          break;
-        case "destroy":
-          dispatchChatMessages({ type: "remove", record });
-          break;
-      }
-    } else if (model === "Recording") {
-      switch (action) {
-        case "create":
-          dispatchChatMessages({
-            type: "update",
-            record: {
-              id: record.targetId,
-              recording: record,
-            } as ChatMessageType,
-          });
-          break;
-      }
-    } else if (model === "Speech") {
-      switch (action) {
-        case "create":
-          if (record.sourceType !== "ChatMessage") return;
-
-          dispatchChatMessages({
-            type: "update",
-            record: {
-              id: record.sourceId,
-              speech: record,
-            } as ChatMessageType,
-          });
-          break;
-      }
-    }
-  };
-
-  const invokeAgent = async (memberId: string) => {
-    if (!chat) {
-      await fetchChat();
-    }
-
-    const member = await EnjoyApp.chatMembers.findOne({
-      where: { id: memberId },
-    });
-    if (chat.type === ChatTypeEnum.CONVERSATION) {
-      return askAgentInConversation(member);
-    } else if (chat.type === ChatTypeEnum.GROUP) {
-      return askAgentInGroup(member);
-    } else if (chat.type === ChatTypeEnum.TTS) {
-      return askAgentInTts(member);
-    }
-  };
-
-  const askAgentInConversation = async (member: ChatMemberType) => {
-    const pendingMessage = chatMessages.find(
-      (m) =>
-        m.role === ChatMessageRoleEnum.USER &&
-        m.state === ChatMessageStateEnum.PENDING
-    );
-    if (!pendingMessage) return;
-
-    const llm = buildLlm(member);
-    const historyBufferSize = member.config.gpt.historyBufferSize || 10;
-    const messages = chatMessages
-      .filter((m) => m.state === ChatMessageStateEnum.COMPLETED)
-      .slice(-historyBufferSize);
-    const chatHistory = new ChatMessageHistory();
-    messages.forEach((message) => {
-      if (message.role === ChatMessageRoleEnum.USER) {
-        chatHistory.addUserMessage(message.content);
-      } else if (message.role === ChatMessageRoleEnum.AGENT) {
-        chatHistory.addAIMessage(message.content);
-      }
-    });
-
-    const memory = new BufferMemory({
-      chatHistory,
-      memoryKey: "history",
-      returnMessages: true,
-    });
-    const prompt = ChatPromptTemplate.fromMessages([
-      ["system" as MessageRoleEnum, buildSystemPrompt(member)],
-      new MessagesPlaceholder("history"),
-      ["human", "{input}"],
-    ]);
-    const chain = new ConversationChain({
-      llm: llm as any,
-      memory,
-      prompt: prompt as any,
-      verbose: true,
-    });
-    let response: LLMResult["generations"][0] = [];
-    await chain.call({ input: pendingMessage.content }, [
-      {
-        handleLLMEnd: async (output) => {
-          response = output.generations[0];
-        },
-      },
-    ]);
-    for (const r of response) {
-      await EnjoyApp.chatMessages.create({
-        chatId,
-        memberId: member.id,
-        content: r.text,
-        state: ChatMessageStateEnum.COMPLETED,
-      });
-    }
-    updateMessage(pendingMessage.id, {
-      state: ChatMessageStateEnum.COMPLETED,
-    });
-  };
-
-  const askAgentInGroup = async (member: ChatMemberType) => {
-    const pendingMessage = chatMessages.find(
-      (m) =>
-        m.role === ChatMessageRoleEnum.USER &&
-        m.state === ChatMessageStateEnum.PENDING
-    );
-
-    const llm = buildLlm(member);
-    const prompt = ChatPromptTemplate.fromMessages([
-      ["system", buildSystemPrompt(member)],
-      ["user", CHAT_GROUP_PROMPT_TEMPLATE],
-    ]);
-    const chain = prompt.pipe(llm);
-    const historyBufferSize = member.config.gpt.historyBufferSize || 10;
-    const history = chatMessages
-      .filter(
-        (m) =>
-          m.role === ChatMessageRoleEnum.AGENT ||
-          m.role === ChatMessageRoleEnum.USER
-      )
-      .slice(-historyBufferSize)
-      .map((message) => {
-        const timestamp = dayjs(message.createdAt).fromNow();
-        switch (message.role) {
-          case ChatMessageRoleEnum.AGENT:
-            return `${message.member.agent.name}: ${message.content} (${timestamp})`;
-          case ChatMessageRoleEnum.USER:
-            return `${user.name}: ${message.content} (${timestamp})`;
-          case ChatMessageRoleEnum.SYSTEM:
-            return `(${message.content}, ${timestamp})`;
-          default:
-            return "";
-        }
-      })
-      .join("\n");
-
-    const reply = await chain.invoke({
-      name: member.agent.name,
-      history,
-    });
-
-    // the reply may contain the member's name like "ChatAgent: xxx". We need to remove it.
-    const content = reply.content
-      .toString()
-      .replace(new RegExp(`^(${member.agent.name}):`), "")
-      .trim();
-
-    const message = await EnjoyApp.chatMessages.create({
-      chatId,
-      memberId: member.id,
-      content,
-      state: ChatMessageStateEnum.COMPLETED,
-    });
-    if (pendingMessage) {
-      updateMessage(pendingMessage.id, {
-        state: ChatMessageStateEnum.COMPLETED,
-      });
-    }
-
-    return message;
-  };
-
-  const askAgentInTts = async (member: ChatMemberType) => {
-    const pendingMessage = chatMessages.find(
-      (m) =>
-        m.role === ChatMessageRoleEnum.USER &&
-        m.state === ChatMessageStateEnum.PENDING
-    );
-    if (!pendingMessage) return;
-
-    const message = await EnjoyApp.chatMessages.create({
-      chatId,
-      memberId: member.id,
-      content: pendingMessage.content,
-      state: ChatMessageStateEnum.COMPLETED,
-    });
-    updateMessage(pendingMessage.id, {
-      state: ChatMessageStateEnum.COMPLETED,
-    });
-
-    return message;
-  };
-
-  const buildLlm = (member: ChatMemberType) => {
-    const {
-      engine = "enjoyai",
-      model = "gpt-4o",
-      temperature,
-      maxCompletionTokens,
-      frequencyPenalty,
-      presencePenalty,
-      numberOfChoices,
-    } = member.config.gpt;
-
-    if (engine === "enjoyai") {
-      if (!user.accessToken) {
-        throw new Error(t("authorizationExpired"));
-      }
-
-      return new ChatOpenAI({
-        openAIApiKey: user.accessToken,
-        configuration: {
-          baseURL: `${apiUrl}/api/ai`,
-        },
-        maxRetries: 0,
-        modelName: model,
-        temperature,
-        maxTokens: maxCompletionTokens,
-        frequencyPenalty,
-        presencePenalty,
-        n: numberOfChoices,
-      });
-    } else if (engine === "openai") {
-      if (!openai.key) {
-        throw new Error(t("openaiKeyRequired"));
-      }
-
-      return new ChatOpenAI({
-        openAIApiKey: openai.key,
-        configuration: {
-          baseURL: openai.baseUrl,
-        },
-        maxRetries: 0,
-        modelName: model,
-        temperature,
-        maxTokens: maxCompletionTokens,
-        frequencyPenalty,
-        presencePenalty,
-        n: numberOfChoices,
-      });
-    } else {
-      throw new Error(t("aiEngineNotSupported"));
-    }
-  };
-
-  const buildSystemPrompt = (member: ChatMemberType) => {
-    return Mustache.render(
-      `{{{agent_prompt}}}
-      {{{chat_prompt}}}
-      {{{member_prompt}}}`,
-      {
-        agent_prompt: member.agent.prompt,
-        chat_prompt: chat.config.prompt,
-        member_prompt: member.config.prompt,
-      }
-    );
-  };
-
-  useEffect(() => {
-    fetchChat();
-  }, [chatId]);
-
-  useEffect(() => {
-    if (!chatId) return;
-
-    addDblistener(onChatMessageRecordUpdate);
-    fetchChatMessages();
-    return () => {
-      removeDbListener(onChatMessageRecordUpdate);
-      dispatchChatMessages({ type: "set", records: [] });
-    };
-  }, [chatId]);
-
-  return {
-    chatMessages,
-    fetchChatMessages,
-    dispatchChatMessages,
-    createUserMessage,
-    updateMessage,
-    deleteMessage,
-    invokeAgent,
-  };
-};
--- a/enjoy/src/renderer/hooks/use-chat-session.tsx
+++ b/enjoy/src/renderer/hooks/use-chat-session.tsx
@@ -14,18 +14,24 @@ import {
 import { BufferMemory, ChatMessageHistory } from "langchain/memory";
 import { ConversationChain } from "langchain/chains";
 import { LLMResult } from "@langchain/core/outputs";
-import { CHAT_GROUP_PROMPT_TEMPLATE } from "@/constants";
+import { CHAT_GROUP_PROMPT_TEMPLATE, DEFAULT_GPT_CONFIG } from "@/constants";
 import dayjs from "@renderer/lib/dayjs";
 import Mustache from "mustache";
 import { t } from "i18next";
 import {
+  ChatAgentTypeEnum,
  ChatMessageRoleEnum,
  ChatMessageStateEnum,
  ChatTypeEnum,
 } from "@/types/enums";

 export const useChatSession = (chatId: string) => {
-  const { EnjoyApp, user, apiUrl } = useContext(AppSettingsProviderContext);
+  const { EnjoyApp, user, apiUrl, learningLanguage } = useContext(
+    AppSettingsProviderContext
+  );
+  const { currentGptEngine, currentTtsEngine } = useContext(
+    AISettingsProviderContext
+  );
  const { openai } = useContext(AISettingsProviderContext);
  const { addDblistener, removeDbListener } = useContext(DbProviderContext);
  const [chatMessages, dispatchChatMessages] = useReducer(
@@ -56,22 +62,6 @@ export const useChatSession = (chatId: string) => {
      });
  };

-  const createUserMessage = (content: string, recordingUrl?: string) => {
-    if (!content) return;
-
-    return EnjoyApp.chatMessages
-      .create({
-        chatId,
-        content,
-        role: ChatMessageRoleEnum.USER,
-        state: ChatMessageStateEnum.PENDING,
-        recordingUrl,
-      })
-      .catch((error) => {
-        toast.error(error.message);
-      });
-  };
-
  const updateMessage = (id: string, data: ChatMessageDtoType) => {
    return EnjoyApp.chatMessages.update(id, data);
  };
@@ -240,12 +230,6 @@ export const useChatSession = (chatId: string) => {
        m.state === ChatMessageStateEnum.PENDING
    );

-    const llm = buildLlm(member);
-    const prompt = ChatPromptTemplate.fromMessages([
-      ["system", buildSystemPrompt(member)],
-      ["user", CHAT_GROUP_PROMPT_TEMPLATE],
-    ]);
-    const chain = prompt.pipe(llm);
    const historyBufferSize = member.config.gpt.historyBufferSize || 10;
    const history = chatMessages
      .filter(
@@ -269,9 +253,18 @@ export const useChatSession = (chatId: string) => {
      })
      .join("\n");

+    const llm = buildLlm(member);
+    const prompt = ChatPromptTemplate.fromMessages([
+      ["system", buildSystemPrompt(member)],
+      ["system", CHAT_GROUP_PROMPT_TEMPLATE],
+      ["user", "{input}"],
+    ]);
+    const chain = prompt.pipe(llm);
+
    const reply = await chain.invoke({
      name: member.agent.name,
      history,
+      input: "Return your reply directly without any extra words.",
    });

    // the reply may contain the member's name like "ChatAgent: xxx". We need to remove it.
@@ -381,6 +374,49 @@ export const useChatSession = (chatId: string) => {
    );
  };

+  const buildAgentMember = async (
+    agentId: string
+  ): Promise<ChatMemberDtoType> => {
+    const agent = await EnjoyApp.chatAgents.findOne({ where: { id: agentId } });
+    if (!agent) {
+      throw new Error(t("models.chatAgent.notFound"));
+    }
+
+    const config =
+      agent.type === ChatAgentTypeEnum.TTS
+        ? {
+            tts: {
+              engine: currentTtsEngine.name,
+              model: currentTtsEngine.model,
+              voice: currentTtsEngine.voice,
+              language: learningLanguage,
+              ...agent.config.tts,
+            },
+          }
+        : {
+            replyOnlyWhenMentioned: false,
+            prompt: "",
+            gpt: {
+              ...DEFAULT_GPT_CONFIG,
+              engine: currentGptEngine.name,
+              model: currentGptEngine.models.default,
+            },
+            tts: {
+              engine: currentTtsEngine.name,
+              model: currentTtsEngine.model,
+              voice: currentTtsEngine.voice,
+              language: learningLanguage,
+            },
+          };
+
+    return {
+      chatId,
+      userId: agent.id,
+      userType: "ChatAgent",
+      config,
+    };
+  };
+
  useEffect(() => {
    if (!chatId) return;

@@ -400,9 +436,9 @@ export const useChatSession = (chatId: string) => {
    chatMessages,
    fetchChatMessages,
    dispatchChatMessages,
-    createUserMessage,
    updateMessage,
    deleteMessage,
    invokeAgent,
+    buildAgentMember,
  };
 };
--- a/enjoy/src/types/chat.d.ts
+++ b/enjoy/src/types/chat.d.ts
@@ -37,7 +37,7 @@ type ChatMemberType = {
  userType: "ChatAgent";
  config: {
    prompt?: string;
-    description?: string;
+    replyOnlyWhenMentioned?: boolean;
    gpt?: GptConfigType;
    tts?: TtsConfigType;
    [key: string]: any;
@@ -53,6 +53,7 @@ type ChatMessageType = {
  chatId: string;
  content: string;
  state: ChatMessageStateEnum;
+  mentions: string[];
  createdAt: Date;
  updatedAt: Date;
  member?: ChatMemberType;
@@ -99,8 +100,7 @@ type ChatMemberDtoType = {
  userType: "ChatAgent";
  config: {
    prompt?: string;
-    description?: string;
-    language?: string;
+    replyOnlyWhenMentioned?: boolean;
    gpt?: GptConfigType;
    tts?: TtsConfigType;
    [key: string]: any;