From d9534bcae8d0042bfed66ea1bc42ac459b3f2448 Mon Sep 17 00:00:00 2001 From: an-lee Date: Tue, 2 Jul 2024 11:23:11 +0800 Subject: [PATCH] Feat: set language in conversation (#757) * may set language in conversation * fix locale * fix warning * fix --- enjoy/src/i18n/en.json | 3 + enjoy/src/i18n/zh-CN.json | 7 +- enjoy/src/main/db/models/conversation.ts | 5 + .../conversations/conversation-card.tsx | 8 +- .../conversation-form-tts.tsx | 39 +++++++- .../conversations/conversation-form/index.tsx | 91 +++++++++++++------ .../conversations/tts-providers.tsx | 4 +- enjoy/src/renderer/pages/conversation.tsx | 7 ++ enjoy/src/renderer/pages/conversations.tsx | 9 +- enjoy/src/types/conversation.d.ts | 1 + 10 files changed, 140 insertions(+), 34 deletions(-) diff --git a/enjoy/src/i18n/en.json b/enjoy/src/i18n/en.json index 384116a1..d42e7edc 100644 --- a/enjoy/src/i18n/en.json +++ b/enjoy/src/i18n/en.json @@ -88,6 +88,7 @@ "ttsEngine": "TTS engine", "ttsModel": "TTS model", "ttsVoice": "TTS voice", + "ttsLanguage": "TTS language", "ttsBaseUrl": "TTS base URL", "ttsBaseUrlDescription": "leave it blank if you don't have one", "notFound": "Conversation not found", @@ -419,6 +420,8 @@ "selectTtsEngine": "Select TTS engine", "selectTtsModel": "Select TTS model", "selectTtsVoice": "Select TTS voice", + "selectTtsLanguage": "Select TTS Language", + "pleaseSelectTtsVoice": "Please select TTS voice", "youNeedToSetupApiKeyBeforeUsingOpenAI": "You need to setup API key before using OpenAI", "ensureYouHaveOllamaRunningLocallyAndHasAtLeastOneModel": "Ensure you have Ollama running locally and has at least one model", "creatingSpeech": "Speech is creating", diff --git a/enjoy/src/i18n/zh-CN.json b/enjoy/src/i18n/zh-CN.json index 05ee6785..549da3c6 100644 --- a/enjoy/src/i18n/zh-CN.json +++ b/enjoy/src/i18n/zh-CN.json @@ -87,7 +87,8 @@ "numberOfChoicesDescription": "大于 1 时，将每次生成多版本的文本", "ttsEngine": "TTS 引擎", "ttsModel": "TTS 模型", - "ttsVoice": "TTS 声音", + "ttsVoice": "TTS 音色", + "ttsLanguage": "TTS 语言", "ttsBaseUrl": "TTS 接口地址", "ttsBaseUrlDescription": "留空则使用默认值", "notFound": "未找到对话", @@ -418,7 +419,9 @@ "selectAiModel": "选择 AI 模型", "selectTtsEngine": "选择 TTS 引擎", "selectTtsModel": "选择 TTS 模型", - "selectTtsVoice": "选择 TTS 角色", + "selectTtsVoice": "选择 TTS 音色", + "selectTtsLanguage": "选择 TTS 语言", + "pleaseSelectTtsVoice": "请选择 TTS 音色", "youNeedToSetupApiKeyBeforeUsingOpenAI": "在使用 OpenAI 之前您需要设置 API 密钥", "ensureYouHaveOllamaRunningLocallyAndHasAtLeastOneModel": "确保您已经在本地运行 Ollama 并且至少有一个模型", "creatingSpeech": "正在生成语音", diff --git a/enjoy/src/main/db/models/conversation.ts b/enjoy/src/main/db/models/conversation.ts index 228ad7ac..2fb8e1ad 100644 --- a/enjoy/src/main/db/models/conversation.ts +++ b/enjoy/src/main/db/models/conversation.ts @@ -63,6 +63,11 @@ export class Conversation extends Model { return this.getDataValue("configuration").roleDefinition; } + @Column(DataType.VIRTUAL) + get language(): string { + return this.getDataValue("configuration").tts?.language; + } + @HasMany(() => Message) messages: Message[]; diff --git a/enjoy/src/renderer/components/conversations/conversation-card.tsx b/enjoy/src/renderer/components/conversations/conversation-card.tsx index 57b1faa8..004f560b 100644 --- a/enjoy/src/renderer/components/conversations/conversation-card.tsx +++ b/enjoy/src/renderer/components/conversations/conversation-card.tsx @@ -1,8 +1,11 @@ import { MessageCircleIcon, SpeechIcon } from "lucide-react"; import dayjs from "@renderer/lib/dayjs"; +import { useContext } from "react"; +import { AppSettingsProviderContext } from "@renderer/context"; export const ConversationCard = (props: { conversation: ConversationType }) => { const { conversation } = props; + const { learningLanguage } = useContext(AppSettingsProviderContext); return (
{
{conversation.name}
- {conversation.engine} /{" "} + {conversation.engine} |{" "} {conversation.type === "tts" ? conversation.configuration?.tts?.model - : conversation.model} + : conversation.model}{" "} + | {conversation.language || learningLanguage}
diff --git a/enjoy/src/renderer/components/conversations/conversation-form/conversation-form-tts.tsx b/enjoy/src/renderer/components/conversations/conversation-form/conversation-form-tts.tsx index 33492048..c2581218 100644 --- a/enjoy/src/renderer/components/conversations/conversation-form/conversation-form-tts.tsx +++ b/enjoy/src/renderer/components/conversations/conversation-form/conversation-form-tts.tsx @@ -15,6 +15,7 @@ import { } from "@renderer/components/ui"; import { useContext } from "react"; import { AppSettingsProviderContext } from "@renderer/context"; +import { LANGUAGES } from "@/constants"; export const ConversationFormTTS = (props: { form: ReturnType; @@ -90,6 +91,39 @@ export const ConversationFormTTS = (props: { /> )} + {ttsProviders[ + form.watch("configuration.tts.engine") + ]?.configurable?.includes("language") && ( + ( + + {t("models.conversation.ttsLanguage")} + + + + )} + /> + )} + {ttsProviders[ form.watch("configuration.tts.engine") ]?.configurable?.includes("voice") && ( @@ -124,7 +158,10 @@ export const ConversationFormTTS = (props: { {voice} ); - } else if (voice.language === learningLanguage) { + } else if ( + voice.language === + form.watch("configuration.tts.language") + ) { return ( {voice.label} diff --git a/enjoy/src/renderer/components/conversations/conversation-form/index.tsx b/enjoy/src/renderer/components/conversations/conversation-form/index.tsx index 097b7c2a..cdbb2677 100644 --- a/enjoy/src/renderer/components/conversations/conversation-form/index.tsx +++ b/enjoy/src/renderer/components/conversations/conversation-form/index.tsx @@ -43,31 +43,6 @@ import { ConversationFormTTS, } from "@renderer/components"; -const conversationFormSchema = z.object({ - name: z.string().optional(), - engine: z - .enum(["enjoyai", "openai", "ollama", "googleGenerativeAi"]) - .default("openai"), - configuration: z.object({ - type: z.enum(["gpt", "tts"]), - model: z.string().optional(), - baseUrl: z.string().optional(), - roleDefinition: z.string().optional(), - temperature: z.number().min(0).max(1).default(0.2), - numberOfChoices: z.number().min(1).default(1), - maxTokens: z.number().min(-1).default(2000), - presencePenalty: z.number().min(-2).max(2).default(0), - frequencyPenalty: z.number().min(-2).max(2).default(0), - historyBufferSize: z.number().min(0).default(10), - tts: z.object({ - engine: z.enum(["openai", "enjoyai"]).default("enjoyai"), - model: z.string().default("openai/tts-1"), - voice: z.string(), - baseUrl: z.string().optional(), - }), - }), -}); - export const ConversationForm = (props: { conversation: Partial; onFinish?: () => void; @@ -76,10 +51,38 @@ export const ConversationForm = (props: { const [submitting, setSubmitting] = useState(false); const [gptProviders, setGptProviders] = useState(GPT_PROVIDERS); const [ttsProviders, setTtsProviders] = useState(TTS_PROVIDERS); - const { EnjoyApp, webApi } = useContext(AppSettingsProviderContext); + const { EnjoyApp, webApi, learningLanguage } = useContext( + AppSettingsProviderContext + ); const { openai } = useContext(AISettingsProviderContext); const navigate = useNavigate(); + const conversationFormSchema = z.object({ + name: z.string().optional(), + engine: z + .enum(["enjoyai", "openai", "ollama", "googleGenerativeAi"]) + .default("openai"), + configuration: z.object({ + type: z.enum(["gpt", "tts"]), + model: z.string().optional(), + baseUrl: z.string().optional(), + roleDefinition: z.string().optional(), + temperature: z.number().min(0).max(1).default(0.2), + numberOfChoices: z.number().min(1).default(1), + maxTokens: z.number().min(-1).default(2000), + presencePenalty: z.number().min(-2).max(2).default(0), + frequencyPenalty: z.number().min(-2).max(2).default(0), + historyBufferSize: z.number().min(0).default(10), + tts: z.object({ + language: z.string().default(learningLanguage).optional(), + engine: z.enum(["openai", "enjoyai"]).default("enjoyai"), + model: z.string().default("openai/tts-1"), + voice: z.string(), + baseUrl: z.string().optional(), + }), + }), + }); + const refreshGptProviders = async () => { let providers = GPT_PROVIDERS; @@ -148,6 +151,14 @@ export const ConversationForm = (props: { } } + if (!defaultConfig.configuration.tts) { + defaultConfig.configuration.tts = {}; + } + + if (!defaultConfig.configuration.tts.language) { + defaultConfig.configuration.tts.language = learningLanguage; + } + const form = useForm>({ resolver: zodResolver(conversationFormSchema), // @ts-ignore @@ -245,6 +256,34 @@ export const ConversationForm = (props: { configuration.tts.baseUrl = gptProviders[engine]?.baseUrl; } + // validates tts voice + const ttsEngine = configuration.tts.engine; + const voice = configuration.tts.voice; + const language = configuration.tts.language; + if (!language) { + configuration.tts.language === learningLanguage; + } + if (ttsEngine === "openai") { + const options = ttsProviders["openai"].voices; + if (!options.includes(voice)) { + throw new Error(t("pleaseSelectTtsVoice")); + } + } + if (ttsEngine === "enjoyai") { + const model = configuration.tts.model.split("/")[0]; + const options = ttsProviders.enjoyai.voices[model]; + if (model === "openai" && !options.includes(voice)) { + throw new Error(t("pleaseSelectTtsVoice")); + } else if ( + model === "azure" && + options.findIndex( + (o: any) => o.language === language && o.value === voice + ) < 0 + ) { + throw new Error(t("pleaseSelectTtsVoice")); + } + } + return configuration; }; diff --git a/enjoy/src/renderer/components/conversations/tts-providers.tsx b/enjoy/src/renderer/components/conversations/tts-providers.tsx index 92af7e97..b77a3ab6 100644 --- a/enjoy/src/renderer/components/conversations/tts-providers.tsx +++ b/enjoy/src/renderer/components/conversations/tts-providers.tsx @@ -671,13 +671,13 @@ export const TTS_PROVIDERS: { [key: string]: any } = { }, ], }, - configurable: ["model", "voice"], + configurable: ["model", "language", "voice"], }, openai: { name: "OpenAI", description: t("youNeedToSetupApiKeyBeforeUsingOpenAI"), models: ["tts-1", "tts-1-hd"], voices: ["alloy", "echo", "fable", "onyx", "nova", "shimmer"], - configurable: ["model", "voice", "baseUrl"], + configurable: ["model", "language", "voice", "baseUrl"], }, }; diff --git a/enjoy/src/renderer/pages/conversation.tsx b/enjoy/src/renderer/pages/conversation.tsx index 89fdac93..f0073a39 100644 --- a/enjoy/src/renderer/pages/conversation.tsx +++ b/enjoy/src/renderer/pages/conversation.tsx @@ -7,6 +7,8 @@ import { SheetContent, SheetTrigger, toast, + SheetHeader, + SheetTitle, } from "@renderer/components/ui"; import { MessageComponent, ConversationForm } from "@renderer/components"; import { SendIcon, BotIcon, LoaderIcon, SettingsIcon } from "lucide-react"; @@ -248,6 +250,11 @@ export default () => { + + + {t("editConversation")} + +
{ setCreating(value)}> -
+ + + {t("startConversation")} + + +
setCreating(false)} diff --git a/enjoy/src/types/conversation.d.ts b/enjoy/src/types/conversation.d.ts index bfd28ed8..51563843 100644 --- a/enjoy/src/types/conversation.d.ts +++ b/enjoy/src/types/conversation.d.ts @@ -5,6 +5,7 @@ type ConversationType = { name: string; configuration: { [key: string]: any }; model: string; + language?: string; messages?: MessageType[]; createdAt?: string; };