lostecho/everyone-can-use-english
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat: add tts conversation (#341)

Browse Source 
* may create tts type conversation

* support tts reply

* upgrade deps

* test: e2e for create TTS conversation

* test: e2e for gpt conversation

* test: e2e for from create conversation to add speech audio to library

* refactor use-conversation

* generate speech before create msg in tts conversation

* refactor conversation-shorts

* revert change in 1000-hours

* revert sass dep changed in 1000-hours

* fix CI
This commit is contained in:
an-lee
2024-02-22 16:00:17 +08:00
committed by GitHub
parent 518cfa0142
commit 8037273549
27 changed files with 14940 additions and 9881 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
enjoy/e2e/main.spec.ts
View File

@@ -18,15 +18,6 @@ declare global {
}
}
const user = {
id: 24000001,
name: "李安",
avatarUrl:
"https://mixin-images.zeromesh.net/9tMscDkZuXyLKMRChmFi5IiFF2XuQHO8PQpED8zKOCBDGKGSVB9J2eqzyjhgJKPDVunXiT-DPiisImX_bhBDPi4=s256",
accessToken:
"eyJhbGciOiJIUzI1NiJ9.eyJpc3MiOm51bGwsInNpZCI6IjkyN2RjNGRhLTI3YTItNDU5MC1hY2ZiLWMxYTJmZjhhMmFjMiIsInVpZCI6MjQwMDAwMDEsImlhdCI6MTcwODMyODk1N30.PCN_SZ7JH-VYLl56XU8kxYN9Cy44sO13mBQNNz6x-pa",
};
let electronApp: ElectronApplication;
const resultDir = path.join(process.cwd(), "test-results");
@@ -65,61 +56,27 @@ test.afterAll(async () => {
await electronApp.close();
});
test.describe("main dependencies", () => {
test("validate whisper command", async () => {
const page = await electronApp.firstWindow();
const res = await page.evaluate(() => {
return window.__ENJOY_APP__.whisper.check();
});
console.info(res.log);
expect(res.success).toBeTruthy();
const settings = fs.readJsonSync(path.join(resultDir, "settings.json"));
expect(settings.whisper.service).toBe("local");
test("validate whisper command", async () => {
const page = await electronApp.firstWindow();
const res = await page.evaluate(() => {
return window.__ENJOY_APP__.whisper.check();
});
console.info(res.log);
expect(res.success).toBeTruthy();
test("valid ffmpeg command", async () => {
const page = await electronApp.firstWindow();
const res = await page.evaluate(() => {
return window.__ENJOY_APP__.ffmpeg.check();
});
expect(res).toBeTruthy();
});
test("should setup default library path", async () => {
const settings = fs.readJsonSync(path.join(resultDir, "settings.json"));
expect(settings.library).not.toBeNull();
});
const settings = fs.readJsonSync(path.join(resultDir, "settings.json"));
expect(settings.whisper.service).toBe("local");
});
test.describe("with login", async () => {
let page: Page;
test.beforeAll(async () => {
const settings = fs.readJsonSync(path.join(resultDir, "settings.json"));
settings.user = user;
fs.writeJsonSync(path.join(resultDir, "settings.json"), settings);
page = await electronApp.firstWindow();
page.route("**/api/me", (route) => {
route.fulfill({
json: user,
});
});
await page.evaluate(() => {
return window.__ENJOY_APP__.app.reload();
});
});
test("should enter homepage after login", async () => {
await page.waitForSelector("[data-testid=layout-home]");
await page.screenshot({ path: "test-results/homepage.png" });
expect(await page.getByTestId("layout-onboarding").isVisible()).toBeFalsy();
expect(await page.getByTestId("layout-db-error").isVisible()).toBeFalsy();
expect(await page.getByTestId("layout-home").isVisible()).toBeTruthy();
expect(await page.getByTestId("sidebar").isVisible()).toBeTruthy();
test("valid ffmpeg command", async () => {
const page = await electronApp.firstWindow();
const res = await page.evaluate(() => {
return window.__ENJOY_APP__.ffmpeg.check();
});
expect(res).toBeTruthy();
});
test("should setup default library path", async () => {
const settings = fs.readJsonSync(path.join(resultDir, "settings.json"));
expect(settings.library).not.toBeNull();
});

201
enjoy/e2e/renderer.spec.ts Normal file
View File

@@ -0,0 +1,201 @@
import { expect, test } from "@playwright/test";
import { findLatestBuild, parseElectronApp } from "electron-playwright-helpers";
import { ElectronApplication, Page, _electron as electron } from "playwright";
import path from "path";
import fs from "fs-extra";
const user = {
id: 24000001,
name: "李安",
avatarUrl:
"https://mixin-images.zeromesh.net/9tMscDkZuXyLKMRChmFi5IiFF2XuQHO8PQpED8zKOCBDGKGSVB9J2eqzyjhgJKPDVunXiT-DPiisImX_bhBDPi4=s256",
accessToken:
"eyJhbGciOiJIUzI1NiJ9.eyJpc3MiOm51bGwsInNpZCI6IjkyN2RjNGRhLTI3YTItNDU5MC1hY2ZiLWMxYTJmZjhhMmFjMiIsInVpZCI6MjQwMDAwMDEsImlhdCI6MTcwODMyODk1N30.PCN_SZ7JH-VYLl56XU8kxYN9Cy44sO13mBQNNz6x-pa",
};
let electronApp: ElectronApplication;
const resultDir = path.join(process.cwd(), "test-results");
test.beforeAll(async () => {
// find the latest build in the out directory
const latestBuild = findLatestBuild();
// parse the directory and find paths and other info
const appInfo = parseElectronApp(latestBuild);
// set the CI environment variable to true
process.env.CI = "e2e";
fs.ensureDirSync(resultDir);
process.env.SETTINGS_PATH = resultDir;
process.env.LIBRARY_PATH = resultDir;
electronApp = await electron.launch({
args: [appInfo.main],
executablePath: appInfo.executable,
});
electronApp.on("window", async (page) => {
const filename = page.url()?.split("/").pop();
console.info(`Window opened: ${filename}`);
// capture errors
page.on("pageerror", (error) => {
console.error(error);
});
// capture console messages
page.on("console", (msg) => {
console.info(msg.text());
});
});
});
test.afterAll(async () => {
await electronApp.close();
});
test.describe("with login", async () => {
let page: Page;
test.beforeAll(async () => {
const settings = fs.readJsonSync(path.join(resultDir, "settings.json"));
settings.user = user;
fs.writeJsonSync(path.join(resultDir, "settings.json"), settings);
page = await electronApp.firstWindow();
page.route("**/api/me", (route) => {
route.fulfill({
json: user,
});
});
await page.evaluate(() => {
return window.__ENJOY_APP__.app.reload();
});
});
test("should enter homepage after login", async () => {
await page.getByTestId("layout-home").waitFor();
await page.screenshot({ path: "test-results/homepage.png" });
expect(await page.getByTestId("layout-onboarding").isVisible()).toBeFalsy();
expect(await page.getByTestId("layout-db-error").isVisible()).toBeFalsy();
expect(await page.getByTestId("layout-home").isVisible()).toBeTruthy();
expect(await page.getByTestId("sidebar").isVisible()).toBeTruthy();
});
test.describe("with conversation", async () => {
test.beforeEach(async () => {
const file = fs.readFileSync(
path.join(process.cwd(), "samples", "speech.mp3")
);
page = await electronApp.firstWindow();
page.on("console", (msg) => {
console.info(msg.text());
});
await page.route("**/api/ai/audio/speech", (route) => {
route.fulfill({
body: file,
});
});
await page.route("**/api/ai/chat/completions", (route) => {
route.fulfill({
json: {
id: "1",
choices: [
{
index: 1,
message: {
role: "assistant",
content: "I'm fine, thank you.",
},
finish_reason: "stop",
},
],
},
});
});
// navigate to the conversations page
await page.getByTestId("sidebar-conversations").click();
});
/*
* steps:
* 1. create a gpt conversation
* 2. submit a message to the conversation, AI should reply
* 3. create a speech from the AI message
* 4. add the speech to the library
* 5. audio waveform player should be visible and transcription should be generated
*/
test("gpt conversation", async () => {
// trigger new conversation modal
await page.getByTestId("conversation-new-button").click();
// create a gpt conversation
await page.getByTestId("conversation-preset-english-coach").click();
await page.getByTestId("conversation-form").waitFor();
await page.click("[data-testid=conversation-form-submit]");
// wait for the conversation to be created
await page.getByTestId("conversation-page").waitFor();
// submit a message to the conversation
await page.getByTestId("conversation-page-input").fill("How are you?");
await page.getByTestId("conversation-page-submit").click();
await page.locator(".ai-message").waitFor();
const message = page.locator(".ai-message").first();
expect(await message.isVisible()).toBeTruthy();
// create a speech
await page.getByTestId("message-create-speech").click();
// wait for the speech player
const player = page
.locator(".ai-message")
.getByTestId("wavesurfer-container");
await player.waitFor();
expect(await player.isVisible()).toBeTruthy();
// add to library
await page.getByTestId("message-start-shadow").click();
await page.getByTestId("audio-detail").waitFor();
await page.getByTestId("media-player-container").waitFor();
await page.getByTestId("media-transcription").waitFor();
await page.getByTestId("media-transcription-result").waitFor();
expect(
await page.getByTestId("media-transcription-result").isVisible()
).toBeTruthy();
});
/*
* steps:
* 1. create a tts conversation
* 2. submit a message to the conversation
* 3. the speech should auto create
*/
test("tts conversation", async () => {
// trigger new conversation modal
await page.getByTestId("conversation-new-button").click();
// create a tts conversation
await page.click("[data-testid=conversation-preset-tts]");
await page.getByTestId("conversation-form").waitFor();
await page.click("[data-testid=conversation-form-submit]");
// wait for the conversation to be created
await page.getByTestId("conversation-page").waitFor();
// submit a message to the conversation
await page.getByTestId("conversation-page-input").fill("How are you?");
await page.getByTestId("conversation-page-submit").click();
await page.locator(".ai-message").waitFor();
const player = page
.locator(".ai-message")
.getByTestId("wavesurfer-container");
await player.waitFor();
expect(await player.isVisible()).toBeTruthy();
});
});
});

12
enjoy/package.json
View File

@@ -45,15 +45,15 @@
"@types/lodash": "^4.14.202",
"@types/mark.js": "^8.11.12",
"@types/node": "^20.11.19",
"@types/react": "^18.2.56",
"@types/react": "^18.2.57",
"@types/react-dom": "^18.2.19",
"@types/validator": "^13.11.9",
"@types/wavesurfer.js": "^6.0.12",
"@typescript-eslint/eslint-plugin": "^7.0.1",
"@typescript-eslint/parser": "^7.0.1",
"@typescript-eslint/eslint-plugin": "^7.0.2",
"@typescript-eslint/parser": "^7.0.2",
"@vitejs/plugin-react": "^4.2.1",
"autoprefixer": "^10.4.17",
"electron": "^28.2.0",
"electron": "^28.2.3",
"electron-playwright-helpers": "^1.7.1",
"eslint": "^8.56.0",
"eslint-import-resolver-typescript": "^3.6.1",
@@ -101,7 +101,7 @@
"@radix-ui/react-toggle": "^1.0.3",
"@radix-ui/react-tooltip": "^1.0.7",
"@uidotdev/usehooks": "^2.4.1",
"@vidstack/react": "^1.10.7",
"@vidstack/react": "^1.10.9",
"autosize": "^6.0.1",
"axios": "^1.6.7",
"camelcase": "^8.0.0",
@@ -129,7 +129,7 @@
"js-md5": "^0.8.3",
"langchain": "^0.1.20",
"lodash": "^4.17.21",
"lucide-react": "^0.334.0",
"lucide-react": "^0.335.0",
"mark.js": "^8.11.1",
"microsoft-cognitiveservices-speech-sdk": "^1.35.0",
"next-themes": "^0.2.1",

BIN
enjoy/samples/speech.mp3 Normal file
View File

Binary file not shown.

9
enjoy/src/i18n/en.json
View File

@@ -66,9 +66,10 @@
"name": "Name",
"engine": "AI engine",
"baseUrl": "Request endpoint",
"baseUrlDescription": "BaseURL, leave it blank if you don't have one",
"baseUrlDescription": "leave it blank if you don't have one",
"configuration": "Configuration",
"model": "AI model",
"type": "AI Type",
"roleDefinition": "Role definition",
"roleDefinitionPlaceholder": "Describe the AI role",
"temperature": "Temperature",
@@ -87,7 +88,7 @@
"ttsModel": "TTS model",
"ttsVoice": "TTS voice",
"ttsBaseUrl": "TTS base URL",
"ttsBaseUrlDescription": "BaseURL for TTS, leave it blank if you don't have one",
"ttsBaseUrlDescription": "leave it blank if you don't have one",
"notFound": "Conversation not found",
"contentRequired": "Content required",
"failedToGenerateResponse": "Failed to generate response, please retry"
@@ -461,5 +462,7 @@
"itMayTakeAWhileToPrepareForTheFirstLoad": "It may take a while to prepare for the first load. Please be patient.",
"loadingTranscription": "Loading transcription",
"cannotFindMicrophone": "Cannot find microphone",
"failedToSaveRecording": "Failed to save recording"
"failedToSaveRecording": "Failed to save recording",
"speechNotCreatedYet": "Speech not created yet",
"goToConversation": "Go to conversation"
}

9
enjoy/src/i18n/zh-CN.json
View File

@@ -66,9 +66,10 @@
"name": "对话标题",
"engine": "AI 引擎",
"baseUrl": "接口地址",
"baseUrlDescription": "接口地址，留空则使用默认值",
"baseUrlDescription": "留空则使用默认值",
"configuration": "AI 配置",
"model": "AI 模型",
"type": "AI 类型",
"roleDefinition": "角色定义",
"roleDefinitionPlaceholder": "描述 AI 扮演的角色",
"temperature": "随机性 (temperature)",
@@ -87,7 +88,7 @@
"ttsModel": "TTS 模型",
"ttsVoice": "TTS 声音",
"ttsBaseUrl": "TTS 接口地址",
"ttsBaseUrlDescription": "TTS 接口地址，留空则使用默认值",
"ttsBaseUrlDescription": "留空则使用默认值",
"notFound": "未找到对话",
"contentRequired": "对话内容不能为空",
"failedToGenerateResponse": "生成失败，请重试"
@@ -460,5 +461,7 @@
"itMayTakeAWhileToPrepareForTheFirstLoad": "首次加载可能需要一些时间，请耐心等候",
"loadingTranscription": "正在加载语音文本",
"cannotFindMicrophone": "无法找到麦克风",
"failedToSaveRecording": "保存录音失败"
"failedToSaveRecording": "保存录音失败",
"speechNotCreatedYet": "尚未生成语音",
"goToConversation": "前往对话"
}

11
enjoy/src/main/db/models/conversation.ts
View File

@@ -18,7 +18,10 @@ import { ConversationChain } from "langchain/chains";
import { ChatOpenAI } from "@langchain/openai";
import { ChatOllama } from "@langchain/community/chat_models/ollama";
import { ChatGoogleGenerativeAI } from "@langchain/google-genai";
import { ChatPromptTemplate, MessagesPlaceholder } from "@langchain/core/prompts";
import {
ChatPromptTemplate,
MessagesPlaceholder,
} from "@langchain/core/prompts";
import { type Generation } from "langchain/dist/schema";
import settings from "@main/settings";
import db from "@main/db";
@@ -58,11 +61,17 @@ export class Conversation extends Model<Conversation> {
@Column(DataType.JSON)
configuration: {
model: string;
type: "gpt" | "tts";
roleDefinition?: string;
temperature?: number;
maxTokens?: number;
} & { [key: string]: any };
@Column(DataType.VIRTUAL)
get type(): 'gpt' | 'tts' {
return this.getDataValue("configuration").type || "gpt";
}
@Column(DataType.VIRTUAL)
get model(): string {
return this.getDataValue("configuration").model;

2
enjoy/src/renderer/components/audios/audio-detail.tsx
View File

@@ -206,7 +206,7 @@ export const AudioDetail = (props: { id?: string; md5?: string }) => {
}
return (
<div className="relative">
<div className="relative" data-testid="audio-detail">
<div className={`grid grid-cols-7 gap-4 ${initialized ? "" : "blur-sm"}`}>
<div className="col-span-5 h-[calc(100vh-6.5rem)] flex flex-col">
<MediaPlayer

592
enjoy/src/renderer/components/conversations/conversation-form.tsx
View File

@@ -44,7 +44,8 @@ const conversationFormSchema = z.object({
.default("openai"),
configuration: z
.object({
model: z.string().nonempty(),
type: z.enum(["gpt", "tts"]),
model: z.string().optional(),
baseUrl: z.string().optional(),
roleDefinition: z.string().optional(),
temperature: z.number().min(0).max(1).default(0.2),
@@ -83,7 +84,7 @@ export const ConversationForm = (props: {
(m: any) => m.name
);
} catch (e) {
console.error(e);
console.warn(`No ollama server found: ${e.message}`);
}
setProviders({ ...providers });
};
@@ -112,7 +113,8 @@ export const ConversationForm = (props: {
defaultConfig.configuration.baseUrl = openai.baseUrl;
}
}
if (defaultConfig.configuration.tts.engine === "openai" && openai) {
if (defaultConfig.configuration.tts?.engine === "openai" && openai) {
if (!defaultConfig.configuration.tts.baseUrl) {
defaultConfig.configuration.tts.baseUrl = openai.baseUrl;
}
@@ -126,10 +128,8 @@ export const ConversationForm = (props: {
name: conversation.name,
engine: conversation.engine,
configuration: {
type: conversation.configuration.type || "gpt",
...conversation.configuration,
tts: {
...conversation.configuration?.tts,
},
},
}
: {
@@ -146,16 +146,24 @@ export const ConversationForm = (props: {
setSubmitting(true);
Object.keys(configuration).forEach((key) => {
if (key === "type") return;
if (!LLM_PROVIDERS[engine]?.configurable.includes(key)) {
// @ts-ignore
delete configuration[key];
}
});
if (configuration.type === "tts") {
conversation.model = configuration.tts.model;
}
// use default base url if not set
if (!configuration.baseUrl) {
configuration.baseUrl = LLM_PROVIDERS[engine]?.baseUrl;
}
// use default base url if not set
if (!configuration.tts.baseUrl) {
configuration.tts.baseUrl = LLM_PROVIDERS[engine]?.baseUrl;
}
@@ -193,6 +201,7 @@ export const ConversationForm = (props: {
<form
onSubmit={form.handleSubmit(onSubmit)}
className="h-full flex flex-col pt-6"
data-testid="conversation-form"
>
<div className="mb-4 px-6 text-lg font-bold">
{conversation.id ? t("editConversation") : t("startConversation")}
@@ -213,10 +222,10 @@ export const ConversationForm = (props: {
<FormField
control={form.control}
name="engine"
name="configuration.type"
render={({ field }) => (
<FormItem>
<FormLabel>{t("models.conversation.engine")}</FormLabel>
<FormLabel>{t("models.conversation.type")}</FormLabel>
<Select
disabled={Boolean(conversation?.id)}
onValueChange={field.onChange}
@@ -224,45 +233,16 @@ export const ConversationForm = (props: {
>
<FormControl>
<SelectTrigger>
<SelectValue placeholder={t("selectAiEngine")} />
<SelectValue placeholder={t("selectAiType")} />
</SelectTrigger>
</FormControl>
<SelectContent>
{Object.keys(providers).map((key) => (
<SelectItem key={key} value={key}>
{providers[key].name}
</SelectItem>
))}
</SelectContent>
</Select>
<FormDescription>
{providers[form.watch("engine")]?.description}
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="configuration.model"
render={({ field }) => (
<FormItem>
<FormLabel>{t("models.conversation.model")}</FormLabel>
<Select onValueChange={field.onChange} value={field.value}>
<FormControl>
<SelectTrigger>
<SelectValue placeholder={t("selectAiModel")} />
</SelectTrigger>
</FormControl>
<SelectContent>
{(providers[form.watch("engine")]?.models || []).map(
(option: string) => (
<SelectItem key={option} value={option}>
{option}
</SelectItem>
)
)}
<SelectItem key="gpt" value="gpt">
GPT
</SelectItem>
<SelectItem key="tts" value="tts">
TTS
</SelectItem>
</SelectContent>
</Select>
<FormMessage />
@@ -270,227 +250,306 @@ export const ConversationForm = (props: {
)}
/>
<FormField
control={form.control}
name="configuration.roleDefinition"
render={({ field }) => (
<FormItem>
<FormLabel>
{t("models.conversation.roleDefinition")}
</FormLabel>
<Textarea
placeholder={t(
"models.conversation.roleDefinitionPlaceholder"
{form.watch("configuration.type") === "gpt" && (
<>
<FormField
control={form.control}
name="engine"
render={({ field }) => (
<FormItem>
<FormLabel>{t("models.conversation.engine")}</FormLabel>
<Select
disabled={Boolean(conversation?.id)}
onValueChange={field.onChange}
value={field.value}
>
<FormControl>
<SelectTrigger>
<SelectValue placeholder={t("selectAiEngine")} />
</SelectTrigger>
</FormControl>
<SelectContent>
{Object.keys(providers)
.filter((key) =>
LLM_PROVIDERS[key].types.includes(
form.watch("configuration.type")
)
)
.map((key) => (
<SelectItem key={key} value={key}>
{providers[key].name}
</SelectItem>
))}
</SelectContent>
</Select>
<FormDescription>
{providers[form.watch("engine")]?.description}
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="configuration.model"
render={({ field }) => (
<FormItem>
<FormLabel>{t("models.conversation.model")}</FormLabel>
<Select
onValueChange={field.onChange}
value={field.value}
>
<FormControl>
<SelectTrigger>
<SelectValue placeholder={t("selectAiModel")} />
</SelectTrigger>
</FormControl>
<SelectContent>
{(providers[form.watch("engine")]?.models || []).map(
(option: string) => (
<SelectItem key={option} value={option}>
{option}
</SelectItem>
)
)}
</SelectContent>
</Select>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="configuration.roleDefinition"
render={({ field }) => (
<FormItem>
<FormLabel>
{t("models.conversation.roleDefinition")}
</FormLabel>
<Textarea
placeholder={t(
"models.conversation.roleDefinitionPlaceholder"
)}
className="h-64"
{...field}
/>
<FormMessage />
</FormItem>
)}
/>
{LLM_PROVIDERS[form.watch("engine")]?.configurable.includes(
"temperature"
) && (
<FormField
control={form.control}
name="configuration.temperature"
render={({ field }) => (
<FormItem>
<FormLabel>
{t("models.conversation.temperature")}
</FormLabel>
<Input
type="number"
min="0"
max="1.0"
step="0.1"
value={field.value}
onChange={(event) => {
field.onChange(
event.target.value
? parseFloat(event.target.value)
: 0.0
);
}}
/>
<FormDescription>
{t("models.conversation.temperatureDescription")}
</FormDescription>
<FormMessage />
</FormItem>
)}
className="h-64"
{...field}
/>
<FormMessage />
</FormItem>
)}
/>
{LLM_PROVIDERS[form.watch("engine")]?.configurable.includes(
"temperature"
) && (
<FormField
control={form.control}
name="configuration.temperature"
render={({ field }) => (
<FormItem>
<FormLabel>
{t("models.conversation.temperature")}
</FormLabel>
<Input
type="number"
min="0"
max="1.0"
step="0.1"
value={field.value}
onChange={(event) => {
field.onChange(
event.target.value
? parseFloat(event.target.value)
: 0.0
);
}}
/>
<FormDescription>
{t("models.conversation.temperatureDescription")}
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
)}
{LLM_PROVIDERS[form.watch("engine")]?.configurable.includes(
"maxTokens"
) && (
<FormField
control={form.control}
name="configuration.maxTokens"
render={({ field }) => (
<FormItem>
<FormLabel>{t("models.conversation.maxTokens")}</FormLabel>
<Input
type="number"
min="0"
value={field.value}
onChange={(event) => {
if (!event.target.value) return;
field.onChange(parseInt(event.target.value));
}}
/>
<FormDescription>
{t("models.conversation.maxTokensDescription")}
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
)}
{LLM_PROVIDERS[form.watch("engine")]?.configurable.includes(
"presencePenalty"
) && (
<FormField
control={form.control}
name="configuration.presencePenalty"
render={({ field }) => (
<FormItem>
<FormLabel>
{t("models.conversation.presencePenalty")}
</FormLabel>
<Input
type="number"
min="-2"
step="0.1"
max="2"
value={field.value}
onChange={(event) => {
if (!event.target.value) return;
field.onChange(parseInt(event.target.value));
}}
/>
<FormDescription>
{t("models.conversation.presencePenaltyDescription")}
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
)}
{LLM_PROVIDERS[form.watch("engine")]?.configurable.includes(
"frequencyPenalty"
) && (
<FormField
control={form.control}
name="configuration.frequencyPenalty"
render={({ field }) => (
<FormItem>
<FormLabel>
{t("models.conversation.frequencyPenalty")}
</FormLabel>
<Input
type="number"
min="-2"
step="0.1"
max="2"
value={field.value}
onChange={(event) => {
if (!event.target.value) return;
field.onChange(parseInt(event.target.value));
}}
/>
<FormDescription>
{t("models.conversation.frequencyPenaltyDescription")}
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
)}
{LLM_PROVIDERS[form.watch("engine")]?.configurable.includes(
"numberOfChoices"
) && (
<FormField
control={form.control}
name="configuration.numberOfChoices"
render={({ field }) => (
<FormItem>
<FormLabel>
{t("models.conversation.numberOfChoices")}
</FormLabel>
<Input
type="number"
min="1"
step="1.0"
value={field.value}
onChange={(event) => {
field.onChange(
event.target.value
? parseInt(event.target.value)
: 1.0
);
}}
/>
<FormDescription>
{t("models.conversation.numberOfChoicesDescription")}
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
)}
<FormField
control={form.control}
name="configuration.historyBufferSize"
render={({ field }) => (
<FormItem>
<FormLabel>
{t("models.conversation.historyBufferSize")}
</FormLabel>
<Input
type="number"
min="0"
step="1"
max="100"
value={field.value}
onChange={(event) => {
field.onChange(
event.target.value ? parseInt(event.target.value) : 0
);
}}
{LLM_PROVIDERS[form.watch("engine")]?.configurable.includes(
"maxTokens"
) && (
<FormField
control={form.control}
name="configuration.maxTokens"
render={({ field }) => (
<FormItem>
<FormLabel>
{t("models.conversation.maxTokens")}
</FormLabel>
<Input
type="number"
min="0"
value={field.value}
onChange={(event) => {
if (!event.target.value) return;
field.onChange(parseInt(event.target.value));
}}
/>
<FormDescription>
{t("models.conversation.maxTokensDescription")}
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<FormDescription>
{t("models.conversation.historyBufferSizeDescription")}
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
{LLM_PROVIDERS[form.watch("engine")]?.configurable.includes(
"baseUrl"
) && (
<FormField
control={form.control}
name="configuration.baseUrl"
render={({ field }) => (
<FormItem>
<FormLabel>{t("models.conversation.baseUrl")}</FormLabel>
<Input {...field} />
<FormDescription>
{t("models.conversation.baseUrlDescription")}
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
{LLM_PROVIDERS[form.watch("engine")]?.configurable.includes(
"presencePenalty"
) && (
<FormField
control={form.control}
name="configuration.presencePenalty"
render={({ field }) => (
<FormItem>
<FormLabel>
{t("models.conversation.presencePenalty")}
</FormLabel>
<Input
type="number"
min="-2"
step="0.1"
max="2"
value={field.value}
onChange={(event) => {
if (!event.target.value) return;
field.onChange(parseInt(event.target.value));
}}
/>
<FormDescription>
{t("models.conversation.presencePenaltyDescription")}
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
)}
{LLM_PROVIDERS[form.watch("engine")]?.configurable.includes(
"frequencyPenalty"
) && (
<FormField
control={form.control}
name="configuration.frequencyPenalty"
render={({ field }) => (
<FormItem>
<FormLabel>
{t("models.conversation.frequencyPenalty")}
</FormLabel>
<Input
type="number"
min="-2"
step="0.1"
max="2"
value={field.value}
onChange={(event) => {
if (!event.target.value) return;
field.onChange(parseInt(event.target.value));
}}
/>
<FormDescription>
{t("models.conversation.frequencyPenaltyDescription")}
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
)}
{LLM_PROVIDERS[form.watch("engine")]?.configurable.includes(
"numberOfChoices"
) && (
<FormField
control={form.control}
name="configuration.numberOfChoices"
render={({ field }) => (
<FormItem>
<FormLabel>
{t("models.conversation.numberOfChoices")}
</FormLabel>
<Input
type="number"
min="1"
step="1.0"
value={field.value}
onChange={(event) => {
field.onChange(
event.target.value
? parseInt(event.target.value)
: 1.0
);
}}
/>
<FormDescription>
{t("models.conversation.numberOfChoicesDescription")}
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
)}
<FormField
control={form.control}
name="configuration.historyBufferSize"
render={({ field }) => (
<FormItem>
<FormLabel>
{t("models.conversation.historyBufferSize")}
</FormLabel>
<Input
type="number"
min="0"
step="1"
max="100"
value={field.value}
onChange={(event) => {
field.onChange(
event.target.value
? parseInt(event.target.value)
: 0
);
}}
/>
<FormDescription>
{t("models.conversation.historyBufferSizeDescription")}
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
{LLM_PROVIDERS[form.watch("engine")]?.configurable.includes(
"baseUrl"
) && (
<FormField
control={form.control}
name="configuration.baseUrl"
render={({ field }) => (
<FormItem>
<FormLabel>
{t("models.conversation.baseUrl")}
</FormLabel>
<Input
{...field}
placeholder={t(
"models.conversation.baseUrlDescription"
)}
/>
<FormMessage />
</FormItem>
)}
/>
)}
</>
)}
<FormField
@@ -603,10 +662,12 @@ export const ConversationForm = (props: {
render={({ field }) => (
<FormItem>
<FormLabel>{t("models.conversation.ttsBaseUrl")}</FormLabel>
<Input {...field} />
<FormDescription>
{t("models.conversation.ttsBaseUrlDescription")}
</FormDescription>
<Input
{...field}
placeholder={t(
"models.conversation.ttsBaseUrlDescription"
)}
/>
<FormMessage />
</FormItem>
)}
@@ -653,6 +714,7 @@ export const ConversationForm = (props: {
submitting || (conversation.id && !form.formState.isDirty)
}
className="w-full h-12"
data-testid="conversation-form-submit"
size="lg"
type="submit"
>
@@ -691,6 +753,7 @@ export const LLM_PROVIDERS: { [key: string]: any } = {
"historyBufferSize",
"tts",
],
types: ["gpt", "tts"],
},
openai: {
name: "OpenAI",
@@ -719,6 +782,7 @@ export const LLM_PROVIDERS: { [key: string]: any } = {
"historyBufferSize",
"tts",
],
types: ["gpt", "tts"],
},
googleGenerativeAi: {
name: "Google Generative AI",
@@ -731,6 +795,7 @@ export const LLM_PROVIDERS: { [key: string]: any } = {
"historyBufferSize",
"tts",
],
types: ["gpt"],
},
ollama: {
name: "Ollama",
@@ -748,6 +813,7 @@ export const LLM_PROVIDERS: { [key: string]: any } = {
"presencePenalty",
"tts",
],
types: ["gpt"],
},
};

205
enjoy/src/renderer/components/conversations/conversation-shortcuts.tsx Normal file
View File

@@ -0,0 +1,205 @@
import { useContext, useEffect, useState } from "react";
import { AppSettingsProviderContext } from "@renderer/context";
import {
Button,
Dialog,
DialogTrigger,
DialogContent,
DialogHeader,
DialogTitle,
DialogClose,
ScrollArea,
toast,
} from "@renderer/components/ui";
import { LoaderSpin } from "@renderer/components";
import {
MessageCircleIcon,
LoaderIcon,
SpeechIcon,
CheckCircleIcon,
} from "lucide-react";
import { t } from "i18next";
import { useConversation } from "@renderer/hooks";
import { useNavigate } from "react-router-dom";
export const ConversationShortcuts = (props: {
trigger: React.ReactNode;
open?: boolean;
onOpenChange?: (open: boolean) => void;
prompt: string;
onReply?: (reply: Partial<MessageType>[]) => void;
excludedIds?: string[];
}) => {
const { EnjoyApp } = useContext(AppSettingsProviderContext);
const {
prompt,
onReply,
excludedIds = [],
open,
onOpenChange,
trigger,
} = props;
const [conversations, setConversations] = useState<ConversationType[]>([]);
const [loading, setLoading] = useState<boolean>(false);
const [offset, setOffset] = useState<number>(0);
const { chat } = useConversation();
const [replies, setReplies] = useState<Partial<MessageType>[]>([]);
const navigate = useNavigate();
const fetchConversations = () => {
if (offset === -1) return;
const limit = 5;
setLoading(true);
EnjoyApp.conversations
.findAll({
order: [["updatedAt", "DESC"]],
limit,
offset,
})
.then((_conversations) => {
if (_conversations.length === 0) {
setOffset(-1);
return;
}
if (_conversations.length < limit) {
setOffset(-1);
} else {
setOffset(offset + _conversations.length);
}
if (offset === 0) {
setConversations(_conversations);
} else {
setConversations([...conversations, ..._conversations]);
}
})
.finally(() => {
setLoading(false);
});
};
const ask = (conversation: ConversationType) => {
setLoading(true);
chat({ content: prompt }, { conversation })
.then((messages) => {
setReplies(messages);
})
.catch((error) => {
toast.error(error.message);
})
.finally(() => {
setLoading(false);
});
};
useEffect(() => {
fetchConversations();
}, [excludedIds]);
const dialogContent = () => {
if (loading) {
return <LoaderSpin />;
}
if (replies.length > 0) {
return (
<div>
<div className="mb-8 flex items-center justify-center">
<CheckCircleIcon className="w-12 h-12 text-green-500" />
</div>
<div className="flex items-center justify-end space-x-4">
<Button
variant="secondary"
onClick={() => {
navigate(`/conversations/${replies[0].conversationId}`);
setReplies([]);
}}
>
{t("goToConversation")}
</Button>
<DialogClose asChild>
<Button
variant="default"
onClick={() => {
onReply && onReply(replies);
setReplies([]);
}}
>
{t("finish")}
</Button>
</DialogClose>
</div>
</div>
);
}
return (
<ScrollArea>
{conversations.filter((c) => !excludedIds.includes(c.id)).length ===
0 && (
<div className="text-center text-sm text-muted-foreground py-4">
{t("noConversationsYet")}
</div>
)}
{conversations
.filter((c) => !excludedIds.includes(c.id))
.map((conversation) => {
return (
<div
key={conversation.id}
onClick={() => ask(conversation)}
className="bg-background text-primary rounded-full w-full mb-2 py-2 px-4 hover:bg-primary hover:text-white cursor-pointer flex items-center border"
style={{
borderLeftColor: `#${conversation.id
.replaceAll("-", "")
.substr(0, 6)}`,
borderLeftWidth: 3,
}}
>
<div className="">
{conversation.type === "gpt" && (
<MessageCircleIcon className="mr-2" />
)}
{conversation.type === "tts" && (
<SpeechIcon className="mr-2" />
)}
</div>
<div className="flex-1 truncated">{conversation.name}</div>
</div>
);
})}
{offset > -1 && (
<div className="flex justify-center">
<Button
variant="ghost"
onClick={() => fetchConversations()}
disabled={loading || offset === -1}
className="px-4 py-2"
>
{t("loadMore")}
{loading && <LoaderIcon className="w-4 h-4 animate-spin ml-2" />}
</Button>
</div>
)}
</ScrollArea>
);
};
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogTrigger asChild>{trigger}</DialogTrigger>
<DialogContent>
<DialogHeader>
<DialogTitle>{t("sendToAIAssistant")}</DialogTitle>
</DialogHeader>
{dialogContent()}
</DialogContent>
</Dialog>
);
};

8
enjoy/src/renderer/components/conversations/conversations-list.tsx
View File

@@ -2,7 +2,7 @@ import { useContext, useEffect, useState } from "react";
import { AppSettingsProviderContext } from "@renderer/context";
import { Button, ScrollArea } from "@renderer/components/ui";
import { LoaderSpin } from "@renderer/components";
import { MessageCircleIcon, LoaderIcon } from "lucide-react";
import { MessageCircleIcon, LoaderIcon, SpeechIcon } from "lucide-react";
import { useNavigate } from "react-router-dom";
import { t } from "i18next";
@@ -86,7 +86,11 @@ export const ConversationsList = (props: {
}}
>
<div className="">
<MessageCircleIcon className="mr-2" />
{conversation.type === "gpt" && (
<MessageCircleIcon className="mr-2" />
)}
{conversation.type === "tts" && <SpeechIcon className="mr-2" />}
</div>
<div className="flex-1 truncated">{conversation.name}</div>
</div>

121
enjoy/src/renderer/components/conversations/conversations-shortcut.tsx
View File

@@ -1,121 +0,0 @@
import { useContext, useEffect, useState } from "react";
import { AppSettingsProviderContext } from "@renderer/context";
import { Button, ScrollArea, toast } from "@renderer/components/ui";
import { LoaderSpin } from "@renderer/components";
import { MessageCircleIcon, LoaderIcon } from "lucide-react";
import { t } from "i18next";
import { useConversation } from "@renderer/hooks";
export const ConversationsShortcut = (props: {
prompt: string;
onReply?: (reply: Partial<MessageType>[]) => void;
}) => {
const { EnjoyApp } = useContext(AppSettingsProviderContext);
const { prompt, onReply } = props;
const [conversations, setConversations] = useState<ConversationType[]>([]);
const [loading, setLoading] = useState<boolean>(false);
const [offset, setOffset] = useState<number>(0);
const { chat } = useConversation();
const fetchConversations = () => {
if (offset === -1) return;
const limit = 5;
setLoading(true);
EnjoyApp.conversations
.findAll({
order: [["updatedAt", "DESC"]],
limit,
offset,
})
.then((_conversations) => {
if (_conversations.length === 0) {
setOffset(-1);
return;
}
if (_conversations.length < limit) {
setOffset(-1);
} else {
setOffset(offset + _conversations.length);
}
if (offset === 0) {
setConversations(_conversations);
} else {
setConversations([...conversations, ..._conversations]);
}
})
.finally(() => {
setLoading(false);
});
};
const ask = (conversation: ConversationType) => {
setLoading(true);
chat({ content: prompt }, { conversation })
.then((replies) => {
onReply(replies);
})
.catch((error) => {
toast.error(error.message);
})
.finally(() => {
setLoading(false);
});
};
useEffect(() => {
fetchConversations();
}, []);
if (loading) {
return <LoaderSpin />;
}
return (
<ScrollArea>
{conversations.length === 0 && (
<div className="text-center text-sm text-muted-foreground py-4">
{t("noConversationsYet")}
</div>
)}
{conversations.map((conversation) => {
return (
<div
key={conversation.id}
onClick={() => ask(conversation)}
className="bg-background text-primary rounded-full w-full mb-2 py-2 px-4 hover:bg-primary hover:text-white cursor-pointer flex items-center border"
style={{
borderLeftColor: `#${conversation.id
.replaceAll("-", "")
.substr(0, 6)}`,
borderLeftWidth: 3,
}}
>
<div className="">
<MessageCircleIcon className="mr-2" />
</div>
<div className="flex-1 truncated">{conversation.name}</div>
</div>
);
})}
{offset > -1 && (
<div className="flex justify-center">
<Button
variant="ghost"
onClick={() => fetchConversations()}
disabled={loading || offset === -1}
className="px-4 py-2"
>
{t("loadMore")}
{loading && <LoaderIcon className="w-4 h-4 animate-spin ml-2" />}
</Button>
</div>
)}
</ScrollArea>
);
};

3
enjoy/src/renderer/components/conversations/index.ts
View File

@@ -1,6 +1,5 @@
export * from "./conversation-form";
export * from "./conversations-shortcut";
export * from "./conversations-list";
export * from "./conversation-shortcuts";
export * from "./speech-form";
export * from "./speech-player";

1
enjoy/src/renderer/components/conversations/speech-player.tsx
View File

@@ -113,6 +113,7 @@ export const SpeechPlayer = (props: {
</div>
<div
data-testid="wavesurfer-container"
className={`col-span-8 ${initialized ? "" : "hidden"}`}
ref={containerRef}
></div>

8
enjoy/src/renderer/components/medias/media-player.tsx
View File

@@ -516,7 +516,7 @@ export const MediaPlayer = (props: {
const duration =
currentSegment.offsets.to / 1000.0 - currentSegment.offsets.from / 1000.0;
const fitZoomRatio = (containerWidth / duration / minPxPerSecBase);
const fitZoomRatio = containerWidth / duration / minPxPerSecBase;
return fitZoomRatio;
};
@@ -534,7 +534,11 @@ export const MediaPlayer = (props: {
return (
<>
<div className="mb-2" ref={containerRef} />
<div
className="mb-2"
ref={containerRef}
data-testid="media-player-container"
/>
<div className="mb-2 flex justify-center">
<MediaPlayerControls
isPlaying={isPlaying}

11
enjoy/src/renderer/components/medias/media-transcription.tsx
View File

@@ -86,7 +86,10 @@ export const MediaTranscription = (props: {
);
return (
<div className="w-full h-full flex flex-col">
<div
className="w-full h-full flex flex-col"
data-testid="media-transcription"
>
<div className="mb-4 flex items-cener justify-between">
<div className="flex items-center space-x-2">
{transcribing || transcription.state === "processing" ? (
@@ -135,7 +138,11 @@ export const MediaTranscription = (props: {
</div>
{transcription?.result ? (
<ScrollArea ref={containerRef} className="flex-1 px-2">
<ScrollArea
ref={containerRef}
className="flex-1 px-2"
data-testid="media-transcription-result"
>
{transcription.result.map((t, index) => (
<div
key={index}

198
enjoy/src/renderer/components/messages/assistant-message.tsx
View File

@@ -2,21 +2,20 @@ import {
Avatar,
AvatarImage,
AvatarFallback,
Dialog,
DialogTrigger,
DialogContent,
DialogHeader,
DialogTitle,
Sheet,
SheetContent,
SheetHeader,
SheetClose,
toast,
DropdownMenu,
DropdownMenuTrigger,
DropdownMenuContent,
DropdownMenuItem,
} from "@renderer/components/ui";
import {
SpeechPlayer,
AudioDetail,
ConversationsList,
ConversationShortcuts,
} from "@renderer/components";
import { useState, useEffect, useContext } from "react";
import {
@@ -27,6 +26,8 @@ import {
MicIcon,
ChevronDownIcon,
ForwardIcon,
AlertCircleIcon,
MoreVerticalIcon,
} from "lucide-react";
import { useCopyToClipboard } from "@uidotdev/usehooks";
import { t } from "i18next";
@@ -37,8 +38,9 @@ import { useConversation } from "@renderer/hooks";
export const AssistantMessageComponent = (props: {
message: MessageType;
configuration: { [key: string]: any };
onRemove: () => void;
}) => {
const { message, configuration } = props;
const { message, configuration, onRemove } = props;
const [_, copyToClipboard] = useCopyToClipboard();
const [copied, setCopied] = useState<boolean>(false);
const [speech, setSpeech] = useState<Partial<SpeechType>>(
@@ -52,10 +54,19 @@ export const AssistantMessageComponent = (props: {
useEffect(() => {
if (speech) return;
if (!configuration?.autoSpeech) return;
if (configuration?.type !== "tts") return;
createSpeech();
}, [message, configuration]);
findOrCreateSpeech();
}, [message]);
const findOrCreateSpeech = async () => {
const msg = await EnjoyApp.messages.findOne({ id: message.id });
if (msg.speeches.length > 0) {
setSpeech(msg.speeches[0]);
} else {
createSpeech();
}
};
const createSpeech = () => {
if (speeching) return;
@@ -103,20 +114,30 @@ export const AssistantMessageComponent = (props: {
return (
<div
id={`message-${message.id}`}
className="flex items-end space-x-2 pr-10"
className="ai-message flex items-end space-x-2 pr-10"
>
<Avatar className="w-8 h-8 bg-background avatar">
<AvatarImage></AvatarImage>
<AvatarFallback className="bg-background">AI</AvatarFallback>
</Avatar>
<div className="flex flex-col gap-2 px-4 py-2 bg-background border rounded-lg shadow-sm w-full">
{configuration?.autoSpeech && speeching ? (
<div className="p-4">
<LoaderIcon className="w-8 h-8 animate-spin" />
</div>
) : (
{configuration.type === "tts" &&
(speeching ? (
<div className="text-muted-foreground text-sm py-2">
<span>{t("creatingSpeech")}</span>
</div>
) : (
!speech && (
<div className="text-muted-foreground text-sm py-2 flex items-center">
<AlertCircleIcon className="w-4 h-4 mr-2 text-yellow-600" />
<span>{t("speechNotCreatedYet")}</span>
</div>
)
))}
{configuration.type === "gpt" && (
<Markdown
className="select-text prose"
className="message-content select-text prose"
components={{
a({ node, children, ...props }) {
try {
@@ -135,78 +156,87 @@ export const AssistantMessageComponent = (props: {
{Boolean(speech) && <SpeechPlayer speech={speech} />}
<div className="flex items-center justify-start space-x-2">
{copied ? (
<CheckIcon className="w-3 h-3 text-green-500" />
) : (
<CopyIcon
data-tooltip-id="global-tooltip"
data-tooltip-content={t("copyText")}
className="w-3 h-3 cursor-pointer"
onClick={() => {
copyToClipboard(message.content);
setCopied(true);
setTimeout(() => {
setCopied(false);
}, 3000);
}}
/>
)}
<DropdownMenu>
<div className="flex items-center justify-start space-x-2">
{!speech &&
(speeching ? (
<LoaderIcon
data-tooltip-id="global-tooltip"
data-tooltip-content={t("creatingSpeech")}
className="w-3 h-3 animate-spin"
/>
) : (
<SpeechIcon
data-tooltip-id="global-tooltip"
data-tooltip-content={t("textToSpeech")}
data-testid="message-create-speech"
onClick={createSpeech}
className="w-3 h-3 cursor-pointer"
/>
))}
{!speech &&
!configuration?.autoSpeech &&
(speeching ? (
<LoaderIcon
data-tooltip-id="global-tooltip"
data-tooltip-content={t("creatingSpeech")}
className="w-3 h-3 animate-spin"
/>
) : (
<SpeechIcon
data-tooltip-id="global-tooltip"
data-tooltip-content={t("textToSpeech")}
onClick={createSpeech}
className="w-3 h-3 cursor-pointer"
/>
))}
<Dialog>
<DialogTrigger>
<ForwardIcon
data-tooltip-id="global-tooltip"
data-tooltip-content={t("forward")}
className="w-3 h-3 cursor-pointer"
/>
</DialogTrigger>
<DialogContent>
<DialogHeader>
<DialogTitle>{t("forward")}</DialogTitle>
</DialogHeader>
<div className="">
<ConversationsList
{configuration.type === "gpt" && (
<>
{copied ? (
<CheckIcon className="w-3 h-3 text-green-500" />
) : (
<CopyIcon
data-tooltip-id="global-tooltip"
data-tooltip-content={t("copyText")}
className="w-3 h-3 cursor-pointer"
onClick={() => {
copyToClipboard(message.content);
setCopied(true);
setTimeout(() => {
setCopied(false);
}, 3000);
}}
/>
)}
<ConversationShortcuts
prompt={message.content}
excludedIds={[message.conversationId]}
trigger={
<ForwardIcon
data-tooltip-id="global-tooltip"
data-tooltip-content={t("forward")}
className="w-3 h-3 cursor-pointer"
/>
}
/>
</div>
</DialogContent>
</Dialog>
</>
)}
{Boolean(speech) &&
(resourcing ? (
<LoaderIcon
data-tooltip-id="global-tooltip"
data-tooltip-content={t("addingResource")}
className="w-3 h-3 animate-spin"
/>
) : (
<MicIcon
data-tooltip-id="global-tooltip"
data-tooltip-content={t("shadowingExercise")}
onClick={startShadow}
className="w-3 h-3 cursor-pointer"
/>
))}
</div>
{Boolean(speech) &&
(resourcing ? (
<LoaderIcon
data-tooltip-id="global-tooltip"
data-tooltip-content={t("addingResource")}
className="w-3 h-3 animate-spin"
/>
) : (
<MicIcon
data-tooltip-id="global-tooltip"
data-tooltip-content={t("shadowingExercise")}
data-testid="message-start-shadow"
onClick={startShadow}
className="w-3 h-3 cursor-pointer"
/>
))}
<DropdownMenuTrigger>
<MoreVerticalIcon className="w-3 h-3" />
</DropdownMenuTrigger>
</div>
<DropdownMenuContent>
<DropdownMenuItem className="cursor-pointer" onClick={onRemove}>
<span className="mr-auto text-destructive capitalize">
{t("delete")}
</span>
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</div>
<Sheet open={shadowing} onOpenChange={(value) => setShadowing(value)}>

5
enjoy/src/renderer/components/messages/message.tsx
View File

@@ -6,8 +6,8 @@ import {
export const MessageComponent = (props: {
message: MessageType;
configuration: { [key: string]: any };
onResend?: () => void;
onRemove?: () => void;
onResend: () => void;
onRemove: () => void;
}) => {
const { message, configuration, onResend, onRemove } = props;
if (message.role === "assistant") {
@@ -15,6 +15,7 @@ export const MessageComponent = (props: {
<AssistantMessageComponent
message={message}
configuration={configuration}
onRemove={onRemove}
/>
);
} else if (message.role === "user") {

72
enjoy/src/renderer/components/messages/user-message.tsx
View File

@@ -12,11 +12,6 @@ import {
AvatarImage,
AvatarFallback,
Button,
Dialog,
DialogTrigger,
DialogContent,
DialogHeader,
DialogTitle,
DropdownMenu,
DropdownMenuTrigger,
DropdownMenuContent,
@@ -24,7 +19,7 @@ import {
DropdownMenuSeparator,
toast,
} from "@renderer/components/ui";
import { SpeechPlayer, ConversationsList } from "@renderer/components";
import { SpeechPlayer, ConversationShortcuts } from "@renderer/components";
import { useContext, useState } from "react";
import { AppSettingsProviderContext } from "@renderer/context";
import {
@@ -35,6 +30,7 @@ import {
CheckIcon,
Share2Icon,
ForwardIcon,
MoreVerticalIcon,
} from "lucide-react";
import { useCopyToClipboard } from "@uidotdev/usehooks";
import { t } from "i18next";
@@ -44,8 +40,8 @@ import Markdown from "react-markdown";
export const UserMessageComponent = (props: {
message: MessageType;
configuration?: { [key: string]: any };
onResend?: () => void;
onRemove?: () => void;
onResend: () => void;
onRemove: () => void;
}) => {
const { message, onResend, onRemove } = props;
const speech = message.speeches?.[0];
@@ -89,12 +85,12 @@ export const UserMessageComponent = (props: {
id={`message-${message.id}`}
className="flex items-end justify-end space-x-2 pl-10"
>
<DropdownMenu>
<div className="flex flex-col gap-2 px-4 py-2 bg-sky-500/30 border-sky-500 rounded-lg shadow-sm w-full">
<Markdown className="select-text prose">{message.content}</Markdown>
<div className="flex flex-col gap-2 px-4 py-2 bg-sky-500/30 border-sky-500 rounded-lg shadow-sm w-full">
<Markdown className="select-text prose">{message.content}</Markdown>
{Boolean(speech) && <SpeechPlayer speech={speech} />}
{Boolean(speech) && <SpeechPlayer speech={speech} />}
<DropdownMenu>
<div className="flex items-center justify-end space-x-2">
{message.createdAt ? (
<CheckCircleIcon
@@ -132,26 +128,17 @@ export const UserMessageComponent = (props: {
/>
)}
<Dialog>
<DialogTrigger>
<ConversationShortcuts
prompt={message.content}
excludedIds={[message.conversationId]}
trigger={
<ForwardIcon
data-tooltip-id="global-tooltip"
data-tooltip-content={t("forward")}
className="w-3 h-3 cursor-pointer"
/>
</DialogTrigger>
<DialogContent>
<DialogHeader>
<DialogTitle>{t("forward")}</DialogTitle>
</DialogHeader>
<div className="">
<ConversationsList
prompt={message.content}
excludedIds={[message.conversationId]}
/>
</div>
</DialogContent>
</Dialog>
}
/>
{message.createdAt && (
<AlertDialog>
@@ -180,20 +167,25 @@ export const UserMessageComponent = (props: {
</AlertDialogContent>
</AlertDialog>
)}
<DropdownMenuTrigger>
<MoreVerticalIcon className="w-3 h-3" />
</DropdownMenuTrigger>
</div>
</div>
<DropdownMenuContent>
<DropdownMenuItem onClick={onResend}>
<span className="mr-auto capitalize">{t("resend")}</span>
</DropdownMenuItem>
<DropdownMenuSeparator />
<DropdownMenuItem onClick={onRemove}>
<span className="mr-auto text-destructive capitalize">
{t("remove")}
</span>
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
<DropdownMenuContent>
<DropdownMenuItem className="cursor-pointer" onClick={onResend}>
<span className="mr-auto capitalize">{t("resend")}</span>
</DropdownMenuItem>
<DropdownMenuSeparator />
<DropdownMenuItem className="cursor-pointer" onClick={onRemove}>
<span className="mr-auto text-destructive capitalize">
{t("remove")}
</span>
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</div>
<Avatar className="w-8 h-8 bg-background">
<AvatarImage src={user.avatarUrl} />

39
enjoy/src/renderer/components/posts/post-actions.tsx
View File

@@ -1,6 +1,6 @@
import { useContext, useState } from "react";
import { AppSettingsProviderContext } from "@renderer/context";
import { ConversationsShortcut } from "@renderer/components";
import { ConversationShortcuts, SpeechPlayer } from "@renderer/components";
import {
AlertDialog,
AlertDialogTrigger,
@@ -12,12 +12,6 @@ import {
AlertDialogCancel,
AlertDialogFooter,
Button,
Dialog,
DialogTrigger,
DialogContent,
DialogHeader,
DialogTitle,
ScrollArea,
toast,
} from "@renderer/components/ui";
import { t } from "i18next";
@@ -150,8 +144,15 @@ export const PostActions = (props: { post: PostType }) => {
</Button>
)}
{post.metadata?.type === "prompt" && (
<Dialog open={asking} onOpenChange={setAsking}>
<DialogTrigger asChild>
<ConversationShortcuts
open={asking}
onOpenChange={setAsking}
prompt={post.metadata.content as string}
onReply={(replies) => {
setAiReplies([...aiReplies, ...replies]);
setAsking(false);
}}
trigger={
<Button
data-tooltip-id="global-tooltip"
data-tooltip-content={t("sendToAIAssistant")}
@@ -162,21 +163,8 @@ export const PostActions = (props: { post: PostType }) => {
>
<BotIcon className="w-5 h-5 text-muted-foreground hover:text-primary" />
</Button>
</DialogTrigger>
<DialogContent>
<DialogHeader>
<DialogTitle>{t("sendToAIAssistant")}</DialogTitle>
</DialogHeader>
<ConversationsShortcut
prompt={post.metadata.content as string}
onReply={(replies) => {
setAiReplies([...aiReplies, ...replies]);
setAsking(false);
}}
/>
</DialogContent>
<ScrollArea></ScrollArea>
</Dialog>
}
/>
)}
</div>
@@ -198,6 +186,9 @@ const AIReplies = (props: { replies: Partial<MessageType>[] }) => {
</Link>
</div>
<Markdown className="prose select-text">{reply.content}</Markdown>
{reply.speeches?.map((speech) => (
<SpeechPlayer key={speech.id} speech={speech} />
))}
</div>
))}
</div>

1
enjoy/src/renderer/components/sidebar.tsx
View File

@@ -167,6 +167,7 @@ export const Sidebar = () => {
to="/conversations"
data-tooltip-id="sidebar-tooltip"
data-tooltip-content={t("sidebar.aiAssistant")}
data-testid="sidebar-conversations"
className="block"
>
<Button

59
enjoy/src/renderer/hooks/use-conversation.tsx
View File

@@ -114,6 +114,28 @@ export const useConversation = () => {
}
): Promise<Partial<MessageType>[]> => {
const { conversation } = params;
if (conversation.type === "gpt") {
return askGPT(message, params);
} else if (conversation.type === "tts") {
return askTTS(message, params);
} else {
return [];
}
};
/*
* Ask GPT
* chat with GPT conversation
* Use LLM to generate response
*/
const askGPT = async (
message: Partial<MessageType>,
params: {
conversation: ConversationType;
}
): Promise<Partial<MessageType>[]> => {
const { conversation } = params;
const chatHistory = await fetchChatHistory(conversation);
const memory = new BufferMemory({
chatHistory,
@@ -128,7 +150,6 @@ export const useConversation = () => {
const llm = pickLlm(conversation);
const chain = new ConversationChain({
// @ts-expect-error
llm,
memory,
prompt,
@@ -160,6 +181,42 @@ export const useConversation = () => {
return replies;
};
/*
* Ask TTS
* chat with TTS conversation
* It reply with the same text
* and create speech using TTS
*/
const askTTS = async (
message: Partial<MessageType>,
params: {
conversation: ConversationType;
}
): Promise<Partial<MessageType>[]> => {
const { conversation } = params;
const reply: MessageType = {
id: v4(),
content: message.content,
role: "assistant" as MessageRoleEnum,
conversationId: conversation.id,
speeches: [],
};
message.role = "user" as MessageRoleEnum;
message.conversationId = conversation.id;
const speech = await tts({
sourceType: "Message",
sourceId: reply.id,
text: reply.content,
configuration: conversation.configuration.tts,
});
await EnjoyApp.messages.createInBatch([message, reply]);
reply.speeches = [speech];
return [reply];
};
const tts = async (params: Partial<SpeechType>) => {
const { configuration } = params;
const {

32
enjoy/src/renderer/pages/conversation.tsx
View File

@@ -146,6 +146,8 @@ export default () => {
}
scrollToMessage(record);
} else if (action === "destroy") {
dispatchMessages({ type: "destroy", record });
}
};
@@ -161,12 +163,15 @@ export default () => {
?.scrollIntoView({
behavior: "smooth",
});
inputRef.current.focus();
}, 500);
};
useEffect(() => {
setOffest(0);
setContent(searchParams.get("text") || "");
dispatchMessages({ type: "set", records: [] });
fetchConversation();
addDblistener(onMessagesUpdate);
@@ -210,7 +215,10 @@ export default () => {
}
return (
<div className="h-screen px-4 py-6 lg:px-8 bg-muted flex flex-col">
<div
data-testid="conversation-page"
className="h-screen px-4 py-6 lg:px-8 bg-muted flex flex-col"
>
<div className="h-[calc(100vh-3rem)] relative w-full max-w-screen-md mx-auto flex flex-col">
<div className="flex items-center justify-center py-2 bg-gradient-to-b from-muted relative">
<div className="cursor-pointer h-6 opacity-50 hover:opacity-100">
@@ -248,17 +256,25 @@ export default () => {
<MessageComponent
key={message.id}
message={message}
configuration={conversation.configuration}
configuration={{
type: conversation.type,
...conversation.configuration,
}}
onResend={() => {
if (message.status !== "error") return;
if (message.status === "error") {
dispatchMessages({ type: "destroy", record: message });
}
dispatchMessages({ type: "destroy", record: message });
handleSubmit(message.content);
}}
onRemove={() => {
if (message.status !== "error") return;
dispatchMessages({ type: "destroy", record: message });
if (message.status === "error") {
dispatchMessages({ type: "destroy", record: message });
} else {
EnjoyApp.messages.destroy(message.id).catch((err) => {
toast.error(err.message);
});
}
}}
/>
))}
@@ -288,12 +304,14 @@ export default () => {
value={content}
onChange={(e) => setContent(e.target.value)}
placeholder={t("pressEnterToSend")}
data-testid="conversation-page-input"
className="px-0 py-0 shadow-none border-none focus-visible:outline-0 focus-visible:ring-0 border-none bg-muted focus:bg-background min-h-[1.25rem] max-h-[3.5rem] !overflow-x-hidden"
/>
<Button
type="submit"
ref={submitRef}
disabled={submitting || !content}
data-testid="conversation-page-submit"
onClick={() => handleSubmit(content)}
className=""
>

41
enjoy/src/renderer/pages/conversations.tsx
View File

@@ -11,7 +11,7 @@ import {
} from "@renderer/components/ui";
import { ConversationForm } from "@renderer/components";
import { useState, useEffect, useContext, useReducer } from "react";
import { ChevronLeftIcon, MessageCircleIcon } from "lucide-react";
import { ChevronLeftIcon, MessageCircleIcon, SpeechIcon } from "lucide-react";
import { Link, useNavigate } from "react-router-dom";
import {
DbProviderContext,
@@ -62,9 +62,11 @@ export default () => {
const PRESETS = [
{
key: "english-coach",
name: "英语教练",
engine: currentEngine.name,
configuration: {
type: "gpt",
model: "gpt-4-1106-preview",
baseUrl: "",
roleDefinition: `你是我的英语教练。
@@ -89,9 +91,25 @@ export default () => {
},
},
{
key: "tts",
name: "TTS",
engine: currentEngine.name,
configuration: {
type: "tts",
tts: {
baseUrl: "",
engine: currentEngine.name,
model: "tts-1",
voice: "alloy",
},
},
},
{
key: "custom",
name: t("custom"),
engine: currentEngine.name,
configuration: {
type: "gpt",
model: "gpt-4-1106-preview",
baseUrl: "",
roleDefinition: "",
@@ -124,7 +142,10 @@ export default () => {
<div className="my-6 flex justify-center">
<Dialog>
<DialogTrigger asChild>
<Button className="h-12 rounded-lg w-96">
<Button
data-testid="conversation-new-button"
className="h-12 rounded-lg w-96"
>
{t("newConversation")}
</Button>
</DialogTrigger>
@@ -133,11 +154,15 @@ export default () => {
<DialogHeader>
<DialogTitle>{t("selectAiRole")}</DialogTitle>
</DialogHeader>
<div className="grid grid-cols-2 gap-4">
<div
data-testid="conversation-presets"
className="grid grid-cols-2 gap-4"
>
{PRESETS.map((preset) => (
<DialogTrigger
key={preset.name}
className="p-4 border hover:shadow rounded-lg cursor-pointer space-y-2"
key={preset.key}
data-testid={`conversation-preset-${preset.key}`}
className="p-4 border hover:shadow rounded-lg cursor-pointer space-y-2 h-32"
onClick={() => {
setPreset(preset);
setCreating(true);
@@ -181,7 +206,11 @@ export default () => {
}}
>
<div className="">
<MessageCircleIcon className="mr-2" />
{conversation.type === "gpt" && (
<MessageCircleIcon className="mr-2" />
)}
{conversation.type === "tts" && <SpeechIcon className="mr-2" />}
</div>
<div className="flex-1 flex items-center justify-between space-x-4">
<span className="line-clamp-1">{conversation.name}</span>

1
enjoy/src/types/conversation.d.ts vendored
View File

@@ -1,5 +1,6 @@
type ConversationType = {
id: string;
type: "gpt" | "tts";
engine: "enjoyai" | "openai" | "ollama" | "googleGenerativeAi";
name: string;
configuration: { [key: string]: any };

3
package.json
View File

@@ -1,8 +1,7 @@
{
"private": true,
"workspaces": [
"enjoy",
"1000-hours"
"enjoy"
],
"scripts": {
"dev:enjoy": "yarn workspace enjoy dev",

23098
yarn.lock
View File

File diff suppressed because it is too large Load Diff