diff --git a/enjoy/e2e/main.spec.ts b/enjoy/e2e/main.spec.ts index c440ff27..d3822bb3 100644 --- a/enjoy/e2e/main.spec.ts +++ b/enjoy/e2e/main.spec.ts @@ -18,15 +18,6 @@ declare global { } } -const user = { - id: 24000001, - name: "李安", - avatarUrl: - "https://mixin-images.zeromesh.net/9tMscDkZuXyLKMRChmFi5IiFF2XuQHO8PQpED8zKOCBDGKGSVB9J2eqzyjhgJKPDVunXiT-DPiisImX_bhBDPi4=s256", - accessToken: - "eyJhbGciOiJIUzI1NiJ9.eyJpc3MiOm51bGwsInNpZCI6IjkyN2RjNGRhLTI3YTItNDU5MC1hY2ZiLWMxYTJmZjhhMmFjMiIsInVpZCI6MjQwMDAwMDEsImlhdCI6MTcwODMyODk1N30.PCN_SZ7JH-VYLl56XU8kxYN9Cy44sO13mBQNNz6x-pa", -}; - let electronApp: ElectronApplication; const resultDir = path.join(process.cwd(), "test-results"); @@ -65,61 +56,27 @@ test.afterAll(async () => { await electronApp.close(); }); -test.describe("main dependencies", () => { - test("validate whisper command", async () => { - const page = await electronApp.firstWindow(); - const res = await page.evaluate(() => { - return window.__ENJOY_APP__.whisper.check(); - }); - console.info(res.log); - expect(res.success).toBeTruthy(); - - const settings = fs.readJsonSync(path.join(resultDir, "settings.json")); - expect(settings.whisper.service).toBe("local"); +test("validate whisper command", async () => { + const page = await electronApp.firstWindow(); + const res = await page.evaluate(() => { + return window.__ENJOY_APP__.whisper.check(); }); + console.info(res.log); + expect(res.success).toBeTruthy(); - test("valid ffmpeg command", async () => { - const page = await electronApp.firstWindow(); - const res = await page.evaluate(() => { - return window.__ENJOY_APP__.ffmpeg.check(); - }); - expect(res).toBeTruthy(); - }); - - test("should setup default library path", async () => { - const settings = fs.readJsonSync(path.join(resultDir, "settings.json")); - expect(settings.library).not.toBeNull(); - }); + const settings = fs.readJsonSync(path.join(resultDir, "settings.json")); + expect(settings.whisper.service).toBe("local"); }); -test.describe("with login", async () => { - let page: Page; - - test.beforeAll(async () => { - const settings = fs.readJsonSync(path.join(resultDir, "settings.json")); - settings.user = user; - fs.writeJsonSync(path.join(resultDir, "settings.json"), settings); - - page = await electronApp.firstWindow(); - page.route("**/api/me", (route) => { - route.fulfill({ - json: user, - }); - }); - - await page.evaluate(() => { - return window.__ENJOY_APP__.app.reload(); - }); - }); - - test("should enter homepage after login", async () => { - await page.waitForSelector("[data-testid=layout-home]"); - - await page.screenshot({ path: "test-results/homepage.png" }); - - expect(await page.getByTestId("layout-onboarding").isVisible()).toBeFalsy(); - expect(await page.getByTestId("layout-db-error").isVisible()).toBeFalsy(); - expect(await page.getByTestId("layout-home").isVisible()).toBeTruthy(); - expect(await page.getByTestId("sidebar").isVisible()).toBeTruthy(); +test("valid ffmpeg command", async () => { + const page = await electronApp.firstWindow(); + const res = await page.evaluate(() => { + return window.__ENJOY_APP__.ffmpeg.check(); }); + expect(res).toBeTruthy(); +}); + +test("should setup default library path", async () => { + const settings = fs.readJsonSync(path.join(resultDir, "settings.json")); + expect(settings.library).not.toBeNull(); }); diff --git a/enjoy/e2e/renderer.spec.ts b/enjoy/e2e/renderer.spec.ts new file mode 100644 index 00000000..e7aa9b24 --- /dev/null +++ b/enjoy/e2e/renderer.spec.ts @@ -0,0 +1,201 @@ +import { expect, test } from "@playwright/test"; +import { findLatestBuild, parseElectronApp } from "electron-playwright-helpers"; +import { ElectronApplication, Page, _electron as electron } from "playwright"; +import path from "path"; +import fs from "fs-extra"; + +const user = { + id: 24000001, + name: "李安", + avatarUrl: + "https://mixin-images.zeromesh.net/9tMscDkZuXyLKMRChmFi5IiFF2XuQHO8PQpED8zKOCBDGKGSVB9J2eqzyjhgJKPDVunXiT-DPiisImX_bhBDPi4=s256", + accessToken: + "eyJhbGciOiJIUzI1NiJ9.eyJpc3MiOm51bGwsInNpZCI6IjkyN2RjNGRhLTI3YTItNDU5MC1hY2ZiLWMxYTJmZjhhMmFjMiIsInVpZCI6MjQwMDAwMDEsImlhdCI6MTcwODMyODk1N30.PCN_SZ7JH-VYLl56XU8kxYN9Cy44sO13mBQNNz6x-pa", +}; + +let electronApp: ElectronApplication; +const resultDir = path.join(process.cwd(), "test-results"); + +test.beforeAll(async () => { + // find the latest build in the out directory + const latestBuild = findLatestBuild(); + // parse the directory and find paths and other info + const appInfo = parseElectronApp(latestBuild); + // set the CI environment variable to true + process.env.CI = "e2e"; + + fs.ensureDirSync(resultDir); + process.env.SETTINGS_PATH = resultDir; + process.env.LIBRARY_PATH = resultDir; + + electronApp = await electron.launch({ + args: [appInfo.main], + executablePath: appInfo.executable, + }); + electronApp.on("window", async (page) => { + const filename = page.url()?.split("/").pop(); + console.info(`Window opened: ${filename}`); + + // capture errors + page.on("pageerror", (error) => { + console.error(error); + }); + // capture console messages + page.on("console", (msg) => { + console.info(msg.text()); + }); + }); +}); + +test.afterAll(async () => { + await electronApp.close(); +}); + +test.describe("with login", async () => { + let page: Page; + + test.beforeAll(async () => { + const settings = fs.readJsonSync(path.join(resultDir, "settings.json")); + settings.user = user; + fs.writeJsonSync(path.join(resultDir, "settings.json"), settings); + + page = await electronApp.firstWindow(); + page.route("**/api/me", (route) => { + route.fulfill({ + json: user, + }); + }); + + await page.evaluate(() => { + return window.__ENJOY_APP__.app.reload(); + }); + }); + + test("should enter homepage after login", async () => { + await page.getByTestId("layout-home").waitFor(); + + await page.screenshot({ path: "test-results/homepage.png" }); + + expect(await page.getByTestId("layout-onboarding").isVisible()).toBeFalsy(); + expect(await page.getByTestId("layout-db-error").isVisible()).toBeFalsy(); + expect(await page.getByTestId("layout-home").isVisible()).toBeTruthy(); + expect(await page.getByTestId("sidebar").isVisible()).toBeTruthy(); + }); + + test.describe("with conversation", async () => { + test.beforeEach(async () => { + const file = fs.readFileSync( + path.join(process.cwd(), "samples", "speech.mp3") + ); + + page = await electronApp.firstWindow(); + page.on("console", (msg) => { + console.info(msg.text()); + }); + + await page.route("**/api/ai/audio/speech", (route) => { + route.fulfill({ + body: file, + }); + }); + await page.route("**/api/ai/chat/completions", (route) => { + route.fulfill({ + json: { + id: "1", + choices: [ + { + index: 1, + message: { + role: "assistant", + content: "I'm fine, thank you.", + }, + finish_reason: "stop", + }, + ], + }, + }); + }); + + // navigate to the conversations page + await page.getByTestId("sidebar-conversations").click(); + }); + + /* + * steps: + * 1. create a gpt conversation + * 2. submit a message to the conversation, AI should reply + * 3. create a speech from the AI message + * 4. add the speech to the library + * 5. audio waveform player should be visible and transcription should be generated + */ + test("gpt conversation", async () => { + // trigger new conversation modal + await page.getByTestId("conversation-new-button").click(); + + // create a gpt conversation + await page.getByTestId("conversation-preset-english-coach").click(); + await page.getByTestId("conversation-form").waitFor(); + await page.click("[data-testid=conversation-form-submit]"); + + // wait for the conversation to be created + await page.getByTestId("conversation-page").waitFor(); + + // submit a message to the conversation + await page.getByTestId("conversation-page-input").fill("How are you?"); + await page.getByTestId("conversation-page-submit").click(); + await page.locator(".ai-message").waitFor(); + const message = page.locator(".ai-message").first(); + expect(await message.isVisible()).toBeTruthy(); + + // create a speech + await page.getByTestId("message-create-speech").click(); + + // wait for the speech player + const player = page + .locator(".ai-message") + .getByTestId("wavesurfer-container"); + await player.waitFor(); + expect(await player.isVisible()).toBeTruthy(); + + // add to library + await page.getByTestId("message-start-shadow").click(); + await page.getByTestId("audio-detail").waitFor(); + await page.getByTestId("media-player-container").waitFor(); + await page.getByTestId("media-transcription").waitFor(); + await page.getByTestId("media-transcription-result").waitFor(); + expect( + await page.getByTestId("media-transcription-result").isVisible() + ).toBeTruthy(); + }); + + /* + * steps: + * 1. create a tts conversation + * 2. submit a message to the conversation + * 3. the speech should auto create + */ + test("tts conversation", async () => { + // trigger new conversation modal + await page.getByTestId("conversation-new-button").click(); + + // create a tts conversation + await page.click("[data-testid=conversation-preset-tts]"); + await page.getByTestId("conversation-form").waitFor(); + await page.click("[data-testid=conversation-form-submit]"); + + // wait for the conversation to be created + await page.getByTestId("conversation-page").waitFor(); + + // submit a message to the conversation + await page.getByTestId("conversation-page-input").fill("How are you?"); + await page.getByTestId("conversation-page-submit").click(); + await page.locator(".ai-message").waitFor(); + const player = page + .locator(".ai-message") + .getByTestId("wavesurfer-container"); + await player.waitFor(); + + expect(await player.isVisible()).toBeTruthy(); + }); + }); +}); diff --git a/enjoy/package.json b/enjoy/package.json index 1270dd80..9deca7a4 100644 --- a/enjoy/package.json +++ b/enjoy/package.json @@ -45,15 +45,15 @@ "@types/lodash": "^4.14.202", "@types/mark.js": "^8.11.12", "@types/node": "^20.11.19", - "@types/react": "^18.2.56", + "@types/react": "^18.2.57", "@types/react-dom": "^18.2.19", "@types/validator": "^13.11.9", "@types/wavesurfer.js": "^6.0.12", - "@typescript-eslint/eslint-plugin": "^7.0.1", - "@typescript-eslint/parser": "^7.0.1", + "@typescript-eslint/eslint-plugin": "^7.0.2", + "@typescript-eslint/parser": "^7.0.2", "@vitejs/plugin-react": "^4.2.1", "autoprefixer": "^10.4.17", - "electron": "^28.2.0", + "electron": "^28.2.3", "electron-playwright-helpers": "^1.7.1", "eslint": "^8.56.0", "eslint-import-resolver-typescript": "^3.6.1", @@ -101,7 +101,7 @@ "@radix-ui/react-toggle": "^1.0.3", "@radix-ui/react-tooltip": "^1.0.7", "@uidotdev/usehooks": "^2.4.1", - "@vidstack/react": "^1.10.7", + "@vidstack/react": "^1.10.9", "autosize": "^6.0.1", "axios": "^1.6.7", "camelcase": "^8.0.0", @@ -129,7 +129,7 @@ "js-md5": "^0.8.3", "langchain": "^0.1.20", "lodash": "^4.17.21", - "lucide-react": "^0.334.0", + "lucide-react": "^0.335.0", "mark.js": "^8.11.1", "microsoft-cognitiveservices-speech-sdk": "^1.35.0", "next-themes": "^0.2.1", diff --git a/enjoy/samples/speech.mp3 b/enjoy/samples/speech.mp3 new file mode 100644 index 00000000..8501460a Binary files /dev/null and b/enjoy/samples/speech.mp3 differ diff --git a/enjoy/src/i18n/en.json b/enjoy/src/i18n/en.json index 0c0bceb3..78e1e6f1 100644 --- a/enjoy/src/i18n/en.json +++ b/enjoy/src/i18n/en.json @@ -66,9 +66,10 @@ "name": "Name", "engine": "AI engine", "baseUrl": "Request endpoint", - "baseUrlDescription": "BaseURL, leave it blank if you don't have one", + "baseUrlDescription": "leave it blank if you don't have one", "configuration": "Configuration", "model": "AI model", + "type": "AI Type", "roleDefinition": "Role definition", "roleDefinitionPlaceholder": "Describe the AI role", "temperature": "Temperature", @@ -87,7 +88,7 @@ "ttsModel": "TTS model", "ttsVoice": "TTS voice", "ttsBaseUrl": "TTS base URL", - "ttsBaseUrlDescription": "BaseURL for TTS, leave it blank if you don't have one", + "ttsBaseUrlDescription": "leave it blank if you don't have one", "notFound": "Conversation not found", "contentRequired": "Content required", "failedToGenerateResponse": "Failed to generate response, please retry" @@ -461,5 +462,7 @@ "itMayTakeAWhileToPrepareForTheFirstLoad": "It may take a while to prepare for the first load. Please be patient.", "loadingTranscription": "Loading transcription", "cannotFindMicrophone": "Cannot find microphone", - "failedToSaveRecording": "Failed to save recording" + "failedToSaveRecording": "Failed to save recording", + "speechNotCreatedYet": "Speech not created yet", + "goToConversation": "Go to conversation" } diff --git a/enjoy/src/i18n/zh-CN.json b/enjoy/src/i18n/zh-CN.json index d1f6411f..da86749e 100644 --- a/enjoy/src/i18n/zh-CN.json +++ b/enjoy/src/i18n/zh-CN.json @@ -66,9 +66,10 @@ "name": "对话标题", "engine": "AI 引擎", "baseUrl": "接口地址", - "baseUrlDescription": "接口地址,留空则使用默认值", + "baseUrlDescription": "留空则使用默认值", "configuration": "AI 配置", "model": "AI 模型", + "type": "AI 类型", "roleDefinition": "角色定义", "roleDefinitionPlaceholder": "描述 AI 扮演的角色", "temperature": "随机性 (temperature)", @@ -87,7 +88,7 @@ "ttsModel": "TTS 模型", "ttsVoice": "TTS 声音", "ttsBaseUrl": "TTS 接口地址", - "ttsBaseUrlDescription": "TTS 接口地址,留空则使用默认值", + "ttsBaseUrlDescription": "留空则使用默认值", "notFound": "未找到对话", "contentRequired": "对话内容不能为空", "failedToGenerateResponse": "生成失败,请重试" @@ -460,5 +461,7 @@ "itMayTakeAWhileToPrepareForTheFirstLoad": "首次加载可能需要一些时间,请耐心等候", "loadingTranscription": "正在加载语音文本", "cannotFindMicrophone": "无法找到麦克风", - "failedToSaveRecording": "保存录音失败" + "failedToSaveRecording": "保存录音失败", + "speechNotCreatedYet": "尚未生成语音", + "goToConversation": "前往对话" } diff --git a/enjoy/src/main/db/models/conversation.ts b/enjoy/src/main/db/models/conversation.ts index 8ca2ec1e..0d1f2e54 100644 --- a/enjoy/src/main/db/models/conversation.ts +++ b/enjoy/src/main/db/models/conversation.ts @@ -18,7 +18,10 @@ import { ConversationChain } from "langchain/chains"; import { ChatOpenAI } from "@langchain/openai"; import { ChatOllama } from "@langchain/community/chat_models/ollama"; import { ChatGoogleGenerativeAI } from "@langchain/google-genai"; -import { ChatPromptTemplate, MessagesPlaceholder } from "@langchain/core/prompts"; +import { + ChatPromptTemplate, + MessagesPlaceholder, +} from "@langchain/core/prompts"; import { type Generation } from "langchain/dist/schema"; import settings from "@main/settings"; import db from "@main/db"; @@ -58,11 +61,17 @@ export class Conversation extends Model { @Column(DataType.JSON) configuration: { model: string; + type: "gpt" | "tts"; roleDefinition?: string; temperature?: number; maxTokens?: number; } & { [key: string]: any }; + @Column(DataType.VIRTUAL) + get type(): 'gpt' | 'tts' { + return this.getDataValue("configuration").type || "gpt"; + } + @Column(DataType.VIRTUAL) get model(): string { return this.getDataValue("configuration").model; diff --git a/enjoy/src/renderer/components/audios/audio-detail.tsx b/enjoy/src/renderer/components/audios/audio-detail.tsx index 35e634a3..e2566b39 100644 --- a/enjoy/src/renderer/components/audios/audio-detail.tsx +++ b/enjoy/src/renderer/components/audios/audio-detail.tsx @@ -206,7 +206,7 @@ export const AudioDetail = (props: { id?: string; md5?: string }) => { } return ( -
+
m.name ); } catch (e) { - console.error(e); + console.warn(`No ollama server found: ${e.message}`); } setProviders({ ...providers }); }; @@ -112,7 +113,8 @@ export const ConversationForm = (props: { defaultConfig.configuration.baseUrl = openai.baseUrl; } } - if (defaultConfig.configuration.tts.engine === "openai" && openai) { + + if (defaultConfig.configuration.tts?.engine === "openai" && openai) { if (!defaultConfig.configuration.tts.baseUrl) { defaultConfig.configuration.tts.baseUrl = openai.baseUrl; } @@ -126,10 +128,8 @@ export const ConversationForm = (props: { name: conversation.name, engine: conversation.engine, configuration: { + type: conversation.configuration.type || "gpt", ...conversation.configuration, - tts: { - ...conversation.configuration?.tts, - }, }, } : { @@ -146,16 +146,24 @@ export const ConversationForm = (props: { setSubmitting(true); Object.keys(configuration).forEach((key) => { + if (key === "type") return; + if (!LLM_PROVIDERS[engine]?.configurable.includes(key)) { // @ts-ignore delete configuration[key]; } }); + if (configuration.type === "tts") { + conversation.model = configuration.tts.model; + } + + // use default base url if not set if (!configuration.baseUrl) { configuration.baseUrl = LLM_PROVIDERS[engine]?.baseUrl; } + // use default base url if not set if (!configuration.tts.baseUrl) { configuration.tts.baseUrl = LLM_PROVIDERS[engine]?.baseUrl; } @@ -193,6 +201,7 @@ export const ConversationForm = (props: {
{conversation.id ? t("editConversation") : t("startConversation")} @@ -213,10 +222,10 @@ export const ConversationForm = (props: { ( - {t("models.conversation.engine")} + {t("models.conversation.type")} - - {providers[form.watch("engine")]?.description} - - - - )} - /> - - ( - - {t("models.conversation.model")} - @@ -270,227 +250,306 @@ export const ConversationForm = (props: { )} /> - ( - - - {t("models.conversation.roleDefinition")} - -