From ef468e8fd67291979122e66baaeda5ca83ff0a14 Mon Sep 17 00:00:00 2001 From: milan Date: Tue, 14 Apr 2026 21:25:11 +0200 Subject: [PATCH] refactoring partially implement tts for human interface --- .gitignore | 3 +- README.md | 13 +++---- own_mcp/src/audio/client.rs | 57 +++++++++++++++++++++++------ src/config.rs | 1 + src/human_interface/audio.rs | 71 ++++++++++++++++++++++++++++++++++++ src/human_interface/cli.rs | 7 +--- src/human_interface/mod.rs | 5 +-- src/main.rs | 19 +++++++--- src/model.rs | 21 +++++++++-- 9 files changed, 160 insertions(+), 37 deletions(-) create mode 100644 src/human_interface/audio.rs diff --git a/.gitignore b/.gitignore index aafa6c3..b66c79e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /target */target assist.toml -mcp_server_collection/server.toml \ No newline at end of file +mcp_server_collection/server.toml +temporary_audio \ No newline at end of file diff --git a/README.md b/README.md index 4e42943..91f47ee 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,13 @@ permissions = [ [ollama] url = "..." # optional -model = { - from_model = "..." - name = "..." name of model to be created - system_prompt = "..." # optional - context_size = 2048 # optional - temperature = 0.8 # optional -} authorization="..." # optional +[ollama.model] +from-model = "..." +name = "..." # name of model to be created +system_prompt = "..." # optional +context_size = 2048 # optional +temperature = 0.8 # optional [mcp-servers] name="..." # optional, otherwise provided server name or random server name is chosen diff --git a/own_mcp/src/audio/client.rs b/own_mcp/src/audio/client.rs index 0913b23..2fba19f 100644 --- a/own_mcp/src/audio/client.rs +++ b/own_mcp/src/audio/client.rs @@ -1,11 +1,11 @@ -use std::path::Path; +use crate::audio::models::{AudioServerStatus, TranscriptionResponse, VoiceRequest}; use reqwest::Client; use reqwest::multipart::Form; +use std::path::Path; use thiserror::Error; -use crate::audio::models::{AudioServerStatus, TranscriptionResponse, VoiceRequest}; use url::Url; -pub struct AudioClient{ +pub struct AudioClient { client: Client, authorization: Option, base_url: Url, @@ -19,9 +19,12 @@ pub trait AudioClientTrait { fn status(&self) -> impl Future>; - fn transcribe(&self, audio_file_path: impl AsRef) -> impl Future>; + fn transcribe( + &self, + audio_file_path: impl AsRef, + ) -> impl Future>; - fn tts(&self, voice_request: VoiceRequest) -> impl Future>; + fn tts(&self, voice_request: VoiceRequest) -> impl Future>; } #[derive(Debug, Error)] @@ -36,13 +39,27 @@ pub enum AudioError { type AudioResult = Result; +impl AudioClient { + fn get(&self, url: Url) -> reqwest::RequestBuilder { + self.client.get(url).header("Authorization", self.authorization.clone().unwrap_or(String::new())) + } +} + impl AudioClientTrait for AudioClient { fn new(base_url: Url) -> Self { - AudioClient { client: Client::new(), authorization: None, base_url } + AudioClient { + client: Client::new(), + authorization: None, + base_url, + } } fn from_client(base_url: Url, client: Client) -> Self { - AudioClient { client, authorization: None, base_url } + AudioClient { + client, + authorization: None, + base_url, + } } fn with_authorization(mut self, authorization: String) -> Self { @@ -51,20 +68,36 @@ impl AudioClientTrait for AudioClient { } async fn status(&self) -> AudioResult { - let response = self.client.get(self.base_url.clone()).send().await?.error_for_status()?; + let response = self + .get(self.base_url.clone()) + .send() + .await? + .error_for_status()?; Ok(response.json().await?) } - async fn transcribe(&self, audio_file_path: impl AsRef) -> AudioResult { + async fn transcribe( + &self, + audio_file_path: impl AsRef, + ) -> AudioResult { let form = Form::new().file("audio_file", audio_file_path).await?; - let response = self.client.get(self.base_url.join("transcribe")?).multipart(form).send().await?; + let response = self + .get(self.base_url.join("transcribe")?) + .multipart(form) + .send() + .await?; Ok(response.json().await?) } async fn tts(&self, voice_request: VoiceRequest) -> AudioResult { - let response = self.client.get(self.base_url.join("tts")?).json(&voice_request).send().await?.error_for_status()?; + let response = self + .get(self.base_url.join("tts")?) + .json(&voice_request) + .send() + .await? + .error_for_status()?; Ok(response.bytes().await?) } -} \ No newline at end of file +} diff --git a/src/config.rs b/src/config.rs index 7f03c69..3cf2c3d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -105,6 +105,7 @@ pub struct OllamaConfig { #[derive(Debug, Deserialize)] pub struct OllamaModelConfig { + #[serde(rename = "from-model")] pub from_model: String, pub name: String, pub system_prompt: Option, diff --git a/src/human_interface/audio.rs b/src/human_interface/audio.rs new file mode 100644 index 0000000..07b0442 --- /dev/null +++ b/src/human_interface/audio.rs @@ -0,0 +1,71 @@ +use crate::human_interface::{HumanInterface, HumanInterfaceError}; +use own_mcp::audio::models::VoiceRequest; +use own_mcp::audio::{AudioClient, AudioClientTrait}; +use own_mcp::mcp::chat::PermissionAnswer; +use std::path::Path; +use tokio::fs::create_dir_all; + +const TEMPORARY_AUDIO_PATH: &str = "temporary_audio"; + +struct Audio { + client: AudioClient, +} + +impl Audio { + async fn new(client: AudioClient) -> Result { + create_dir_all(TEMPORARY_AUDIO_PATH) + .await + .map_err(HumanInterfaceError::IoError)?; + Ok(Self { client }) + } +} + +impl HumanInterface for Audio { + async fn agent_message(&mut self, message: String) -> Result<(), HumanInterfaceError> { + let bytes = self + .client + .tts(VoiceRequest { + text: message, + config: None, + }) + .await + .map_err(|e| HumanInterfaceError::Other(format!("Failed getting tts: {e}")))?; + + tokio::fs::write(Path::new(TEMPORARY_AUDIO_PATH).join("tts.wav"), &bytes) + .await + .map_err(|e| HumanInterfaceError::IoError(e)) + } + + async fn expect_user_message(&mut self) -> Result { + todo!() + } + + async fn ask_for_permission( + &self, + mcp_server_name: String, + tool_name: String, + ) -> Result { + todo!() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_tts_fetching() { + let mut human_interface = Audio::new( + AudioClient::new( + url::Url::parse("https://audio.mboemer.de" + ).unwrap()). + with_authorization( + "Basic bWlsYW46a3lLRVIjOVJAaDlAXldBI3pVXkAhb0pRIXViczNNV0g3U1NzQDVUenppJVFmciY5WEpxJXB1YmN6YiEkdmY1Z2FQd0N2aEd3".to_string()) + ).await.unwrap(); + + human_interface + .agent_message(String::from("Peter ist jetzt in deinem PC.")) + .await + .unwrap(); + } +} diff --git a/src/human_interface/cli.rs b/src/human_interface/cli.rs index 1eb71b9..52c9abe 100644 --- a/src/human_interface/cli.rs +++ b/src/human_interface/cli.rs @@ -22,7 +22,7 @@ impl CommandLine { } impl HumanInterface for CommandLine { - fn agent_message(&mut self, message: String) { + async fn agent_message(&mut self, message: String) -> Result<(), HumanInterfaceError> { // stopped waiting for message if let Some(progress_bar) = &self.progress_bar { progress_bar.finish_and_clear(); @@ -30,6 +30,7 @@ impl HumanInterface for CommandLine { } println!("{}: {}",style( tlt!("assistant")).bold().italic(),message); + Ok(()) } async fn expect_user_message(&mut self) -> Result { @@ -63,9 +64,5 @@ impl HumanInterface for CommandLine { false => Ok(PermissionAnswer::Denied), } } - - async fn run(&mut self) -> Result<(), HumanInterfaceError> { - Ok(()) - } } diff --git a/src/human_interface/mod.rs b/src/human_interface/mod.rs index 3674314..1866683 100644 --- a/src/human_interface/mod.rs +++ b/src/human_interface/mod.rs @@ -1,4 +1,5 @@ pub mod cli; +pub mod audio; use thiserror::Error; use own_mcp::mcp::chat::PermissionAnswer; @@ -12,11 +13,9 @@ pub enum HumanInterfaceError { } pub trait HumanInterface { - fn agent_message(&mut self, message: String); + fn agent_message(&mut self, message: String) -> impl Future>; fn expect_user_message(&mut self) -> impl Future>; fn ask_for_permission(&self, mcp_server_name: String, tool_name:String) -> impl Future>; - - fn run(&mut self) -> impl Future>; } \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 63a6066..527e7ff 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,9 +7,9 @@ use crate::config::Config; use crate::human_interface::HumanInterface; use crate::i18n::translate; use crate::model::create_model_from_config; +use base64::Engine; use clap::Parser; use std::fmt::Display; -use base64::Engine; /// partial mcp client with cli and voice interaction #[derive(Parser, Debug)] @@ -31,7 +31,7 @@ fn basic_auth_tool() { .interact() .unwrap(); - let encoded= base64::prelude::BASE64_STANDARD.encode(format!("{}:{}", username, password)); + let encoded = base64::prelude::BASE64_STANDARD.encode(format!("{}:{}", username, password)); println!("Basic {}", encoded); } @@ -86,7 +86,6 @@ async fn main() { ); let mut human_interface = human_interface::cli::CommandLine::new(); - human_interface.run().await.unwrap(); loop { let user_message = human_interface.expect_user_message().await.unwrap(); @@ -96,13 +95,21 @@ async fn main() { immutable_interface .ask_for_permission(mcp_server_name, tool_name) .await - .inspect_err(exit_msg!("Human interface error")) + .inspect_err(exit_msg!( + "Human interface error while asking for permission" + )) .unwrap() }) .await - .inspect_err(exit_msg!("failed communicating with agent")) + .inspect_err(exit_msg!("Failed communicating with agent.")) + .unwrap(); + human_interface + .agent_message(agent_message.content) + .await + .inspect_err(exit_msg!( + "Human interface error while trying to display message" + )) .unwrap(); - human_interface.agent_message(agent_message.content); } } diff --git a/src/model.rs b/src/model.rs index f0420d4..9f69b59 100644 --- a/src/model.rs +++ b/src/model.rs @@ -1,10 +1,10 @@ -use crate::translate; use crate::config::OllamaModelConfig; +use crate::tlt; +use crate::translate; use ollama_rs::Ollama; use ollama_rs::error::OllamaError; use ollama_rs::models::ModelOptions; use ollama_rs::models::create::{CreateModelRequest, CreateModelStatus}; -use crate::tlt; pub async fn create_model_from_config( ollama: &Ollama, @@ -14,7 +14,22 @@ pub async fn create_model_from_config( .num_ctx(config.context_size.unwrap_or(2048)) // 2048 is the ollama default. .temperature(config.temperature.unwrap_or(0.8)); // 0.8 is the ollama default - let system_prompt = config.system_prompt.clone().unwrap_or(tlt!("system_prompt")); + let system_prompt = config + .system_prompt + .clone() + .unwrap_or(tlt!("system_prompt")); + + // print message to warn user of long waiting times + if let Ok(models) = ollama.list_local_models().await + && !models.iter().any(|m| m.name == config.from_model) + { + let message = format!( + "Model `{}` is not installed and will be downloaded first.", + config.from_model + ); + log::info!("{}", message); + println!("{}", console::style(message).yellow().bold()); + } ollama .create_model(