refactoring

partially implement tts for human interface

.gitignore

@@ -2,3 +2,4 @@
 */target
 assist.toml
 mcp_server_collection/server.toml
+temporary_audio

README.md

@@ -6,14 +6,13 @@ permissions = [
 
 [ollama]
 url = "..." # optional
-model = {
-  from_model = "..."
-  name = "..." name of model to be created
-  system_prompt = "..." # optional
-  context_size = 2048 # optional
-  temperature = 0.8 # optional
-}
 authorization="..." # optional
+[ollama.model]
+from-model = "..."
+name = "..." #  name of model to be created
+system_prompt = "..." # optional
+context_size = 2048 # optional
+temperature = 0.8 # optional
 
 [mcp-servers]
 name="..." # optional, otherwise provided server name or random server name is chosen

own_mcp/src/audio/client.rs

@@ -1,11 +1,11 @@
-use std::path::Path;
+use crate::audio::models::{AudioServerStatus, TranscriptionResponse, VoiceRequest};
 use reqwest::Client;
 use reqwest::multipart::Form;
+use std::path::Path;
 use thiserror::Error;
-use crate::audio::models::{AudioServerStatus, TranscriptionResponse, VoiceRequest};
 use url::Url;
 
-pub struct AudioClient{
+pub struct AudioClient {
     client: Client,
     authorization: Option<String>,
     base_url: Url,
@@ -19,9 +19,12 @@ pub trait AudioClientTrait {
 
     fn status(&self) -> impl Future<Output = Result<AudioServerStatus, AudioError>>;
 
-    fn transcribe(&self, audio_file_path: impl AsRef<Path>) -> impl Future<Output=AudioResult<TranscriptionResponse>>;
+    fn transcribe(
+        &self,
+        audio_file_path: impl AsRef<Path>,
+    ) -> impl Future<Output = AudioResult<TranscriptionResponse>>;
 
-    fn tts(&self, voice_request: VoiceRequest) -> impl Future<Output=AudioResult<bytes::Bytes>>;
+    fn tts(&self, voice_request: VoiceRequest) -> impl Future<Output = AudioResult<bytes::Bytes>>;
 }
 
 #[derive(Debug, Error)]
@@ -36,13 +39,27 @@ pub enum AudioError {
 
 type AudioResult<T> = Result<T, AudioError>;
 
+impl AudioClient {
+    fn get(&self, url: Url) -> reqwest::RequestBuilder {
+        self.client.get(url).header("Authorization", self.authorization.clone().unwrap_or(String::new()))
+    }
+}
+
 impl AudioClientTrait for AudioClient {
     fn new(base_url: Url) -> Self {
-        AudioClient { client: Client::new(), authorization: None, base_url }
+        AudioClient {
+            client: Client::new(),
+            authorization: None,
+            base_url,
+        }
     }
 
     fn from_client(base_url: Url, client: Client) -> Self {
-        AudioClient { client, authorization: None, base_url }
+        AudioClient {
+            client,
+            authorization: None,
+            base_url,
+        }
     }
 
     fn with_authorization(mut self, authorization: String) -> Self {
@@ -51,19 +68,35 @@ impl AudioClientTrait for AudioClient {
     }
 
     async fn status(&self) -> AudioResult<AudioServerStatus> {
-        let response = self.client.get(self.base_url.clone()).send().await?.error_for_status()?;
+        let response = self
+            .get(self.base_url.clone())
+            .send()
+            .await?
+            .error_for_status()?;
         Ok(response.json().await?)
     }
 
-    async fn transcribe(&self, audio_file_path: impl AsRef<Path>) -> AudioResult<TranscriptionResponse> {
+    async fn transcribe(
+        &self,
+        audio_file_path: impl AsRef<Path>,
+    ) -> AudioResult<TranscriptionResponse> {
         let form = Form::new().file("audio_file", audio_file_path).await?;
 
-        let response = self.client.get(self.base_url.join("transcribe")?).multipart(form).send().await?;
+        let response = self
+            .get(self.base_url.join("transcribe")?)
+            .multipart(form)
+            .send()
+            .await?;
         Ok(response.json().await?)
     }
 
     async fn tts(&self, voice_request: VoiceRequest) -> AudioResult<bytes::Bytes> {
-        let response = self.client.get(self.base_url.join("tts")?).json(&voice_request).send().await?.error_for_status()?;
+        let response = self
+            .get(self.base_url.join("tts")?)
+            .json(&voice_request)
+            .send()
+            .await?
+            .error_for_status()?;
 
         Ok(response.bytes().await?)
     }

src/config.rs

@@ -105,6 +105,7 @@ pub struct OllamaConfig {
 
 #[derive(Debug, Deserialize)]
 pub struct OllamaModelConfig {
+    #[serde(rename = "from-model")]
     pub from_model: String,
     pub name: String,
     pub system_prompt: Option<String>,

src/human_interface/audio.rs

@@ -0,0 +1,71 @@
+use crate::human_interface::{HumanInterface, HumanInterfaceError};
+use own_mcp::audio::models::VoiceRequest;
+use own_mcp::audio::{AudioClient, AudioClientTrait};
+use own_mcp::mcp::chat::PermissionAnswer;
+use std::path::Path;
+use tokio::fs::create_dir_all;
+
+const TEMPORARY_AUDIO_PATH: &str = "temporary_audio";
+
+struct Audio {
+    client: AudioClient,
+}
+
+impl Audio {
+    async fn new(client: AudioClient) -> Result<Self, HumanInterfaceError> {
+        create_dir_all(TEMPORARY_AUDIO_PATH)
+            .await
+            .map_err(HumanInterfaceError::IoError)?;
+        Ok(Self { client })
+    }
+}
+
+impl HumanInterface for Audio {
+    async fn agent_message(&mut self, message: String) -> Result<(), HumanInterfaceError> {
+        let bytes = self
+            .client
+            .tts(VoiceRequest {
+                text: message,
+                config: None,
+            })
+            .await
+            .map_err(|e| HumanInterfaceError::Other(format!("Failed getting tts: {e}")))?;
+
+        tokio::fs::write(Path::new(TEMPORARY_AUDIO_PATH).join("tts.wav"), &bytes)
+            .await
+            .map_err(|e| HumanInterfaceError::IoError(e))
+    }
+
+    async fn expect_user_message(&mut self) -> Result<String, HumanInterfaceError> {
+        todo!()
+    }
+
+    async fn ask_for_permission(
+        &self,
+        mcp_server_name: String,
+        tool_name: String,
+    ) -> Result<PermissionAnswer, HumanInterfaceError> {
+        todo!()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_tts_fetching() {
+        let mut human_interface = Audio::new(
+            AudioClient::new(
+                url::Url::parse("https://audio.mboemer.de"
+                ).unwrap()).
+                with_authorization(
+                    "Basic bWlsYW46a3lLRVIjOVJAaDlAXldBI3pVXkAhb0pRIXViczNNV0g3U1NzQDVUenppJVFmciY5WEpxJXB1YmN6YiEkdmY1Z2FQd0N2aEd3".to_string())
+        ).await.unwrap();
+
+        human_interface
+            .agent_message(String::from("Peter ist jetzt in deinem PC."))
+            .await
+            .unwrap();
+    }
+}

src/human_interface/cli.rs

@@ -22,7 +22,7 @@ impl CommandLine {
 }
 
 impl HumanInterface for CommandLine {
-    fn agent_message(&mut self, message: String) {
+    async fn agent_message(&mut self, message: String) -> Result<(), HumanInterfaceError> {
         // stopped waiting for message
         if let Some(progress_bar) = &self.progress_bar {
             progress_bar.finish_and_clear();
@@ -30,6 +30,7 @@ impl HumanInterface for CommandLine {
         }
 
         println!("{}: {}",style( tlt!("assistant")).bold().italic(),message);
+        Ok(())
     }
 
     async fn expect_user_message(&mut self) -> Result<String, HumanInterfaceError> {
@@ -63,9 +64,5 @@ impl HumanInterface for CommandLine {
             false => Ok(PermissionAnswer::Denied),
         }
     }
-
-    async fn run(&mut self) -> Result<(), HumanInterfaceError> {
-        Ok(())
-    }
 }
 

src/human_interface/mod.rs

@@ -1,4 +1,5 @@
 pub mod cli;
+pub mod audio;
 
 use thiserror::Error;
 use own_mcp::mcp::chat::PermissionAnswer;
@@ -12,11 +13,9 @@ pub enum HumanInterfaceError {
 }
 
 pub trait HumanInterface {
-    fn agent_message(&mut self, message: String);
+    fn agent_message(&mut self, message: String) -> impl Future<Output = Result<(), HumanInterfaceError>>;
 
     fn expect_user_message(&mut self) -> impl Future<Output = Result<String, HumanInterfaceError>>;
 
     fn ask_for_permission(&self, mcp_server_name: String, tool_name:String) -> impl Future<Output = Result<PermissionAnswer, HumanInterfaceError>>;
-
-    fn run(&mut self) -> impl Future<Output = Result<(), HumanInterfaceError>>;
 }

src/main.rs

@@ -7,9 +7,9 @@ use crate::config::Config;
 use crate::human_interface::HumanInterface;
 use crate::i18n::translate;
 use crate::model::create_model_from_config;
+use base64::Engine;
 use clap::Parser;
 use std::fmt::Display;
-use base64::Engine;
 
 /// partial mcp client with cli and voice interaction
 #[derive(Parser, Debug)]
@@ -31,7 +31,7 @@ fn basic_auth_tool() {
         .interact()
         .unwrap();
 
-    let encoded= base64::prelude::BASE64_STANDARD.encode(format!("{}:{}", username, password));
+    let encoded = base64::prelude::BASE64_STANDARD.encode(format!("{}:{}", username, password));
     println!("Basic {}", encoded);
 }
 
@@ -86,7 +86,6 @@ async fn main() {
     );
 
     let mut human_interface = human_interface::cli::CommandLine::new();
-    human_interface.run().await.unwrap();
 
     loop {
         let user_message = human_interface.expect_user_message().await.unwrap();
@@ -96,13 +95,21 @@ async fn main() {
                 immutable_interface
                     .ask_for_permission(mcp_server_name, tool_name)
                     .await
-                    .inspect_err(exit_msg!("Human interface error"))
+                    .inspect_err(exit_msg!(
+                        "Human interface error while asking for permission"
+                    ))
                     .unwrap()
             })
             .await
-            .inspect_err(exit_msg!("failed communicating with agent"))
+            .inspect_err(exit_msg!("Failed communicating with agent."))
+            .unwrap();
+        human_interface
+            .agent_message(agent_message.content)
+            .await
+            .inspect_err(exit_msg!(
+                "Human interface error while trying to display message"
+            ))
             .unwrap();
-        human_interface.agent_message(agent_message.content);
     }
 }
 

src/model.rs

@@ -1,10 +1,10 @@
-use crate::translate;
 use crate::config::OllamaModelConfig;
+use crate::tlt;
+use crate::translate;
 use ollama_rs::Ollama;
 use ollama_rs::error::OllamaError;
 use ollama_rs::models::ModelOptions;
 use ollama_rs::models::create::{CreateModelRequest, CreateModelStatus};
-use crate::tlt;
 
 pub async fn create_model_from_config(
     ollama: &Ollama,
@@ -14,7 +14,22 @@ pub async fn create_model_from_config(
         .num_ctx(config.context_size.unwrap_or(2048)) // 2048 is the ollama default.
         .temperature(config.temperature.unwrap_or(0.8)); // 0.8 is the ollama default
 
-    let system_prompt = config.system_prompt.clone().unwrap_or(tlt!("system_prompt"));
+    let system_prompt = config
+        .system_prompt
+        .clone()
+        .unwrap_or(tlt!("system_prompt"));
+
+    // print message to warn user of long waiting times
+    if let Ok(models) = ollama.list_local_models().await
+        && !models.iter().any(|m| m.name == config.from_model)
+    {
+        let message = format!(
+            "Model `{}` is not installed and will be downloaded first.",
+            config.from_model
+        );
+        log::info!("{}", message);
+        println!("{}", console::style(message).yellow().bold());
+    }
 
     ollama
         .create_model(