diff --git a/.gitignore b/.gitignore index b66c79e..11d96b0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ */target assist.toml mcp_server_collection/server.toml -temporary_audio \ No newline at end of file +temporary_audio +src/human_interface/.AUTH_HEADER \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index df63387..6e92a90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,7 +18,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812947049edcd670a82cd5c73c3661d2e58468577ba8489de58e1a73c04cbd5d" dependencies = [ "alsa-sys", - "bitflags", + "bitflags 2.11.0", "cfg-if", "libc", ] @@ -98,6 +98,12 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "async-stream" version = "0.3.6" @@ -171,6 +177,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.11.0" @@ -201,6 +213,12 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + [[package]] name = "bytes" version = "1.11.1" @@ -371,7 +389,7 @@ version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16dd574a72a021b90c7656c474ea31d11a2f0366a8eff574186e761e0b9e3586" dependencies = [ - "bitflags", + "bitflags 2.11.0", "libc", "objc2-audio-toolbox", "objc2-core-audio", @@ -505,7 +523,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags", + "bitflags 2.11.0", "objc2", ] @@ -586,6 +604,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fastrand" version = "2.4.0" @@ -1224,6 +1248,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "leb128fmt" version = "0.1.0" @@ -1236,6 +1266,12 @@ version = "0.2.184" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1325,7 +1361,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4" dependencies = [ - "bitflags", + "bitflags 2.11.0", "jni-sys 0.3.1", "log", "ndk-sys", @@ -1354,12 +1390,22 @@ version = "0.31.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cfg-if", "cfg_aliases", "libc", ] +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-derive" version = "0.4.2" @@ -1371,6 +1417,26 @@ dependencies = [ "syn", ] +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1378,6 +1444,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1436,7 +1503,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" dependencies = [ - "bitflags", + "bitflags 2.11.0", "libc", "objc2", "objc2-core-audio", @@ -1474,7 +1541,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" dependencies = [ - "bitflags", + "bitflags 2.11.0", "objc2", ] @@ -1484,7 +1551,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags", + "bitflags 2.11.0", "block2", "dispatch2", "libc", @@ -1503,7 +1570,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "bitflags", + "bitflags 2.11.0", "block2", "libc", "objc2", @@ -1558,7 +1625,7 @@ version = "0.10.76" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cfg-if", "foreign-types", "libc", @@ -1603,6 +1670,7 @@ dependencies = [ "base64", "clap", "console", + "cpal", "dialoguer", "env_logger", "indicatif", @@ -1611,6 +1679,7 @@ dependencies = [ "own_assist_common", "own_mcp", "rmcp", + "rodio", "serde", "thiserror 2.0.18", "tokio", @@ -1897,6 +1966,16 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" +[[package]] +name = "rand_distr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d431c2703ccf129de4d45253c03f49ebb22b97d6ad79ee3ecfc7e3f4862c1d8" +dependencies = [ + "num-traits", + "rand 0.10.0", +] + [[package]] name = "ref-cast" version = "1.0.25" @@ -2083,6 +2162,28 @@ dependencies = [ "syn", ] +[[package]] +name = "rodio" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a536bb79db59098ef71a4dd4246c02eb87b316deceb1b68e0cde7167ec01eb" +dependencies = [ + "cpal", + "dasp_sample", + "num-rational", + "rand 0.10.0", + "rand_distr", + "rtrb", + "symphonia", + "thiserror 2.0.18", +] + +[[package]] +name = "rtrb" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7204ed6420f698836b76d4d5c2ec5dec7585fd5c3a788fd1cde855d1de598239" + [[package]] name = "rustc-hash" version = "2.1.2" @@ -2095,7 +2196,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.11.0", "errno", "libc", "linux-raw-sys", @@ -2239,7 +2340,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags", + "bitflags 2.11.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -2441,6 +2542,153 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symphonia" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-pcm", + "symphonia-codec-vorbis", + "symphonia-core", + "symphonia-format-isomp4", + "symphonia-format-ogg", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91565e180aea25d9b80a910c546802526ffd0072d0b8974e3ebe59b686c9976" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4872dd6bb56bf5eac799e3e957aa1981086c3e613b27e0ac23b176054f7c57ed" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c263845aa86881416849c1729a54c7f55164f8b96111dba59de46849e73a790" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e89d716c01541ad3ebe7c91ce4c8d38a7cf266a3f7b2f090b108fb0cb031d95" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f025837c309cd69ffef572750b4a2257b59552c5399a5e49707cc5b1b85d1c73" +dependencies = [ + "log", + "symphonia-core", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-core" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea00cc4f79b7f6bb7ff87eddc065a1066f3a43fe1875979056672c9ef948c2af" +dependencies = [ + "arrayvec", + "bitflags 1.3.2", + "bytemuck", + "lazy_static", + "log", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "243739585d11f81daf8dac8d9f3d18cc7898f6c09a259675fc364b382c30e0a5" +dependencies = [ + "encoding_rs", + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b4955c67c1ed3aa8ae8428d04ca8397fbef6a19b2b051e73b5da8b1435639cb" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d7c3df0e7d94efb68401d81906eae73c02b40d5ec1a141962c592d0f11a96f" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36306ff42b9ffe6e5afc99d49e121e0bd62fe79b9db7b9681d48e29fa19e6b16" +dependencies = [ + "encoding_rs", + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-utils-xiph" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27c85ab799a338446b68eec77abf42e1a6f1bb490656e121c6e27bfbab9f16" +dependencies = [ + "symphonia-core", + "symphonia-metadata", +] + [[package]] name = "syn" version = "2.0.117" @@ -2478,7 +2726,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags", + "bitflags 2.11.0", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -2714,7 +2962,7 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ - "bitflags", + "bitflags 2.11.0", "bytes", "futures-util", "http", @@ -2993,7 +3241,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags", + "bitflags 2.11.0", "hashbrown 0.15.5", "indexmap", "semver", @@ -3447,7 +3695,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags", + "bitflags 2.11.0", "indexmap", "log", "serde", diff --git a/Cargo.toml b/Cargo.toml index aa0a5d3..5b93f82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" [dependencies] ollama-rs = { version = "0.3.4", features = ["macros", "headers"] } -tokio = { version = "1.50.0", features = ["rt", "rt-multi-thread", "macros"] } +tokio = { version = "1.50.0", features = ["rt", "rt-multi-thread", "macros", "io-std"] } rmcp = { version = "1.3.0", features = ["client"] } log = { version = "0.4.29" } env_logger = "0.11.10" @@ -18,4 +18,6 @@ dialoguer = "0.12.0" indicatif = "0.18.4" console = "0.16.3" clap = { version = "4.6.0", features = ["derive"] } -base64 = "0.22.1" \ No newline at end of file +base64 = "0.22.1" +cpal = "0.17.3" +rodio = "0.22.2" \ No newline at end of file diff --git a/src/config.rs b/src/config.rs index 3cf2c3d..4bbcc63 100644 --- a/src/config.rs +++ b/src/config.rs @@ -13,20 +13,15 @@ use own_assist_common::config_from_file; #[derive(Debug, Deserialize)] pub struct Config { + #[serde(default)] + pub interface: HumanInterface, permissions: Option>, ollama: OllamaConfig, - #[serde(rename = "audio-server")] - audio_server: Option, + audio: Option, #[serde(rename = "mcp-servers")] mcp_servers: Vec, } -#[derive(Debug, Deserialize)] -pub struct AudioServerConfig { - url: Url, - authorization: Option, -} - impl Config { pub fn from_file() -> Result { config_from_file("assist.toml") @@ -51,7 +46,8 @@ impl Config { } pub fn audio_client(&self) -> Option { - if let Some(audio_server) = &self.audio_server { + if let Some(audio_config) = &self.audio { + let audio_server = &audio_config.server; let mut audio_client = AudioClient::new(audio_server.url.clone()); if let Some(authorization) = &audio_server.authorization { audio_client = audio_client.with_authorization(authorization.clone()); @@ -96,6 +92,24 @@ impl Config { } } +#[derive(Debug, Deserialize, Default)] +pub enum HumanInterface { + #[default] + Cli, + Audio, +} + +#[derive(Debug, Deserialize)] +pub struct AudioConfig { + server: AudioServerConfig, +} + +#[derive(Debug, Deserialize)] +pub struct AudioServerConfig { + url: Url, + authorization: Option, +} + #[derive(Debug, Deserialize)] pub struct OllamaConfig { pub url: Option, diff --git a/src/human_interface/audio.rs b/src/human_interface/audio.rs index 07b0442..3eae9ea 100644 --- a/src/human_interface/audio.rs +++ b/src/human_interface/audio.rs @@ -1,43 +1,128 @@ +use crate::translate; use crate::human_interface::{HumanInterface, HumanInterfaceError}; +use cpal::traits::DeviceTrait; use own_mcp::audio::models::VoiceRequest; use own_mcp::audio::{AudioClient, AudioClientTrait}; use own_mcp::mcp::chat::PermissionAnswer; +use std::io; use std::path::Path; use tokio::fs::create_dir_all; +use tokio::io::AsyncBufReadExt; +use crate::tlt; const TEMPORARY_AUDIO_PATH: &str = "temporary_audio"; -struct Audio { +pub struct Audio { client: AudioClient, + device: cpal::Device, } impl Audio { - async fn new(client: AudioClient) -> Result { + pub async fn new( + client: AudioClient, + device: cpal::Device, + ) -> Result { create_dir_all(TEMPORARY_AUDIO_PATH) .await .map_err(HumanInterfaceError::IoError)?; - Ok(Self { client }) + Ok(Self { client, device }) } -} -impl HumanInterface for Audio { - async fn agent_message(&mut self, message: String) -> Result<(), HumanInterfaceError> { + async fn wait_for_input_line(expected_line: &str) -> Result<(), io::Error> { + loop { + let stdin_reader = tokio::io::BufReader::new(tokio::io::stdin()); + log::trace!("wating for '{expected_line}' to continue"); + if let Some(line) = stdin_reader.lines().next_line().await? { + log::debug!("Received line: {line:?}"); + if line == expected_line { + log::trace!("stopped waiting"); + return Ok(()); + } + } + } + } + + async fn wait_for_decision() -> Result { + const GRANTED_LINE: &str = "y"; + const DENIED_LINE: &str = "n"; + + loop { + let stdin_reader = tokio::io::BufReader::new(tokio::io::stdin()); + log::trace!("wating for '{GRANTED_LINE}' or '{DENIED_LINE}' to continue"); + if let Some(line) = stdin_reader.lines().next_line().await? { + log::debug!("Received line: {line:?}"); + + match line.as_str() { + GRANTED_LINE => { return Ok(PermissionAnswer::Granted) }, + DENIED_LINE => { return Ok(PermissionAnswer::Denied) }, + _ => {} + }; + } + } + } + + async fn play_text(&self, text: String) -> Result<(), HumanInterfaceError> { let bytes = self .client .tts(VoiceRequest { - text: message, + text, config: None, }) .await .map_err(|e| HumanInterfaceError::Other(format!("Failed getting tts: {e}")))?; - tokio::fs::write(Path::new(TEMPORARY_AUDIO_PATH).join("tts.wav"), &bytes) - .await - .map_err(|e| HumanInterfaceError::IoError(e)) + let wav_file_path = Path::new(TEMPORARY_AUDIO_PATH).join("tts.wav"); + + tokio::fs::write(&wav_file_path, &bytes).await?; + + let handle = rodio::DeviceSinkBuilder::open_default_sink().expect("open default audio stream"); + let file = std::fs::File::open(wav_file_path)?; + let player = rodio::play(handle.mixer(), file).map_err(|e| HumanInterfaceError::Other(format!("Failed playing audio: {e}")))?; + + player.sleep_until_end(); + + Ok(()) + } +} + +impl HumanInterface for Audio { + async fn agent_message(&mut self, message: String) -> Result<(), HumanInterfaceError> { + self.play_text(message).await?; + + Ok(()) } async fn expect_user_message(&mut self) -> Result { - todo!() + use own_mcp::audio::recording::{samples_to_wav, start, stop_and_take_data}; + + const RECORDING_START: &str = "r"; + const RECORDING_STOP: &str = "s"; + + let input_configs = self.device.default_input_config().map_err(|e| { + HumanInterfaceError::Other(format!("Could not get supported input configs: {e}")) + })?; + + Self::wait_for_input_line(RECORDING_START).await?; + let recording_handler = start(self.device.clone(), input_configs); + Self::wait_for_input_line(RECORDING_STOP).await?; + + let (samples, wav_spec) = stop_and_take_data(recording_handler) + .map_err(|e| HumanInterfaceError::Other(format!("failed recording: {e}")))?; + + let wav_file_path = Path::new(TEMPORARY_AUDIO_PATH).join("recording.wav"); + + samples_to_wav( + samples, + &wav_spec, + &wav_file_path, + ) + .map_err(|e| HumanInterfaceError::Other(format!("hound error: {e}")))?; + + let transcription = self.client.transcribe(wav_file_path).await.map_err(|e|HumanInterfaceError::Other(format!("failed to transcribe: {e}")))?; + + log::info!("transcription: {transcription:?}"); + + Ok(transcription.text) } async fn ask_for_permission( @@ -45,27 +130,42 @@ impl HumanInterface for Audio { mcp_server_name: String, tool_name: String, ) -> Result { - todo!() + self.play_text(tlt!("tool_allow_question_audio", mcp_server_name, tool_name)).await?; + + let decision = Self::wait_for_decision().await?; + log::debug!("decision: {decision:?}"); + Ok(decision) } } #[cfg(test)] mod tests { use super::*; + use cpal::traits::HostTrait; + + const AUTH_HEADER: &str = include_str!(".AUTH_HEADER"); + + async fn get_test_human_interface() -> Audio { + Audio::new(AudioClient::new( + url::Url::parse("https://audio.mboemer.de" + ).unwrap()). + with_authorization( + AUTH_HEADER.to_string()), cpal::Host::default().default_input_device().unwrap()).await.unwrap() + } #[tokio::test] async fn test_tts_fetching() { - let mut human_interface = Audio::new( - AudioClient::new( - url::Url::parse("https://audio.mboemer.de" - ).unwrap()). - with_authorization( - "Basic bWlsYW46a3lLRVIjOVJAaDlAXldBI3pVXkAhb0pRIXViczNNV0g3U1NzQDVUenppJVFmciY5WEpxJXB1YmN6YiEkdmY1Z2FQd0N2aEd3".to_string()) - ).await.unwrap(); + let mut human_interface = get_test_human_interface().await; human_interface .agent_message(String::from("Peter ist jetzt in deinem PC.")) .await .unwrap(); } + + #[tokio::test] + async fn test_tts_playing() { + let human_interface = get_test_human_interface().await; + human_interface.play_text(String::from("Peter ist jetzt in deinem PC.")).await.unwrap(); + } } diff --git a/src/human_interface/cli.rs b/src/human_interface/cli.rs index 52c9abe..3ce300c 100644 --- a/src/human_interface/cli.rs +++ b/src/human_interface/cli.rs @@ -55,7 +55,7 @@ impl HumanInterface for CommandLine { } let confirmation = Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt(format!("Allow usage of {mcp_server_name}:{tool_name}")) + .with_prompt(tlt!("tool_allow_question_cli", mcp_server_name, tool_name)) .interact() .map_err(|e| HumanInterfaceError::IoError(e.into()))?; diff --git a/src/human_interface/mod.rs b/src/human_interface/mod.rs index 1866683..aedd57f 100644 --- a/src/human_interface/mod.rs +++ b/src/human_interface/mod.rs @@ -7,7 +7,7 @@ use own_mcp::mcp::chat::PermissionAnswer; #[derive(Debug, Error)] pub enum HumanInterfaceError { #[error(transparent)] - IoError(std::io::Error), + IoError(#[from] std::io::Error), #[error("other error in human interface: {0}")] Other(String) } diff --git a/src/i18n/translations/de.json b/src/i18n/translations/de.json index 7df9fdf..f2faead 100644 --- a/src/i18n/translations/de.json +++ b/src/i18n/translations/de.json @@ -4,6 +4,9 @@ "you": "Du", "assistant": "Assistent", "username": "Nutzername", - "password": "Passwort" + "password": "Passwort", + "model_download_needed": "Das Modell '{...}' ist noch nicht installiert und muss zuerst heruntergeladen werden", + "tool_allow_question_cli": "Benutzung von {...}:{...} zulassen", + "tool_allow_question_audio": "Benutzung von {...} des Dienstes {...} zulassen?" } } \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 527e7ff..0d7fd39 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,6 +9,8 @@ use crate::i18n::translate; use crate::model::create_model_from_config; use base64::Engine; use clap::Parser; +use cpal::traits::HostTrait; +use own_mcp::AgentChat; use std::fmt::Display; /// partial mcp client with cli and voice interaction @@ -35,6 +37,33 @@ fn basic_auth_tool() { println!("Basic {}", encoded); } +async fn chat_loop(mut human_interface: impl HumanInterface, mut agent_chat: AgentChat) { + loop { + let user_message = human_interface.expect_user_message().await.unwrap(); + let immutable_interface = &human_interface; + let agent_message = agent_chat + .message(user_message, async |mcp_server_name, tool_name| { + immutable_interface + .ask_for_permission(mcp_server_name, tool_name) + .await + .inspect_err(exit_msg!( + "Human interface error while asking for permission" + )) + .unwrap() + }) + .await + .inspect_err(exit_msg!("Failed communicating with agent.")) + .unwrap(); + human_interface + .agent_message(agent_message.content) + .await + .inspect_err(exit_msg!( + "Human interface error while trying to display message" + )) + .unwrap(); + } +} + #[tokio::main] async fn main() { let args = Args::parse(); @@ -64,7 +93,7 @@ async fn main() { create_model_from_config(&ollama, &config.ollama_config().model) .await .inspect_err(exit_msg!(format!( - "failed creating ollama model {model_name}" + "failed creating ollama model `{model_name}`" ))) .unwrap(); @@ -85,37 +114,36 @@ async fn main() { agent_chat.get_all_tools().collect::>() ); - let mut human_interface = human_interface::cli::CommandLine::new(); - - loop { - let user_message = human_interface.expect_user_message().await.unwrap(); - let immutable_interface = &human_interface; - let agent_message = agent_chat - .message(user_message, async |mcp_server_name, tool_name| { - immutable_interface - .ask_for_permission(mcp_server_name, tool_name) + match config.interface { + config::HumanInterface::Cli => { + let human_interface = human_interface::cli::CommandLine::new(); + chat_loop(human_interface, agent_chat).await; + } + config::HumanInterface::Audio => { + match config.audio_client() { + Some(audio_client) => { + let human_interface = human_interface::audio::Audio::new( + audio_client, + cpal::Host::default() + .default_input_device() + .expect("no default device"), + ) .await - .inspect_err(exit_msg!( - "Human interface error while asking for permission" - )) - .unwrap() - }) - .await - .inspect_err(exit_msg!("Failed communicating with agent.")) - .unwrap(); - human_interface - .agent_message(agent_message.content) - .await - .inspect_err(exit_msg!( - "Human interface error while trying to display message" - )) - .unwrap(); - } + .inspect_err(exit_msg!("failed creating audio client")).unwrap(); + + chat_loop(human_interface, agent_chat).await; + } + None => { + eprintln!("audio client was not configured"); + } + }; + } + }; } fn exit_with_error_message(error: impl Display, message: impl Display) { log::error!("{message}: {error}"); - println!("{}", message); // makes sure that message is printed even if logging is deactivated + eprintln!("{}", message); // makes sure that message is printed even if logging is deactivated std::process::exit(1); } diff --git a/src/model.rs b/src/model.rs index 9f69b59..06ef259 100644 --- a/src/model.rs +++ b/src/model.rs @@ -23,10 +23,7 @@ pub async fn create_model_from_config( if let Ok(models) = ollama.list_local_models().await && !models.iter().any(|m| m.name == config.from_model) { - let message = format!( - "Model `{}` is not installed and will be downloaded first.", - config.from_model - ); + let message = tlt!("model_download_needed", config.from_model.clone()); log::info!("{}", message); println!("{}", console::style(message).yellow().bold()); }