add working Audio HumanInterface

refactoring as always
remove secrets

.gitignore

@@ -3,3 +3,4 @@
 assist.toml
 mcp_server_collection/server.toml
 temporary_audio
+src/human_interface/.AUTH_HEADER

Cargo.lock

@@ -18,7 +18,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "812947049edcd670a82cd5c73c3661d2e58468577ba8489de58e1a73c04cbd5d"
 dependencies = [
  "alsa-sys",
- "bitflags",
+ "bitflags 2.11.0",
  "cfg-if",
  "libc",
 ]
@@ -98,6 +98,12 @@ version = "1.0.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
 
+[[package]]
+name = "arrayvec"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
+
 [[package]]
 name = "async-stream"
 version = "0.3.6"
@@ -171,6 +177,12 @@ version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
 [[package]]
 name = "bitflags"
 version = "2.11.0"
@@ -201,6 +213,12 @@ version = "3.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
 
+[[package]]
+name = "bytemuck"
+version = "1.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
+
 [[package]]
 name = "bytes"
 version = "1.11.1"
@@ -371,7 +389,7 @@ version = "0.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "16dd574a72a021b90c7656c474ea31d11a2f0366a8eff574186e761e0b9e3586"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "libc",
  "objc2-audio-toolbox",
  "objc2-core-audio",
@@ -505,7 +523,7 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "objc2",
 ]
 
@@ -586,6 +604,12 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
+[[package]]
+name = "extended"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365"
+
 [[package]]
 name = "fastrand"
 version = "2.4.0"
@@ -1224,6 +1248,12 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "lazy_static"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+
 [[package]]
 name = "leb128fmt"
 version = "0.1.0"
@@ -1236,6 +1266,12 @@ version = "0.2.184"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af"
 
+[[package]]
+name = "libm"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.12.1"
@@ -1325,7 +1361,7 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "jni-sys 0.3.1",
  "log",
  "ndk-sys",
@@ -1354,12 +1390,22 @@ version = "0.31.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "cfg-if",
  "cfg_aliases",
  "libc",
 ]
 
+[[package]]
+name = "num-bigint"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
+dependencies = [
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-derive"
 version = "0.4.2"
@@ -1371,6 +1417,26 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "num-integer"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-rational"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
+dependencies = [
+ "num-bigint",
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.19"
@@ -1378,6 +1444,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
 dependencies = [
  "autocfg",
+ "libm",
 ]
 
 [[package]]
@@ -1436,7 +1503,7 @@ version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "libc",
  "objc2",
  "objc2-core-audio",
@@ -1474,7 +1541,7 @@ version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "objc2",
 ]
 
@@ -1484,7 +1551,7 @@ version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "block2",
  "dispatch2",
  "libc",
@@ -1503,7 +1570,7 @@ version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "block2",
  "libc",
  "objc2",
@@ -1558,7 +1625,7 @@ version = "0.10.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "cfg-if",
  "foreign-types",
  "libc",
@@ -1603,6 +1670,7 @@ dependencies = [
  "base64",
  "clap",
  "console",
+ "cpal",
  "dialoguer",
  "env_logger",
  "indicatif",
@@ -1611,6 +1679,7 @@ dependencies = [
  "own_assist_common",
  "own_mcp",
  "rmcp",
+ "rodio",
  "serde",
  "thiserror 2.0.18",
  "tokio",
@@ -1897,6 +1966,16 @@ version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba"
 
+[[package]]
+name = "rand_distr"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4d431c2703ccf129de4d45253c03f49ebb22b97d6ad79ee3ecfc7e3f4862c1d8"
+dependencies = [
+ "num-traits",
+ "rand 0.10.0",
+]
+
 [[package]]
 name = "ref-cast"
 version = "1.0.25"
@@ -2083,6 +2162,28 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "rodio"
+version = "0.22.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0a536bb79db59098ef71a4dd4246c02eb87b316deceb1b68e0cde7167ec01eb"
+dependencies = [
+ "cpal",
+ "dasp_sample",
+ "num-rational",
+ "rand 0.10.0",
+ "rand_distr",
+ "rtrb",
+ "symphonia",
+ "thiserror 2.0.18",
+]
+
+[[package]]
+name = "rtrb"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7204ed6420f698836b76d4d5c2ec5dec7585fd5c3a788fd1cde855d1de598239"
+
 [[package]]
 name = "rustc-hash"
 version = "2.1.2"
@@ -2095,7 +2196,7 @@ version = "1.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "errno",
  "libc",
  "linux-raw-sys",
@@ -2239,7 +2340,7 @@ version = "3.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "core-foundation 0.10.1",
  "core-foundation-sys",
  "libc",
@@ -2441,6 +2542,153 @@ version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
+[[package]]
+name = "symphonia"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039"
+dependencies = [
+ "lazy_static",
+ "symphonia-bundle-flac",
+ "symphonia-bundle-mp3",
+ "symphonia-codec-aac",
+ "symphonia-codec-pcm",
+ "symphonia-codec-vorbis",
+ "symphonia-core",
+ "symphonia-format-isomp4",
+ "symphonia-format-ogg",
+ "symphonia-format-riff",
+ "symphonia-metadata",
+]
+
+[[package]]
+name = "symphonia-bundle-flac"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c91565e180aea25d9b80a910c546802526ffd0072d0b8974e3ebe59b686c9976"
+dependencies = [
+ "log",
+ "symphonia-core",
+ "symphonia-metadata",
+ "symphonia-utils-xiph",
+]
+
+[[package]]
+name = "symphonia-bundle-mp3"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4872dd6bb56bf5eac799e3e957aa1981086c3e613b27e0ac23b176054f7c57ed"
+dependencies = [
+ "lazy_static",
+ "log",
+ "symphonia-core",
+ "symphonia-metadata",
+]
+
+[[package]]
+name = "symphonia-codec-aac"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c263845aa86881416849c1729a54c7f55164f8b96111dba59de46849e73a790"
+dependencies = [
+ "lazy_static",
+ "log",
+ "symphonia-core",
+]
+
+[[package]]
+name = "symphonia-codec-pcm"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e89d716c01541ad3ebe7c91ce4c8d38a7cf266a3f7b2f090b108fb0cb031d95"
+dependencies = [
+ "log",
+ "symphonia-core",
+]
+
+[[package]]
+name = "symphonia-codec-vorbis"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f025837c309cd69ffef572750b4a2257b59552c5399a5e49707cc5b1b85d1c73"
+dependencies = [
+ "log",
+ "symphonia-core",
+ "symphonia-utils-xiph",
+]
+
+[[package]]
+name = "symphonia-core"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea00cc4f79b7f6bb7ff87eddc065a1066f3a43fe1875979056672c9ef948c2af"
+dependencies = [
+ "arrayvec",
+ "bitflags 1.3.2",
+ "bytemuck",
+ "lazy_static",
+ "log",
+]
+
+[[package]]
+name = "symphonia-format-isomp4"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "243739585d11f81daf8dac8d9f3d18cc7898f6c09a259675fc364b382c30e0a5"
+dependencies = [
+ "encoding_rs",
+ "log",
+ "symphonia-core",
+ "symphonia-metadata",
+ "symphonia-utils-xiph",
+]
+
+[[package]]
+name = "symphonia-format-ogg"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b4955c67c1ed3aa8ae8428d04ca8397fbef6a19b2b051e73b5da8b1435639cb"
+dependencies = [
+ "log",
+ "symphonia-core",
+ "symphonia-metadata",
+ "symphonia-utils-xiph",
+]
+
+[[package]]
+name = "symphonia-format-riff"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2d7c3df0e7d94efb68401d81906eae73c02b40d5ec1a141962c592d0f11a96f"
+dependencies = [
+ "extended",
+ "log",
+ "symphonia-core",
+ "symphonia-metadata",
+]
+
+[[package]]
+name = "symphonia-metadata"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36306ff42b9ffe6e5afc99d49e121e0bd62fe79b9db7b9681d48e29fa19e6b16"
+dependencies = [
+ "encoding_rs",
+ "lazy_static",
+ "log",
+ "symphonia-core",
+]
+
+[[package]]
+name = "symphonia-utils-xiph"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee27c85ab799a338446b68eec77abf42e1a6f1bb490656e121c6e27bfbab9f16"
+dependencies = [
+ "symphonia-core",
+ "symphonia-metadata",
+]
+
 [[package]]
 name = "syn"
 version = "2.0.117"
@@ -2478,7 +2726,7 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "core-foundation 0.9.4",
  "system-configuration-sys",
 ]
@@ -2714,7 +2962,7 @@ version = "0.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "bytes",
  "futures-util",
  "http",
@@ -2993,7 +3241,7 @@ version = "0.244.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "hashbrown 0.15.5",
  "indexmap",
  "semver",
@@ -3447,7 +3695,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
 dependencies = [
  "anyhow",
- "bitflags",
+ "bitflags 2.11.0",
  "indexmap",
  "log",
  "serde",

Cargo.toml

@@ -5,7 +5,7 @@ edition = "2024"
 
 [dependencies]
 ollama-rs = { version = "0.3.4", features = ["macros", "headers"] }
-tokio = { version = "1.50.0", features = ["rt", "rt-multi-thread", "macros"] }
+tokio = { version = "1.50.0", features = ["rt", "rt-multi-thread", "macros", "io-std"] }
 rmcp = { version = "1.3.0", features = ["client"] }
 log = { version = "0.4.29" }
 env_logger = "0.11.10"
@@ -19,3 +19,5 @@ indicatif = "0.18.4"
 console = "0.16.3"
 clap = { version = "4.6.0", features = ["derive"] }
 base64 = "0.22.1"
+cpal = "0.17.3"
+rodio = "0.22.2"

src/config.rs

@@ -13,20 +13,15 @@ use own_assist_common::config_from_file;
 
 #[derive(Debug, Deserialize)]
 pub struct Config {
+    #[serde(default)]
+    pub interface: HumanInterface,
     permissions: Option<Vec<PermissionConfig>>,
     ollama: OllamaConfig,
-    #[serde(rename = "audio-server")]
+    audio: Option<AudioConfig>,
-    audio_server: Option<AudioServerConfig>,
     #[serde(rename = "mcp-servers")]
     mcp_servers: Vec<MCPServerConfig>,
 }
 
-#[derive(Debug, Deserialize)]
-pub struct AudioServerConfig {
-    url: Url,
-    authorization: Option<String>,
-}
-
 impl Config {
     pub fn from_file() -> Result<Self, ConfigLoadingError> {
         config_from_file("assist.toml")
@@ -51,7 +46,8 @@ impl Config {
     }
 
     pub fn audio_client(&self) -> Option<AudioClient> {
-        if let Some(audio_server) = &self.audio_server {
+        if let Some(audio_config) = &self.audio {
+            let audio_server = &audio_config.server;
             let mut audio_client = AudioClient::new(audio_server.url.clone());
             if let Some(authorization) = &audio_server.authorization {
                 audio_client = audio_client.with_authorization(authorization.clone());
@@ -96,6 +92,24 @@ impl Config {
     }
 }
 
+#[derive(Debug, Deserialize, Default)]
+pub enum HumanInterface {
+    #[default]
+    Cli,
+    Audio,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct AudioConfig {
+    server: AudioServerConfig,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct AudioServerConfig {
+    url: Url,
+    authorization: Option<String>,
+}
+
 #[derive(Debug, Deserialize)]
 pub struct OllamaConfig {
     pub url: Option<Url>,

src/human_interface/audio.rs

@@ -1,43 +1,128 @@
+use crate::translate;
 use crate::human_interface::{HumanInterface, HumanInterfaceError};
+use cpal::traits::DeviceTrait;
 use own_mcp::audio::models::VoiceRequest;
 use own_mcp::audio::{AudioClient, AudioClientTrait};
 use own_mcp::mcp::chat::PermissionAnswer;
+use std::io;
 use std::path::Path;
 use tokio::fs::create_dir_all;
+use tokio::io::AsyncBufReadExt;
+use crate::tlt;
 
 const TEMPORARY_AUDIO_PATH: &str = "temporary_audio";
 
-struct Audio {
+pub struct Audio {
     client: AudioClient,
+    device: cpal::Device,
 }
 
 impl Audio {
-    async fn new(client: AudioClient) -> Result<Self, HumanInterfaceError> {
+    pub async fn new(
+        client: AudioClient,
+        device: cpal::Device,
+    ) -> Result<Self, HumanInterfaceError> {
         create_dir_all(TEMPORARY_AUDIO_PATH)
             .await
             .map_err(HumanInterfaceError::IoError)?;
-        Ok(Self { client })
+        Ok(Self { client, device })
     }
-}
 
-impl HumanInterface for Audio {
+    async fn wait_for_input_line(expected_line: &str) -> Result<(), io::Error> {
-    async fn agent_message(&mut self, message: String) -> Result<(), HumanInterfaceError> {
+        loop {
+            let stdin_reader = tokio::io::BufReader::new(tokio::io::stdin());
+            log::trace!("wating for '{expected_line}' to continue");
+            if let Some(line) = stdin_reader.lines().next_line().await? {
+                log::debug!("Received line: {line:?}");
+                if line == expected_line {
+                    log::trace!("stopped waiting");
+                    return Ok(());
+                }
+            }
+        }
+    }
+
+    async fn wait_for_decision() -> Result<PermissionAnswer, io::Error> {
+        const GRANTED_LINE: &str = "y";
+        const DENIED_LINE: &str = "n";
+
+        loop {
+            let stdin_reader = tokio::io::BufReader::new(tokio::io::stdin());
+            log::trace!("wating for '{GRANTED_LINE}' or '{DENIED_LINE}' to continue");
+            if let Some(line) = stdin_reader.lines().next_line().await? {
+                log::debug!("Received line: {line:?}");
+
+                match line.as_str() {
+                    GRANTED_LINE => { return Ok(PermissionAnswer::Granted) },
+                    DENIED_LINE => { return Ok(PermissionAnswer::Denied) },
+                    _ => {}
+                };
+            }
+        }
+    }
+
+    async fn play_text(&self, text: String) -> Result<(), HumanInterfaceError> {
         let bytes = self
             .client
             .tts(VoiceRequest {
-                text: message,
+                text,
                 config: None,
             })
             .await
             .map_err(|e| HumanInterfaceError::Other(format!("Failed getting tts: {e}")))?;
 
-        tokio::fs::write(Path::new(TEMPORARY_AUDIO_PATH).join("tts.wav"), &bytes)
+        let wav_file_path = Path::new(TEMPORARY_AUDIO_PATH).join("tts.wav");
-            .await
+
-            .map_err(|e| HumanInterfaceError::IoError(e))
+        tokio::fs::write(&wav_file_path, &bytes).await?;
+
+        let handle = rodio::DeviceSinkBuilder::open_default_sink().expect("open default audio stream");
+        let file = std::fs::File::open(wav_file_path)?;
+        let player = rodio::play(handle.mixer(), file).map_err(|e| HumanInterfaceError::Other(format!("Failed playing audio: {e}")))?;
+
+        player.sleep_until_end();
+
+        Ok(())
+    }
+}
+
+impl HumanInterface for Audio {
+    async fn agent_message(&mut self, message: String) -> Result<(), HumanInterfaceError> {
+        self.play_text(message).await?;
+
+        Ok(())
     }
 
     async fn expect_user_message(&mut self) -> Result<String, HumanInterfaceError> {
-        todo!()
+        use own_mcp::audio::recording::{samples_to_wav, start, stop_and_take_data};
+
+        const RECORDING_START: &str = "r";
+        const RECORDING_STOP: &str = "s";
+
+        let input_configs = self.device.default_input_config().map_err(|e| {
+            HumanInterfaceError::Other(format!("Could not get supported input configs: {e}"))
+        })?;
+
+        Self::wait_for_input_line(RECORDING_START).await?;
+        let recording_handler = start(self.device.clone(), input_configs);
+        Self::wait_for_input_line(RECORDING_STOP).await?;
+
+        let (samples, wav_spec) = stop_and_take_data(recording_handler)
+            .map_err(|e| HumanInterfaceError::Other(format!("failed recording: {e}")))?;
+
+        let wav_file_path = Path::new(TEMPORARY_AUDIO_PATH).join("recording.wav");
+
+        samples_to_wav(
+            samples,
+            &wav_spec,
+            &wav_file_path,
+        )
+        .map_err(|e| HumanInterfaceError::Other(format!("hound error: {e}")))?;
+
+        let transcription = self.client.transcribe(wav_file_path).await.map_err(|e|HumanInterfaceError::Other(format!("failed to transcribe: {e}")))?;
+
+        log::info!("transcription: {transcription:?}");
+
+        Ok(transcription.text)
     }
 
     async fn ask_for_permission(
@@ -45,27 +130,42 @@ impl HumanInterface for Audio {
         mcp_server_name: String,
         tool_name: String,
     ) -> Result<PermissionAnswer, HumanInterfaceError> {
-        todo!()
+        self.play_text(tlt!("tool_allow_question_audio", mcp_server_name, tool_name)).await?;
+
+        let decision = Self::wait_for_decision().await?;
+        log::debug!("decision: {decision:?}");
+        Ok(decision)
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use cpal::traits::HostTrait;
 
-    #[tokio::test]
+    const AUTH_HEADER: &str = include_str!(".AUTH_HEADER");
-    async fn test_tts_fetching() {
+
-        let mut human_interface = Audio::new(
+    async fn get_test_human_interface() -> Audio {
-            AudioClient::new(
+        Audio::new(AudioClient::new(
             url::Url::parse("https://audio.mboemer.de"
             ).unwrap()).
             with_authorization(
-                    "Basic bWlsYW46a3lLRVIjOVJAaDlAXldBI3pVXkAhb0pRIXViczNNV0g3U1NzQDVUenppJVFmciY5WEpxJXB1YmN6YiEkdmY1Z2FQd0N2aEd3".to_string())
+                AUTH_HEADER.to_string()), cpal::Host::default().default_input_device().unwrap()).await.unwrap()
-        ).await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_tts_fetching() {
+        let mut human_interface = get_test_human_interface().await;
 
         human_interface
             .agent_message(String::from("Peter ist jetzt in deinem PC."))
             .await
             .unwrap();
     }
+
+    #[tokio::test]
+    async fn test_tts_playing() {
+        let human_interface = get_test_human_interface().await;
+        human_interface.play_text(String::from("Peter ist jetzt in deinem PC.")).await.unwrap();
+    }
 }

src/human_interface/cli.rs

@@ -55,7 +55,7 @@ impl HumanInterface for CommandLine {
         }
 
         let confirmation = Confirm::with_theme(&ColorfulTheme::default())
-            .with_prompt(format!("Allow usage of {mcp_server_name}:{tool_name}"))
+            .with_prompt(tlt!("tool_allow_question_cli", mcp_server_name, tool_name))
             .interact()
             .map_err(|e| HumanInterfaceError::IoError(e.into()))?;
 

src/human_interface/mod.rs

@@ -7,7 +7,7 @@ use own_mcp::mcp::chat::PermissionAnswer;
 #[derive(Debug, Error)]
 pub enum HumanInterfaceError {
     #[error(transparent)]
-    IoError(std::io::Error),
+    IoError(#[from] std::io::Error),
     #[error("other error in human interface: {0}")]
     Other(String)
 }

src/i18n/translations/de.json

@@ -4,6 +4,9 @@
     "you": "Du",
     "assistant": "Assistent",
     "username": "Nutzername",
-    "password": "Passwort"
+    "password": "Passwort",
+    "model_download_needed": "Das Modell '{...}' ist noch nicht installiert und muss zuerst heruntergeladen werden",
+    "tool_allow_question_cli": "Benutzung von {...}:{...} zulassen",
+    "tool_allow_question_audio": "Benutzung von {...} des Dienstes {...} zulassen?"
   }
 }

src/main.rs

@@ -9,6 +9,8 @@ use crate::i18n::translate;
 use crate::model::create_model_from_config;
 use base64::Engine;
 use clap::Parser;
+use cpal::traits::HostTrait;
+use own_mcp::AgentChat;
 use std::fmt::Display;
 
 /// partial mcp client with cli and voice interaction
@@ -35,6 +37,33 @@ fn basic_auth_tool() {
     println!("Basic {}", encoded);
 }
 
+async fn chat_loop(mut human_interface: impl HumanInterface, mut agent_chat: AgentChat) {
+    loop {
+        let user_message = human_interface.expect_user_message().await.unwrap();
+        let immutable_interface = &human_interface;
+        let agent_message = agent_chat
+            .message(user_message, async |mcp_server_name, tool_name| {
+                immutable_interface
+                    .ask_for_permission(mcp_server_name, tool_name)
+                    .await
+                    .inspect_err(exit_msg!(
+                        "Human interface error while asking for permission"
+                    ))
+                    .unwrap()
+            })
+            .await
+            .inspect_err(exit_msg!("Failed communicating with agent."))
+            .unwrap();
+        human_interface
+            .agent_message(agent_message.content)
+            .await
+            .inspect_err(exit_msg!(
+                "Human interface error while trying to display message"
+            ))
+            .unwrap();
+    }
+}
+
 #[tokio::main]
 async fn main() {
     let args = Args::parse();
@@ -64,7 +93,7 @@ async fn main() {
     create_model_from_config(&ollama, &config.ollama_config().model)
         .await
         .inspect_err(exit_msg!(format!(
-            "failed creating ollama model {model_name}"
+            "failed creating ollama model `{model_name}`"
         )))
         .unwrap();
 
@@ -85,37 +114,36 @@ async fn main() {
         agent_chat.get_all_tools().collect::<Vec<_>>()
     );
 
-    let mut human_interface = human_interface::cli::CommandLine::new();
+    match config.interface {
-
+        config::HumanInterface::Cli => {
-    loop {
+            let human_interface = human_interface::cli::CommandLine::new();
-        let user_message = human_interface.expect_user_message().await.unwrap();
+            chat_loop(human_interface, agent_chat).await;
-        let immutable_interface = &human_interface;
-        let agent_message = agent_chat
-            .message(user_message, async |mcp_server_name, tool_name| {
-                immutable_interface
-                    .ask_for_permission(mcp_server_name, tool_name)
-                    .await
-                    .inspect_err(exit_msg!(
-                        "Human interface error while asking for permission"
-                    ))
-                    .unwrap()
-            })
-            .await
-            .inspect_err(exit_msg!("Failed communicating with agent."))
-            .unwrap();
-        human_interface
-            .agent_message(agent_message.content)
-            .await
-            .inspect_err(exit_msg!(
-                "Human interface error while trying to display message"
-            ))
-            .unwrap();
         }
+        config::HumanInterface::Audio => {
+            match config.audio_client() {
+                Some(audio_client) => {
+                    let human_interface = human_interface::audio::Audio::new(
+                        audio_client,
+                        cpal::Host::default()
+                            .default_input_device()
+                            .expect("no default device"),
+                    )
+                    .await
+                    .inspect_err(exit_msg!("failed creating audio client")).unwrap();
+
+                    chat_loop(human_interface, agent_chat).await;
+                }
+                None => {
+                    eprintln!("audio client was not configured");
+                }
+            };
+        }
+    };
 }
 
 fn exit_with_error_message(error: impl Display, message: impl Display) {
     log::error!("{message}: {error}");
-    println!("{}", message); // makes sure that message is printed even if logging is deactivated
+    eprintln!("{}", message); // makes sure that message is printed even if logging is deactivated
     std::process::exit(1);
 }
 

src/model.rs

@@ -23,10 +23,7 @@ pub async fn create_model_from_config(
     if let Ok(models) = ollama.list_local_models().await
         && !models.iter().any(|m| m.name == config.from_model)
     {
-        let message = format!(
+        let message = tlt!("model_download_needed", config.from_model.clone());
-            "Model `{}` is not installed and will be downloaded first.",
-            config.from_model
-        );
         log::info!("{}", message);
         println!("{}", console::style(message).yellow().bold());
     }