milan/own_assist
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add working Audio HumanInterface

Browse Source 
refactoring as always
remove secrets
This commit is contained in:
milan
2026-04-21 21:55:17 +02:00
parent ef468e8fd6
commit 20a5b01311
10 changed files with 474 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2 -1
View File
@@ -2,4 +2,5 @@
*/target
assist.toml
mcp_server_collection/server.toml
temporary_audio
temporary_audio
src/human_interface/.AUTH_HEADER
Cargo.lock
Generated
+264 -16
View File
@@ -18,7 +18,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "812947049edcd670a82cd5c73c3661d2e58468577ba8489de58e1a73c04cbd5d"
dependencies = [
"alsa-sys",
"bitflags",
"bitflags 2.11.0",
"cfg-if",
"libc",
]
@@ -98,6 +98,12 @@ version = "1.0.102"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
[[package]]
name = "arrayvec"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "async-stream"
version = "0.3.6"
@@ -171,6 +177,12 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.11.0"
@@ -201,6 +213,12 @@ version = "3.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
[[package]]
name = "bytemuck"
version = "1.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
[[package]]
name = "bytes"
version = "1.11.1"
@@ -371,7 +389,7 @@ version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16dd574a72a021b90c7656c474ea31d11a2f0366a8eff574186e761e0b9e3586"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"libc",
"objc2-audio-toolbox",
"objc2-core-audio",
@@ -505,7 +523,7 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"objc2",
]
@@ -586,6 +604,12 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "extended"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365"
[[package]]
name = "fastrand"
version = "2.4.0"
@@ -1224,6 +1248,12 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "lazy_static"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]]
name = "leb128fmt"
version = "0.1.0"
@@ -1236,6 +1266,12 @@ version = "0.2.184"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af"
[[package]]
name = "libm"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "linux-raw-sys"
version = "0.12.1"
@@ -1325,7 +1361,7 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"jni-sys 0.3.1",
"log",
"ndk-sys",
@@ -1354,12 +1390,22 @@ version = "0.31.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"cfg-if",
"cfg_aliases",
"libc",
]
[[package]]
name = "num-bigint"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
dependencies = [
"num-integer",
"num-traits",
]
[[package]]
name = "num-derive"
version = "0.4.2"
@@ -1371,6 +1417,26 @@ dependencies = [
"syn",
]
[[package]]
name = "num-integer"
version = "0.1.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
dependencies = [
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
dependencies = [
"num-bigint",
"num-integer",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.19"
@@ -1378,6 +1444,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg",
"libm",
]
[[package]]
@@ -1436,7 +1503,7 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"libc",
"objc2",
"objc2-core-audio",
@@ -1474,7 +1541,7 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"objc2",
]
@@ -1484,7 +1551,7 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"block2",
"dispatch2",
"libc",
@@ -1503,7 +1570,7 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"block2",
"libc",
"objc2",
@@ -1558,7 +1625,7 @@ version = "0.10.76"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"cfg-if",
"foreign-types",
"libc",
@@ -1603,6 +1670,7 @@ dependencies = [
"base64",
"clap",
"console",
"cpal",
"dialoguer",
"env_logger",
"indicatif",
@@ -1611,6 +1679,7 @@ dependencies = [
"own_assist_common",
"own_mcp",
"rmcp",
"rodio",
"serde",
"thiserror 2.0.18",
"tokio",
@@ -1897,6 +1966,16 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba"
[[package]]
name = "rand_distr"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d431c2703ccf129de4d45253c03f49ebb22b97d6ad79ee3ecfc7e3f4862c1d8"
dependencies = [
"num-traits",
"rand 0.10.0",
]
[[package]]
name = "ref-cast"
version = "1.0.25"
@@ -2083,6 +2162,28 @@ dependencies = [
"syn",
]
[[package]]
name = "rodio"
version = "0.22.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a536bb79db59098ef71a4dd4246c02eb87b316deceb1b68e0cde7167ec01eb"
dependencies = [
"cpal",
"dasp_sample",
"num-rational",
"rand 0.10.0",
"rand_distr",
"rtrb",
"symphonia",
"thiserror 2.0.18",
]
[[package]]
name = "rtrb"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7204ed6420f698836b76d4d5c2ec5dec7585fd5c3a788fd1cde855d1de598239"
[[package]]
name = "rustc-hash"
version = "2.1.2"
@@ -2095,7 +2196,7 @@ version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"errno",
"libc",
"linux-raw-sys",
@@ -2239,7 +2340,7 @@ version = "3.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"core-foundation 0.10.1",
"core-foundation-sys",
"libc",
@@ -2441,6 +2542,153 @@ version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "symphonia"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039"
dependencies = [
"lazy_static",
"symphonia-bundle-flac",
"symphonia-bundle-mp3",
"symphonia-codec-aac",
"symphonia-codec-pcm",
"symphonia-codec-vorbis",
"symphonia-core",
"symphonia-format-isomp4",
"symphonia-format-ogg",
"symphonia-format-riff",
"symphonia-metadata",
]
[[package]]
name = "symphonia-bundle-flac"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c91565e180aea25d9b80a910c546802526ffd0072d0b8974e3ebe59b686c9976"
dependencies = [
"log",
"symphonia-core",
"symphonia-metadata",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-bundle-mp3"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4872dd6bb56bf5eac799e3e957aa1981086c3e613b27e0ac23b176054f7c57ed"
dependencies = [
"lazy_static",
"log",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-codec-aac"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c263845aa86881416849c1729a54c7f55164f8b96111dba59de46849e73a790"
dependencies = [
"lazy_static",
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-pcm"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e89d716c01541ad3ebe7c91ce4c8d38a7cf266a3f7b2f090b108fb0cb031d95"
dependencies = [
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-vorbis"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f025837c309cd69ffef572750b4a2257b59552c5399a5e49707cc5b1b85d1c73"
dependencies = [
"log",
"symphonia-core",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-core"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea00cc4f79b7f6bb7ff87eddc065a1066f3a43fe1875979056672c9ef948c2af"
dependencies = [
"arrayvec",
"bitflags 1.3.2",
"bytemuck",
"lazy_static",
"log",
]
[[package]]
name = "symphonia-format-isomp4"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "243739585d11f81daf8dac8d9f3d18cc7898f6c09a259675fc364b382c30e0a5"
dependencies = [
"encoding_rs",
"log",
"symphonia-core",
"symphonia-metadata",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-format-ogg"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b4955c67c1ed3aa8ae8428d04ca8397fbef6a19b2b051e73b5da8b1435639cb"
dependencies = [
"log",
"symphonia-core",
"symphonia-metadata",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-format-riff"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2d7c3df0e7d94efb68401d81906eae73c02b40d5ec1a141962c592d0f11a96f"
dependencies = [
"extended",
"log",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-metadata"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36306ff42b9ffe6e5afc99d49e121e0bd62fe79b9db7b9681d48e29fa19e6b16"
dependencies = [
"encoding_rs",
"lazy_static",
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-utils-xiph"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee27c85ab799a338446b68eec77abf42e1a6f1bb490656e121c6e27bfbab9f16"
dependencies = [
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "syn"
version = "2.0.117"
@@ -2478,7 +2726,7 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"core-foundation 0.9.4",
"system-configuration-sys",
]
@@ -2714,7 +2962,7 @@ version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"bytes",
"futures-util",
"http",
@@ -2993,7 +3241,7 @@ version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
dependencies = [
"bitflags",
"bitflags 2.11.0",
"hashbrown 0.15.5",
"indexmap",
"semver",
@@ -3447,7 +3695,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
dependencies = [
"anyhow",
"bitflags",
"bitflags 2.11.0",
"indexmap",
"log",
"serde",
Cargo.toml
+4 -2
View File
@@ -5,7 +5,7 @@ edition = "2024"
[dependencies]
ollama-rs = { version = "0.3.4", features = ["macros", "headers"] }
tokio = { version = "1.50.0", features = ["rt", "rt-multi-thread", "macros"] }
tokio = { version = "1.50.0", features = ["rt", "rt-multi-thread", "macros", "io-std"] }
rmcp = { version = "1.3.0", features = ["client"] }
log = { version = "0.4.29" }
env_logger = "0.11.10"
@@ -18,4 +18,6 @@ dialoguer = "0.12.0"
indicatif = "0.18.4"
console = "0.16.3"
clap = { version = "4.6.0", features = ["derive"] }
base64 = "0.22.1"
base64 = "0.22.1"
cpal = "0.17.3"
rodio = "0.22.2"
src/config.rs
+23 -9
View File
@@ -13,20 +13,15 @@ use own_assist_common::config_from_file;
#[derive(Debug, Deserialize)]
pub struct Config {
#[serde(default)]
pub interface: HumanInterface,
permissions: Option<Vec<PermissionConfig>>,
ollama: OllamaConfig,
#[serde(rename = "audio-server")]
audio_server: Option<AudioServerConfig>,
audio: Option<AudioConfig>,
#[serde(rename = "mcp-servers")]
mcp_servers: Vec<MCPServerConfig>,
}
#[derive(Debug, Deserialize)]
pub struct AudioServerConfig {
url: Url,
authorization: Option<String>,
}
impl Config {
pub fn from_file() -> Result<Self, ConfigLoadingError> {
config_from_file("assist.toml")
@@ -51,7 +46,8 @@ impl Config {
}
pub fn audio_client(&self) -> Option<AudioClient> {
if let Some(audio_server) = &self.audio_server {
if let Some(audio_config) = &self.audio {
let audio_server = &audio_config.server;
let mut audio_client = AudioClient::new(audio_server.url.clone());
if let Some(authorization) = &audio_server.authorization {
audio_client = audio_client.with_authorization(authorization.clone());
@@ -96,6 +92,24 @@ impl Config {
}
}
#[derive(Debug, Deserialize, Default)]
pub enum HumanInterface {
#[default]
Cli,
Audio,
}
#[derive(Debug, Deserialize)]
pub struct AudioConfig {
server: AudioServerConfig,
}
#[derive(Debug, Deserialize)]
pub struct AudioServerConfig {
url: Url,
authorization: Option<String>,
}
#[derive(Debug, Deserialize)]
pub struct OllamaConfig {
pub url: Option<Url>,
src/human_interface/audio.rs
+119 -19
View File
@@ -1,43 +1,128 @@
use crate::translate;
use crate::human_interface::{HumanInterface, HumanInterfaceError};
use cpal::traits::DeviceTrait;
use own_mcp::audio::models::VoiceRequest;
use own_mcp::audio::{AudioClient, AudioClientTrait};
use own_mcp::mcp::chat::PermissionAnswer;
use std::io;
use std::path::Path;
use tokio::fs::create_dir_all;
use tokio::io::AsyncBufReadExt;
use crate::tlt;
const TEMPORARY_AUDIO_PATH: &str = "temporary_audio";
struct Audio {
pub struct Audio {
client: AudioClient,
device: cpal::Device,
}
impl Audio {
async fn new(client: AudioClient) -> Result<Self, HumanInterfaceError> {
pub async fn new(
client: AudioClient,
device: cpal::Device,
) -> Result<Self, HumanInterfaceError> {
create_dir_all(TEMPORARY_AUDIO_PATH)
.await
.map_err(HumanInterfaceError::IoError)?;
Ok(Self { client })
Ok(Self { client, device })
}
}
impl HumanInterface for Audio {
async fn agent_message(&mut self, message: String) -> Result<(), HumanInterfaceError> {
async fn wait_for_input_line(expected_line: &str) -> Result<(), io::Error> {
loop {
let stdin_reader = tokio::io::BufReader::new(tokio::io::stdin());
log::trace!("wating for '{expected_line}' to continue");
if let Some(line) = stdin_reader.lines().next_line().await? {
log::debug!("Received line: {line:?}");
if line == expected_line {
log::trace!("stopped waiting");
return Ok(());
}
}
}
}
async fn wait_for_decision() -> Result<PermissionAnswer, io::Error> {
const GRANTED_LINE: &str = "y";
const DENIED_LINE: &str = "n";
loop {
let stdin_reader = tokio::io::BufReader::new(tokio::io::stdin());
log::trace!("wating for '{GRANTED_LINE}' or '{DENIED_LINE}' to continue");
if let Some(line) = stdin_reader.lines().next_line().await? {
log::debug!("Received line: {line:?}");
match line.as_str() {
GRANTED_LINE => { return Ok(PermissionAnswer::Granted) },
DENIED_LINE => { return Ok(PermissionAnswer::Denied) },
_ => {}
};
}
}
}
async fn play_text(&self, text: String) -> Result<(), HumanInterfaceError> {
let bytes = self
.client
.tts(VoiceRequest {
text: message,
text,
config: None,
})
.await
.map_err(|e| HumanInterfaceError::Other(format!("Failed getting tts: {e}")))?;
tokio::fs::write(Path::new(TEMPORARY_AUDIO_PATH).join("tts.wav"), &bytes)
.await
.map_err(|e| HumanInterfaceError::IoError(e))
let wav_file_path = Path::new(TEMPORARY_AUDIO_PATH).join("tts.wav");
tokio::fs::write(&wav_file_path, &bytes).await?;
let handle = rodio::DeviceSinkBuilder::open_default_sink().expect("open default audio stream");
let file = std::fs::File::open(wav_file_path)?;
let player = rodio::play(handle.mixer(), file).map_err(|e| HumanInterfaceError::Other(format!("Failed playing audio: {e}")))?;
player.sleep_until_end();
Ok(())
}
}
impl HumanInterface for Audio {
async fn agent_message(&mut self, message: String) -> Result<(), HumanInterfaceError> {
self.play_text(message).await?;
Ok(())
}
async fn expect_user_message(&mut self) -> Result<String, HumanInterfaceError> {
todo!()
use own_mcp::audio::recording::{samples_to_wav, start, stop_and_take_data};
const RECORDING_START: &str = "r";
const RECORDING_STOP: &str = "s";
let input_configs = self.device.default_input_config().map_err(|e| {
HumanInterfaceError::Other(format!("Could not get supported input configs: {e}"))
})?;
Self::wait_for_input_line(RECORDING_START).await?;
let recording_handler = start(self.device.clone(), input_configs);
Self::wait_for_input_line(RECORDING_STOP).await?;
let (samples, wav_spec) = stop_and_take_data(recording_handler)
.map_err(|e| HumanInterfaceError::Other(format!("failed recording: {e}")))?;
let wav_file_path = Path::new(TEMPORARY_AUDIO_PATH).join("recording.wav");
samples_to_wav(
samples,
&wav_spec,
&wav_file_path,
)
.map_err(|e| HumanInterfaceError::Other(format!("hound error: {e}")))?;
let transcription = self.client.transcribe(wav_file_path).await.map_err(|e|HumanInterfaceError::Other(format!("failed to transcribe: {e}")))?;
log::info!("transcription: {transcription:?}");
Ok(transcription.text)
}
async fn ask_for_permission(
@@ -45,27 +130,42 @@ impl HumanInterface for Audio {
mcp_server_name: String,
tool_name: String,
) -> Result<PermissionAnswer, HumanInterfaceError> {
todo!()
self.play_text(tlt!("tool_allow_question_audio", mcp_server_name, tool_name)).await?;
let decision = Self::wait_for_decision().await?;
log::debug!("decision: {decision:?}");
Ok(decision)
}
}
#[cfg(test)]
mod tests {
use super::*;
use cpal::traits::HostTrait;
const AUTH_HEADER: &str = include_str!(".AUTH_HEADER");
async fn get_test_human_interface() -> Audio {
Audio::new(AudioClient::new(
url::Url::parse("https://audio.mboemer.de"
).unwrap()).
with_authorization(
AUTH_HEADER.to_string()), cpal::Host::default().default_input_device().unwrap()).await.unwrap()
}
#[tokio::test]
async fn test_tts_fetching() {
let mut human_interface = Audio::new(
AudioClient::new(
url::Url::parse("https://audio.mboemer.de"
).unwrap()).
with_authorization(
"Basic bWlsYW46a3lLRVIjOVJAaDlAXldBI3pVXkAhb0pRIXViczNNV0g3U1NzQDVUenppJVFmciY5WEpxJXB1YmN6YiEkdmY1Z2FQd0N2aEd3".to_string())
).await.unwrap();
let mut human_interface = get_test_human_interface().await;
human_interface
.agent_message(String::from("Peter ist jetzt in deinem PC."))
.await
.unwrap();
}
#[tokio::test]
async fn test_tts_playing() {
let human_interface = get_test_human_interface().await;
human_interface.play_text(String::from("Peter ist jetzt in deinem PC.")).await.unwrap();
}
}
src/human_interface/cli.rs
+1 -1
View File
@@ -55,7 +55,7 @@ impl HumanInterface for CommandLine {
}
let confirmation = Confirm::with_theme(&ColorfulTheme::default())
.with_prompt(format!("Allow usage of {mcp_server_name}:{tool_name}"))
.with_prompt(tlt!("tool_allow_question_cli", mcp_server_name, tool_name))
.interact()
.map_err(|e| HumanInterfaceError::IoError(e.into()))?;
src/human_interface/mod.rs
+1 -1
View File
@@ -7,7 +7,7 @@ use own_mcp::mcp::chat::PermissionAnswer;
#[derive(Debug, Error)]
pub enum HumanInterfaceError {
#[error(transparent)]
IoError(std::io::Error),
IoError(#[from] std::io::Error),
#[error("other error in human interface: {0}")]
Other(String)
}
src/i18n/translations/de.json
+4 -1
View File
@@ -4,6 +4,9 @@
"you": "Du",
"assistant": "Assistent",
"username": "Nutzername",
"password": "Passwort"
"password": "Passwort",
"model_download_needed": "Das Modell '{...}' ist noch nicht installiert und muss zuerst heruntergeladen werden",
"tool_allow_question_cli": "Benutzung von {...}:{...} zulassen",
"tool_allow_question_audio": "Benutzung von {...} des Dienstes {...} zulassen?"
}
}
src/main.rs
+55 -27
View File
@@ -9,6 +9,8 @@ use crate::i18n::translate;
use crate::model::create_model_from_config;
use base64::Engine;
use clap::Parser;
use cpal::traits::HostTrait;
use own_mcp::AgentChat;
use std::fmt::Display;
/// partial mcp client with cli and voice interaction
@@ -35,6 +37,33 @@ fn basic_auth_tool() {
println!("Basic {}", encoded);
}
async fn chat_loop(mut human_interface: impl HumanInterface, mut agent_chat: AgentChat) {
loop {
let user_message = human_interface.expect_user_message().await.unwrap();
let immutable_interface = &human_interface;
let agent_message = agent_chat
.message(user_message, async |mcp_server_name, tool_name| {
immutable_interface
.ask_for_permission(mcp_server_name, tool_name)
.await
.inspect_err(exit_msg!(
"Human interface error while asking for permission"
))
.unwrap()
})
.await
.inspect_err(exit_msg!("Failed communicating with agent."))
.unwrap();
human_interface
.agent_message(agent_message.content)
.await
.inspect_err(exit_msg!(
"Human interface error while trying to display message"
))
.unwrap();
}
}
#[tokio::main]
async fn main() {
let args = Args::parse();
@@ -64,7 +93,7 @@ async fn main() {
create_model_from_config(&ollama, &config.ollama_config().model)
.await
.inspect_err(exit_msg!(format!(
"failed creating ollama model {model_name}"
"failed creating ollama model `{model_name}`"
)))
.unwrap();
@@ -85,37 +114,36 @@ async fn main() {
agent_chat.get_all_tools().collect::<Vec<_>>()
);
let mut human_interface = human_interface::cli::CommandLine::new();
loop {
let user_message = human_interface.expect_user_message().await.unwrap();
let immutable_interface = &human_interface;
let agent_message = agent_chat
.message(user_message, async |mcp_server_name, tool_name| {
immutable_interface
.ask_for_permission(mcp_server_name, tool_name)
match config.interface {
config::HumanInterface::Cli => {
let human_interface = human_interface::cli::CommandLine::new();
chat_loop(human_interface, agent_chat).await;
}
config::HumanInterface::Audio => {
match config.audio_client() {
Some(audio_client) => {
let human_interface = human_interface::audio::Audio::new(
audio_client,
cpal::Host::default()
.default_input_device()
.expect("no default device"),
)
.await
.inspect_err(exit_msg!(
"Human interface error while asking for permission"
))
.unwrap()
})
.await
.inspect_err(exit_msg!("Failed communicating with agent."))
.unwrap();
human_interface
.agent_message(agent_message.content)
.await
.inspect_err(exit_msg!(
"Human interface error while trying to display message"
))
.unwrap();
}
.inspect_err(exit_msg!("failed creating audio client")).unwrap();
chat_loop(human_interface, agent_chat).await;
}
None => {
eprintln!("audio client was not configured");
}
};
}
};
}
fn exit_with_error_message(error: impl Display, message: impl Display) {
log::error!("{message}: {error}");
println!("{}", message); // makes sure that message is printed even if logging is deactivated
eprintln!("{}", message); // makes sure that message is printed even if logging is deactivated
std::process::exit(1);
}
src/model.rs
+1 -4
View File
@@ -23,10 +23,7 @@ pub async fn create_model_from_config(
if let Ok(models) = ollama.list_local_models().await
&& !models.iter().any(|m| m.name == config.from_model)
{
let message = format!(
"Model `{}` is not installed and will be downloaded first.",
config.from_model
);
let message = tlt!("model_download_needed", config.from_model.clone());
log::info!("{}", message);
println!("{}", console::style(message).yellow().bold());
}