milan/own_assist
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add working Audio HumanInterface

Browse Source 
refactoring as always
remove secrets
This commit is contained in:
milan
2026-04-21 21:55:17 +02:00
parent ef468e8fd6
commit 20a5b01311
10 changed files with 474 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/config.rs
+23 -9
View File
@@ -13,20 +13,15 @@ use own_assist_common::config_from_file;
#[derive(Debug, Deserialize)]
pub struct Config {
#[serde(default)]
pub interface: HumanInterface,
permissions: Option<Vec<PermissionConfig>>,
ollama: OllamaConfig,
#[serde(rename = "audio-server")]
audio_server: Option<AudioServerConfig>,
audio: Option<AudioConfig>,
#[serde(rename = "mcp-servers")]
mcp_servers: Vec<MCPServerConfig>,
}
#[derive(Debug, Deserialize)]
pub struct AudioServerConfig {
url: Url,
authorization: Option<String>,
}
impl Config {
pub fn from_file() -> Result<Self, ConfigLoadingError> {
config_from_file("assist.toml")
@@ -51,7 +46,8 @@ impl Config {
}
pub fn audio_client(&self) -> Option<AudioClient> {
if let Some(audio_server) = &self.audio_server {
if let Some(audio_config) = &self.audio {
let audio_server = &audio_config.server;
let mut audio_client = AudioClient::new(audio_server.url.clone());
if let Some(authorization) = &audio_server.authorization {
audio_client = audio_client.with_authorization(authorization.clone());
@@ -96,6 +92,24 @@ impl Config {
}
}
#[derive(Debug, Deserialize, Default)]
pub enum HumanInterface {
#[default]
Cli,
Audio,
}
#[derive(Debug, Deserialize)]
pub struct AudioConfig {
server: AudioServerConfig,
}
#[derive(Debug, Deserialize)]
pub struct AudioServerConfig {
url: Url,
authorization: Option<String>,
}
#[derive(Debug, Deserialize)]
pub struct OllamaConfig {
pub url: Option<Url>,
src/human_interface/audio.rs
+119 -19
View File
@@ -1,43 +1,128 @@
use crate::translate;
use crate::human_interface::{HumanInterface, HumanInterfaceError};
use cpal::traits::DeviceTrait;
use own_mcp::audio::models::VoiceRequest;
use own_mcp::audio::{AudioClient, AudioClientTrait};
use own_mcp::mcp::chat::PermissionAnswer;
use std::io;
use std::path::Path;
use tokio::fs::create_dir_all;
use tokio::io::AsyncBufReadExt;
use crate::tlt;
const TEMPORARY_AUDIO_PATH: &str = "temporary_audio";
struct Audio {
pub struct Audio {
client: AudioClient,
device: cpal::Device,
}
impl Audio {
async fn new(client: AudioClient) -> Result<Self, HumanInterfaceError> {
pub async fn new(
client: AudioClient,
device: cpal::Device,
) -> Result<Self, HumanInterfaceError> {
create_dir_all(TEMPORARY_AUDIO_PATH)
.await
.map_err(HumanInterfaceError::IoError)?;
Ok(Self { client })
Ok(Self { client, device })
}
}
impl HumanInterface for Audio {
async fn agent_message(&mut self, message: String) -> Result<(), HumanInterfaceError> {
async fn wait_for_input_line(expected_line: &str) -> Result<(), io::Error> {
loop {
let stdin_reader = tokio::io::BufReader::new(tokio::io::stdin());
log::trace!("wating for '{expected_line}' to continue");
if let Some(line) = stdin_reader.lines().next_line().await? {
log::debug!("Received line: {line:?}");
if line == expected_line {
log::trace!("stopped waiting");
return Ok(());
}
}
}
}
async fn wait_for_decision() -> Result<PermissionAnswer, io::Error> {
const GRANTED_LINE: &str = "y";
const DENIED_LINE: &str = "n";
loop {
let stdin_reader = tokio::io::BufReader::new(tokio::io::stdin());
log::trace!("wating for '{GRANTED_LINE}' or '{DENIED_LINE}' to continue");
if let Some(line) = stdin_reader.lines().next_line().await? {
log::debug!("Received line: {line:?}");
match line.as_str() {
GRANTED_LINE => { return Ok(PermissionAnswer::Granted) },
DENIED_LINE => { return Ok(PermissionAnswer::Denied) },
_ => {}
};
}
}
}
async fn play_text(&self, text: String) -> Result<(), HumanInterfaceError> {
let bytes = self
.client
.tts(VoiceRequest {
text: message,
text,
config: None,
})
.await
.map_err(|e| HumanInterfaceError::Other(format!("Failed getting tts: {e}")))?;
tokio::fs::write(Path::new(TEMPORARY_AUDIO_PATH).join("tts.wav"), &bytes)
.await
.map_err(|e| HumanInterfaceError::IoError(e))
let wav_file_path = Path::new(TEMPORARY_AUDIO_PATH).join("tts.wav");
tokio::fs::write(&wav_file_path, &bytes).await?;
let handle = rodio::DeviceSinkBuilder::open_default_sink().expect("open default audio stream");
let file = std::fs::File::open(wav_file_path)?;
let player = rodio::play(handle.mixer(), file).map_err(|e| HumanInterfaceError::Other(format!("Failed playing audio: {e}")))?;
player.sleep_until_end();
Ok(())
}
}
impl HumanInterface for Audio {
async fn agent_message(&mut self, message: String) -> Result<(), HumanInterfaceError> {
self.play_text(message).await?;
Ok(())
}
async fn expect_user_message(&mut self) -> Result<String, HumanInterfaceError> {
todo!()
use own_mcp::audio::recording::{samples_to_wav, start, stop_and_take_data};
const RECORDING_START: &str = "r";
const RECORDING_STOP: &str = "s";
let input_configs = self.device.default_input_config().map_err(|e| {
HumanInterfaceError::Other(format!("Could not get supported input configs: {e}"))
})?;
Self::wait_for_input_line(RECORDING_START).await?;
let recording_handler = start(self.device.clone(), input_configs);
Self::wait_for_input_line(RECORDING_STOP).await?;
let (samples, wav_spec) = stop_and_take_data(recording_handler)
.map_err(|e| HumanInterfaceError::Other(format!("failed recording: {e}")))?;
let wav_file_path = Path::new(TEMPORARY_AUDIO_PATH).join("recording.wav");
samples_to_wav(
samples,
&wav_spec,
&wav_file_path,
)
.map_err(|e| HumanInterfaceError::Other(format!("hound error: {e}")))?;
let transcription = self.client.transcribe(wav_file_path).await.map_err(|e|HumanInterfaceError::Other(format!("failed to transcribe: {e}")))?;
log::info!("transcription: {transcription:?}");
Ok(transcription.text)
}
async fn ask_for_permission(
@@ -45,27 +130,42 @@ impl HumanInterface for Audio {
mcp_server_name: String,
tool_name: String,
) -> Result<PermissionAnswer, HumanInterfaceError> {
todo!()
self.play_text(tlt!("tool_allow_question_audio", mcp_server_name, tool_name)).await?;
let decision = Self::wait_for_decision().await?;
log::debug!("decision: {decision:?}");
Ok(decision)
}
}
#[cfg(test)]
mod tests {
use super::*;
use cpal::traits::HostTrait;
const AUTH_HEADER: &str = include_str!(".AUTH_HEADER");
async fn get_test_human_interface() -> Audio {
Audio::new(AudioClient::new(
url::Url::parse("https://audio.mboemer.de"
).unwrap()).
with_authorization(
AUTH_HEADER.to_string()), cpal::Host::default().default_input_device().unwrap()).await.unwrap()
}
#[tokio::test]
async fn test_tts_fetching() {
let mut human_interface = Audio::new(
AudioClient::new(
url::Url::parse("https://audio.mboemer.de"
).unwrap()).
with_authorization(
"Basic bWlsYW46a3lLRVIjOVJAaDlAXldBI3pVXkAhb0pRIXViczNNV0g3U1NzQDVUenppJVFmciY5WEpxJXB1YmN6YiEkdmY1Z2FQd0N2aEd3".to_string())
).await.unwrap();
let mut human_interface = get_test_human_interface().await;
human_interface
.agent_message(String::from("Peter ist jetzt in deinem PC."))
.await
.unwrap();
}
#[tokio::test]
async fn test_tts_playing() {
let human_interface = get_test_human_interface().await;
human_interface.play_text(String::from("Peter ist jetzt in deinem PC.")).await.unwrap();
}
}
src/human_interface/cli.rs
+1 -1
View File
@@ -55,7 +55,7 @@ impl HumanInterface for CommandLine {
}
let confirmation = Confirm::with_theme(&ColorfulTheme::default())
.with_prompt(format!("Allow usage of {mcp_server_name}:{tool_name}"))
.with_prompt(tlt!("tool_allow_question_cli", mcp_server_name, tool_name))
.interact()
.map_err(|e| HumanInterfaceError::IoError(e.into()))?;
src/human_interface/mod.rs
+1 -1
View File
@@ -7,7 +7,7 @@ use own_mcp::mcp::chat::PermissionAnswer;
#[derive(Debug, Error)]
pub enum HumanInterfaceError {
#[error(transparent)]
IoError(std::io::Error),
IoError(#[from] std::io::Error),
#[error("other error in human interface: {0}")]
Other(String)
}
src/i18n/translations/de.json
+4 -1
View File
@@ -4,6 +4,9 @@
"you": "Du",
"assistant": "Assistent",
"username": "Nutzername",
"password": "Passwort"
"password": "Passwort",
"model_download_needed": "Das Modell '{...}' ist noch nicht installiert und muss zuerst heruntergeladen werden",
"tool_allow_question_cli": "Benutzung von {...}:{...} zulassen",
"tool_allow_question_audio": "Benutzung von {...} des Dienstes {...} zulassen?"
}
}
src/main.rs
+55 -27
View File
@@ -9,6 +9,8 @@ use crate::i18n::translate;
use crate::model::create_model_from_config;
use base64::Engine;
use clap::Parser;
use cpal::traits::HostTrait;
use own_mcp::AgentChat;
use std::fmt::Display;
/// partial mcp client with cli and voice interaction
@@ -35,6 +37,33 @@ fn basic_auth_tool() {
println!("Basic {}", encoded);
}
async fn chat_loop(mut human_interface: impl HumanInterface, mut agent_chat: AgentChat) {
loop {
let user_message = human_interface.expect_user_message().await.unwrap();
let immutable_interface = &human_interface;
let agent_message = agent_chat
.message(user_message, async |mcp_server_name, tool_name| {
immutable_interface
.ask_for_permission(mcp_server_name, tool_name)
.await
.inspect_err(exit_msg!(
"Human interface error while asking for permission"
))
.unwrap()
})
.await
.inspect_err(exit_msg!("Failed communicating with agent."))
.unwrap();
human_interface
.agent_message(agent_message.content)
.await
.inspect_err(exit_msg!(
"Human interface error while trying to display message"
))
.unwrap();
}
}
#[tokio::main]
async fn main() {
let args = Args::parse();
@@ -64,7 +93,7 @@ async fn main() {
create_model_from_config(&ollama, &config.ollama_config().model)
.await
.inspect_err(exit_msg!(format!(
"failed creating ollama model {model_name}"
"failed creating ollama model `{model_name}`"
)))
.unwrap();
@@ -85,37 +114,36 @@ async fn main() {
agent_chat.get_all_tools().collect::<Vec<_>>()
);
let mut human_interface = human_interface::cli::CommandLine::new();
loop {
let user_message = human_interface.expect_user_message().await.unwrap();
let immutable_interface = &human_interface;
let agent_message = agent_chat
.message(user_message, async |mcp_server_name, tool_name| {
immutable_interface
.ask_for_permission(mcp_server_name, tool_name)
match config.interface {
config::HumanInterface::Cli => {
let human_interface = human_interface::cli::CommandLine::new();
chat_loop(human_interface, agent_chat).await;
}
config::HumanInterface::Audio => {
match config.audio_client() {
Some(audio_client) => {
let human_interface = human_interface::audio::Audio::new(
audio_client,
cpal::Host::default()
.default_input_device()
.expect("no default device"),
)
.await
.inspect_err(exit_msg!(
"Human interface error while asking for permission"
))
.unwrap()
})
.await
.inspect_err(exit_msg!("Failed communicating with agent."))
.unwrap();
human_interface
.agent_message(agent_message.content)
.await
.inspect_err(exit_msg!(
"Human interface error while trying to display message"
))
.unwrap();
}
.inspect_err(exit_msg!("failed creating audio client")).unwrap();
chat_loop(human_interface, agent_chat).await;
}
None => {
eprintln!("audio client was not configured");
}
};
}
};
}
fn exit_with_error_message(error: impl Display, message: impl Display) {
log::error!("{message}: {error}");
println!("{}", message); // makes sure that message is printed even if logging is deactivated
eprintln!("{}", message); // makes sure that message is printed even if logging is deactivated
std::process::exit(1);
}
src/model.rs
+1 -4
View File
@@ -23,10 +23,7 @@ pub async fn create_model_from_config(
if let Ok(models) = ollama.list_local_models().await
&& !models.iter().any(|m| m.name == config.from_model)
{
let message = format!(
"Model `{}` is not installed and will be downloaded first.",
config.from_model
);
let message = tlt!("model_download_needed", config.from_model.clone());
log::info!("{}", message);
println!("{}", console::style(message).yellow().bold());
}