From 5bf98a1efc9764663bcb407e09b4d99093965115 Mon Sep 17 00:00:00 2001 From: milan Date: Thu, 9 Apr 2026 16:21:58 +0200 Subject: [PATCH] add logging config to audio server add audio client change delimiter for mcp tools from "::" to ":" to save tokens --- Cargo.lock | 302 ++++++++++++++++++++++ audio_server/Containerfile | 2 +- audio_server/logging_config.yaml | 42 +++ audio_server/requirements.txt | 3 +- audio_server/src/audio_server/__init__.py | 38 +-- audio_server/src/audio_server/models.py | 32 +++ mcp_server_collection/src/main.rs | 2 +- own_mcp/Cargo.toml | 7 +- own_mcp/src/audio/client.rs | 71 +++++ own_mcp/src/audio/mod.rs | 4 + own_mcp/src/audio/models.rs | 60 +++++ own_mcp/src/audio/recording.rs | 197 ++++++++++++++ own_mcp/src/lib.rs | 1 + own_mcp/src/mcp/chat.rs | 10 +- own_mcp/src/mcp/translation.rs | 4 +- src/config.rs | 18 ++ src/main.rs | 18 +- 17 files changed, 776 insertions(+), 35 deletions(-) create mode 100644 audio_server/logging_config.yaml create mode 100644 audio_server/src/audio_server/models.py create mode 100644 own_mcp/src/audio/client.rs create mode 100644 own_mcp/src/audio/mod.rs create mode 100644 own_mcp/src/audio/models.rs create mode 100644 own_mcp/src/audio/recording.rs diff --git a/Cargo.lock b/Cargo.lock index 662d702..25d3f7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,28 @@ dependencies = [ "memchr", ] +[[package]] +name = "alsa" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812947049edcd670a82cd5c73c3661d2e58468577ba8489de58e1a73c04cbd5d" +dependencies = [ + "alsa-sys", + "bitflags", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad7569085a265dd3f607ebecce7458eaab2132a84393534c95b18dcbc3f31e04" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -164,6 +186,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -282,6 +313,50 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "coreaudio-rs" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16dd574a72a021b90c7656c474ea31d11a2f0366a8eff574186e761e0b9e3586" +dependencies = [ + "bitflags", + "libc", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "cpal" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8942da362c0f0d895d7cac616263f2f9424edc5687364dfd1d25ef7eba506d7" +dependencies = [ + "alsa", + "coreaudio-rs", + "dasp_sample", + "jni", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "objc2", + "objc2-audio-toolbox", + "objc2-avf-audio", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -344,6 +419,12 @@ dependencies = [ "syn", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "digest" version = "0.10.7" @@ -354,6 +435,16 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags", + "objc2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -658,6 +749,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hound" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f" + [[package]] name = "http" version = "1.4.0" @@ -1080,6 +1177,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "mach2" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1b95cd5421ec55b445b5ae102f5ea0e768de1f82bd3001e11f426c269c3aea" +dependencies = [ + "libc", +] + [[package]] name = "memchr" version = "2.8.0" @@ -1092,6 +1198,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "mio" version = "1.2.0" @@ -1120,6 +1236,35 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndk" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4" +dependencies = [ + "bitflags", + "jni-sys 0.3.1", + "log", + "ndk-sys", + "num_enum", + "thiserror 1.0.69", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "ndk-sys" +version = "0.6.0+11769913" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6cda3051665f1fb8d9e08fc35c96d5a244fb1be711a03b71118828afc9a873" +dependencies = [ + "jni-sys 0.3.1", +] + [[package]] name = "nix" version = "0.31.2" @@ -1132,6 +1277,17 @@ dependencies = [ "libc", ] +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1141,6 +1297,28 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "oauth2" version = "5.0.0" @@ -1160,6 +1338,95 @@ dependencies = [ "url", ] +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-audio-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" +dependencies = [ + "bitflags", + "libc", + "objc2", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-avf-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" +dependencies = [ + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags", + "objc2", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", + "block2", + "dispatch2", + "libc", + "objc2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags", + "block2", + "libc", + "objc2", + "objc2-core-foundation", +] + [[package]] name = "ollama-rs" version = "0.3.4" @@ -1274,7 +1541,10 @@ dependencies = [ name = "own_mcp" version = "0.1.0" dependencies = [ + "bytes", + "cpal", "env_logger", + "hound", "log", "ollama-rs", "rand 0.10.0", @@ -1353,6 +1623,15 @@ dependencies = [ "syn", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -1635,6 +1914,7 @@ dependencies = [ "js-sys", "log", "mime", + "mime_guess", "percent-encoding", "pin-project-lite", "quinn", @@ -1643,6 +1923,7 @@ dependencies = [ "rustls-platform-verifier", "serde", "serde_json", + "serde_urlencoded", "sync_wrapper", "tokio", "tokio-rustls", @@ -2290,6 +2571,18 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.25.11+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + [[package]] name = "toml_parser" version = "1.1.2+spec-1.1.0" @@ -2393,6 +2686,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -2979,6 +3278,9 @@ name = "winnow" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" +dependencies = [ + "memchr", +] [[package]] name = "wit-bindgen" diff --git a/audio_server/Containerfile b/audio_server/Containerfile index eae642a..f3e903c 100644 --- a/audio_server/Containerfile +++ b/audio_server/Containerfile @@ -16,4 +16,4 @@ RUN apt -y install ffmpeg WORKDIR /app EXPOSE 8000 -CMD ["uvicorn", "audio_server:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file +CMD ["uvicorn", "audio_server:app", "--host", "0.0.0.0", "--port", "8000", "--log-config", "logging_config.yaml"] \ No newline at end of file diff --git a/audio_server/logging_config.yaml b/audio_server/logging_config.yaml new file mode 100644 index 0000000..fb379f0 --- /dev/null +++ b/audio_server/logging_config.yaml @@ -0,0 +1,42 @@ +version: 1 +disable_existing_logger: false +formatters: + main: + format: "\x1b[1;32m%(name)s \x1b[0m- %(levelname)s - %(message)s" + default: + format: "\x1b[1;93m%(name)s \x1b[0m- %(levelname)s - %(message)s" + use_colors: yes +handlers: + main: + formatter: main + class: logging.StreamHandler + stream: ext://sys.stdout + default: + formatter: default + class: logging.StreamHandler + stream: ext://sys.stdout + access: + formatter: default + class: logging.StreamHandler + stream: ext://sys.stdout +loggers: + audio_server: + level: INFO + handlers: + - main + propagate: no + uvicorn.error: + level: INFO + handlers: + - default + propagate: no + uvicorn.access: + level: INFO + handlers: + - default + propagate: no +root: + level: INFO + handlers: + - default + propagate: no \ No newline at end of file diff --git a/audio_server/requirements.txt b/audio_server/requirements.txt index 99d547f..89f83e5 100644 --- a/audio_server/requirements.txt +++ b/audio_server/requirements.txt @@ -5,4 +5,5 @@ uvicorn~=0.44.0 starlette~=1.0.0 python-multipart~=0.0.24 aiofiles~=25.1.0 -piper-tts~=1.4.2 \ No newline at end of file +piper-tts~=1.4.2 +PyYAML~=6.0.3 \ No newline at end of file diff --git a/audio_server/src/audio_server/__init__.py b/audio_server/src/audio_server/__init__.py index 67d4590..00f210e 100644 --- a/audio_server/src/audio_server/__init__.py +++ b/audio_server/src/audio_server/__init__.py @@ -5,13 +5,14 @@ from wave import Wave_write from pathlib import Path import aiofiles -from piper import PiperVoice, SynthesisConfig +from piper import PiperVoice from piper.download_voices import download_voice import psutil -from fastapi import FastAPI, UploadFile +from fastapi import FastAPI, UploadFile, HTTPException import whisper -from pydantic import BaseModel -from fastapi.responses import FileResponse, JSONResponse +from fastapi.responses import FileResponse + +from .models import TranscriptionResponse, VoiceRequest, StatusResponse app = FastAPI() @@ -28,26 +29,28 @@ download_voice(piper_model_name, piper_path) temporary_audio_path = Path("temporary_audio") temporary_audio_path.mkdir(parents=True, exist_ok=True) -logger = logging.getLogger(__name__) +logger = logging.getLogger("audio_server") + @app.get("/") -async def index(): - return { - "memory_usage": psutil.Process(os.getpid()).memory_info().rss, - "piper_model": piper_model_name, - "whisper_model": whisper_model_name, - } +async def index() -> StatusResponse: + return StatusResponse(memory_usage=psutil.Process(os.getpid()).memory_info().rss, piper_model=piper_model_name, + whisper_model=whisper_model_name) + @app.get("/transcribe") -async def transcribe(audio_file: UploadFile): +async def transcribe(audio_file: UploadFile) -> TranscriptionResponse: file_extensions = { + "audio/wav": "wav", "audio/wave": "wav", "audio/mpeg": "mp3", } if audio_file.content_type not in file_extensions.keys(): logger.error("%s is not supported", audio_file.content_type) - return JSONResponse({"detail": "wrong file type"}, status_code=400) + print("CONTENT TYPE:") + print(audio_file.content_type) + raise HTTPException(status_code=400, detail="file type not supported") file_path = temporary_audio_path / f"{uuid.uuid4()}.{file_extensions[audio_file.content_type]}" @@ -58,14 +61,11 @@ async def transcribe(audio_file: UploadFile): # noinspection PyArgumentList transcription = whisper_model.transcribe(file_path.as_posix()) - return JSONResponse(transcription) + return TranscriptionResponse.model_validate(transcription) -class VoiceRequest(BaseModel): - text: str - config: SynthesisConfig | None @app.get("/tts") -async def tts(voice_request: VoiceRequest): +async def tts(voice_request: VoiceRequest) -> FileResponse: voice = PiperVoice.load(piper_path / f"{piper_model_name}.onnx") audio_file_path = temporary_audio_path / f"{uuid.uuid4()}.wav" @@ -73,4 +73,4 @@ async def tts(voice_request: VoiceRequest): with Wave_write(audio_file_path.as_posix()) as writer: voice.synthesize_wav(voice_request.text, writer, syn_config=voice_request.config) - return FileResponse(audio_file_path, media_type="audio/wav") \ No newline at end of file + return FileResponse(audio_file_path, media_type="audio/wav") diff --git a/audio_server/src/audio_server/models.py b/audio_server/src/audio_server/models.py new file mode 100644 index 0000000..8f8fab0 --- /dev/null +++ b/audio_server/src/audio_server/models.py @@ -0,0 +1,32 @@ +from piper import SynthesisConfig +from pydantic import BaseModel + + +class StatusResponse(BaseModel): + memory_usage: int + piper_model: str + whisper_model: str + + +# noinspection SpellCheckingInspection +class TranscriptionSegment(BaseModel): + id: int + seek: int + start: float + end: float + text: str + tokens: list[int] + temperature: float + avg_logprob: float + compression_ratio: float + no_speech_prob: float + +class TranscriptionResponse(BaseModel): + text: str + segments: list[TranscriptionSegment] + language: str + + +class VoiceRequest(BaseModel): + text: str + config: SynthesisConfig | None \ No newline at end of file diff --git a/mcp_server_collection/src/main.rs b/mcp_server_collection/src/main.rs index fe053ae..cf01a31 100644 --- a/mcp_server_collection/src/main.rs +++ b/mcp_server_collection/src/main.rs @@ -67,7 +67,7 @@ async fn main() { let router = mcp_router(&config.servers).route("/", axum::routing::get(|| async { Json(config.servers) })); - let bind_address = config.bind_address.unwrap_or(url::Url::parse("localhost:8000").unwrap()); + let bind_address = config.bind_address.unwrap_or(url::Url::parse("http://localhost:8000").unwrap()); println!("binding address at {bind_address}"); let tcp_listener = tokio::net::TcpListener::bind(bind_address_format(bind_address)).await.inspect_err(|e|{ diff --git a/own_mcp/Cargo.toml b/own_mcp/Cargo.toml index 1add7f5..f355461 100644 --- a/own_mcp/Cargo.toml +++ b/own_mcp/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" [dependencies] ollama-rs = {version = "0.3.4", features = ["macros", "headers"]} -reqwest = "0.13.2" +reqwest = { version = "0.13.2", features = ["stream", "multipart", "form"] } tokio = { version = "1.50.0", features = ["rt", "rt-multi-thread", "macros"] } rmcp = {version="1.3.0", features = ["transport-streamable-http-client-reqwest", "reqwest", "client", "auth", "transport-child-process"]} log = {version = "0.4.29"} @@ -13,4 +13,7 @@ env_logger = "0.11.10" serde = { version = "1.0.228", features = ["derive"] } thiserror = "2.0.17" url = "2.5.8" -rand = "0.10.0" \ No newline at end of file +rand = "0.10.0" +cpal = "0.17.3" +hound = "3.5.1" +bytes = "1.11.1" \ No newline at end of file diff --git a/own_mcp/src/audio/client.rs b/own_mcp/src/audio/client.rs new file mode 100644 index 0000000..ebc45e9 --- /dev/null +++ b/own_mcp/src/audio/client.rs @@ -0,0 +1,71 @@ +use std::io::ErrorKind; +use std::path::Path; +use reqwest::Client; +use reqwest::multipart::Form; +use thiserror::Error; +use crate::audio::models::{AudioServerStatus, TranscriptionResponse, VoiceRequest}; +use url::Url; + +pub struct AudioClient{ + client: Client, + authorization: Option, + base_url: Url, +} + +pub trait AudioClientTrait { + fn new(base_url: Url) -> Self; + fn from_client(base_url: Url, client: Client) -> Self; + + fn with_authorization(self, authorization: String) -> Self; + + fn status(&self) -> impl Future>; + + fn transcribe(&self, audio_file_path: impl AsRef) -> impl Future>; + + fn tts(&self, voice_request: VoiceRequest) -> impl Future>; +} + +#[derive(Debug, Error)] +pub enum AudioError { + #[error(transparent)] + RequestError(#[from] reqwest::Error), + #[error(transparent)] + UrlParseError(#[from] url::ParseError), + #[error(transparent)] + IOError(#[from] std::io::Error), +} + +type AudioResult = Result; + +impl AudioClientTrait for AudioClient { + fn new(base_url: Url) -> Self { + AudioClient { client: Client::new(), authorization: None, base_url } + } + + fn from_client(base_url: Url, client: Client) -> Self { + AudioClient { client, authorization: None, base_url } + } + + fn with_authorization(mut self, authorization: String) -> Self { + self.authorization = Some(authorization); + self + } + + async fn status(&self) -> AudioResult { + let response = self.client.get(self.base_url.clone()).send().await?.error_for_status()?; + Ok(response.json().await?) + } + + async fn transcribe(&self, audio_file_path: impl AsRef) -> AudioResult { + let form = Form::new().file("audio_file", audio_file_path).await?; + + let response = self.client.get(self.base_url.join("transcribe")?).multipart(form).send().await?; + Ok(response.json().await?) + } + + async fn tts(&self, voice_request: VoiceRequest) -> AudioResult { + let response = self.client.get(self.base_url.join("tts")?).json(&voice_request).send().await?.error_for_status()?; + + Ok(response.bytes().await?) + } +} \ No newline at end of file diff --git a/own_mcp/src/audio/mod.rs b/own_mcp/src/audio/mod.rs new file mode 100644 index 0000000..b963b73 --- /dev/null +++ b/own_mcp/src/audio/mod.rs @@ -0,0 +1,4 @@ +pub mod recording; +pub mod client; +pub mod models; +pub use client::{AudioClient, AudioClientTrait}; \ No newline at end of file diff --git a/own_mcp/src/audio/models.rs b/own_mcp/src/audio/models.rs new file mode 100644 index 0000000..02067a7 --- /dev/null +++ b/own_mcp/src/audio/models.rs @@ -0,0 +1,60 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Deserialize)] +pub struct AudioServerStatus { + #[serde(rename = "memory_usage")] + pub memory_usage_bytes: usize, + pub piper_model: String, + pub whisper_model: String, +} + +//noinspection SpellCheckingInspection +#[derive(Debug, Clone, Deserialize)] +pub struct TranscriptionSegment { + pub id: u32, + pub seek: u32, + pub start: f32, + pub end: f32, + pub text: String, + pub tokens: Vec, + pub temperature: f32, + pub avg_logprob: f32, + pub compression_ratio: f32, + pub no_speech_prob: f32 +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TranscriptionResponse { + pub text: String, + pub segments: Vec, + pub language: String +} + +#[derive(Debug, Clone, Serialize)] +pub struct SynthesisConfig { + pub speaker_id: Option, + pub length_scale: Option, + pub noise_scale: Option, + pub noise_w_scale: Option, + pub normalize_audio: bool, + pub volume: f32, +} + +impl Default for SynthesisConfig { + fn default() -> Self { + SynthesisConfig { + speaker_id: None, + length_scale: None, + noise_scale: None, + noise_w_scale: None, + normalize_audio: true, + volume: 1.0, + } + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct VoiceRequest { + pub text: String, + pub config: Option, +} \ No newline at end of file diff --git a/own_mcp/src/audio/recording.rs b/own_mcp/src/audio/recording.rs new file mode 100644 index 0000000..22dc31f --- /dev/null +++ b/own_mcp/src/audio/recording.rs @@ -0,0 +1,197 @@ +use cpal::traits::{DeviceTrait, StreamTrait}; +use cpal::{BuildStreamError, Device, SampleFormat, SizedSample, Stream, SupportedStreamConfig}; +use std::fmt::Debug; +use std::sync::{Arc, Mutex}; +use std::{path, thread}; +use std::thread::JoinHandle; +use std::time::Duration; +use hound::WavSpec; +use log::{debug, error, info, trace}; +use thiserror::__private18::AsDisplay; +use thiserror::Error; + +fn generic_sample_to_i16(sample: impl SizedSample + Into, format: SampleFormat) -> i16 { + match format { + SampleFormat::I8 => (sample.into() * 2.0) as i16, + SampleFormat::I16 => sample.into() as i16, + SampleFormat::I32 => (sample.into() / 2.0) as i16, + SampleFormat::F32 => (sample.into() * (i16::MAX as f64)) as i16, + _ => panic!("Unsupported sample format {:?}", format), + } +} + +pub type SampleArc = Arc>>; + +fn stream_callback>( + input: &[T], + samples: SampleArc, + sample_format: SampleFormat, +) { + if let Ok(mut guard) = samples.lock() { + for &sample in input { + guard.push(generic_sample_to_i16(sample, sample_format)); + } + } +} + +fn build_stream>( + mic: &Device, + device_config: SupportedStreamConfig, + samples: SampleArc, + sample_format: SampleFormat, +) -> Result { + mic.build_input_stream( + &device_config.into(), + move |input: &[T], _info| stream_callback::(input, samples.clone(), sample_format), + |e| { + error!("a stream error occurred while trying to record: {:?}", e); + }, + None, + ) +} + +#[derive(Debug)] +pub struct RecordingHandler { + pub samples: SampleArc, + pub spec: WavSpec, + thread_handle: JoinHandle>, + should_stop: Arc>, +} + +impl RecordingHandler { + pub fn stop_recording(self) -> Result<(), RecordingError> { + info!("Stopping recording"); + *self.should_stop.lock().unwrap() = true; + self.thread_handle.join().unwrap() + } +} + +#[derive(Error, Debug)] +pub enum RecordingError { + #[error("unsupported sample format")] + UnsupportedSampleFormat(SampleFormat), + #[error(transparent)] + PlayStreamError(#[from] cpal::PlayStreamError), + #[error(transparent)] + BuildStreamError(#[from] BuildStreamError), + #[error("thread poisoned")] + ThreadPoison, + #[error("recording was already stopped")] + RecordingAlreadyStopped, +} + +fn start_recording_blocking_with_parameters( + mic: &Device, + device_config: &SupportedStreamConfig, + sample_format: SampleFormat, + samples: SampleArc, + should_stop: Arc>, +) -> Result<(), RecordingError> { + let stream = match sample_format { + SampleFormat::I8 => { + build_stream::(mic, device_config.clone(), samples.clone(), sample_format)? + } + SampleFormat::I16 => { + build_stream::(mic, device_config.clone(), samples.clone(), sample_format)? + } + SampleFormat::I32 => { + build_stream::(mic, device_config.clone(), samples.clone(), sample_format)? + } + SampleFormat::F32 => { + build_stream::(mic, device_config.clone(), samples.clone(), sample_format)? + } + sample_format => { + return Err(RecordingError::UnsupportedSampleFormat(sample_format)); + } + }; + + stream.play()?; + + loop { + thread::sleep(Duration::from_millis(100)); + if let Ok(guard) = should_stop.lock() { + if *guard { + debug!("Stopping recording gracefully"); + break; + } + } + } + + drop(stream); + + Ok(()) +} + +pub fn start( + microphone: Device, + device_config: SupportedStreamConfig, +) -> Arc>> { + let sample_format = device_config.sample_format(); + + let samples: SampleArc = Arc::new(Mutex::new(Vec::new())); + let recording_should_stop = Arc::new(Mutex::new(false)); + + info!("started recording"); + debug!("sample format: {:?}", sample_format); + + let thread_samples = samples.clone(); + let thread_recording_stop = recording_should_stop.clone(); + let thread_device_config = device_config.clone(); + let record_thread_handle = thread::spawn(move || + start_recording_blocking_with_parameters( + µphone, + &thread_device_config, + sample_format, + thread_samples, + thread_recording_stop, + ) + ); + + let spec = WavSpec { + channels: device_config.channels(), + sample_rate: device_config.sample_rate(), + bits_per_sample: 16, + sample_format: hound::SampleFormat::Int, + }; + + debug!("device spec: {:?}", spec); + + Arc::new(Mutex::new(Some(RecordingHandler { + spec, + samples, + thread_handle: record_thread_handle, + should_stop: recording_should_stop, + }))) +} + +pub fn stop_and_take_data(handler: Arc>>) -> Result<(Vec, WavSpec), RecordingError> { + let handler = handler.lock().map_err(|_|RecordingError::ThreadPoison)?.take().ok_or(RecordingError::RecordingAlreadyStopped)?; + + let samples = handler.samples.clone(); + let spec = handler.spec.clone(); + + handler.stop_recording()?; + + Ok((samples.lock().map_err(|_|RecordingError::ThreadPoison)?.to_vec(), spec)) +} + +pub fn samples_to_wav( + samples: impl IntoIterator, + spec: &WavSpec, + filepath: impl AsRef +) -> Result, hound::Error> { + let wav_bytes = std::io::Cursor::new(Vec::::new()); + + trace!("creating a wav file writer to {}", filepath.as_ref().display()); + let mut file_writer = hound::WavWriter::create(&filepath, *spec)?; + + trace!("writing samples..."); + for sample in samples { + file_writer.write_sample(sample)?; + } + + trace!("finalizing for file {}", filepath.as_ref().display()); + file_writer.finalize()?; + + Ok(wav_bytes.into_inner()) +} \ No newline at end of file diff --git a/own_mcp/src/lib.rs b/own_mcp/src/lib.rs index 78c922a..8d92737 100644 --- a/own_mcp/src/lib.rs +++ b/own_mcp/src/lib.rs @@ -1,3 +1,4 @@ pub mod mcp; +pub mod audio; pub use mcp::chat::AgentChat; \ No newline at end of file diff --git a/own_mcp/src/mcp/chat.rs b/own_mcp/src/mcp/chat.rs index 290d5b6..d8b40a5 100644 --- a/own_mcp/src/mcp/chat.rs +++ b/own_mcp/src/mcp/chat.rs @@ -132,7 +132,7 @@ impl AgentChat { /// /// # Arguments /// - /// * `name`: must be in the format "mcp_server_name::tool_or_resource_name". + /// * `name`: must be in the format "mcp_server_name:tool_or_resource_name". /// /// returns: /// Ok(tuple) => tuple of the server name and the tool or resource name @@ -141,12 +141,12 @@ impl AgentChat { /// # Examples /// /// ```ignore - /// let result = AgentChat::parse_tool_name("example::get_foo".to_string()); + /// let result = AgentChat::parse_tool_name("example:get_foo".to_string()); /// assert_eq!(result.unwrap(), ("example".to_string(), "get_foo".to_string())) /// ``` fn parse_tool_name(name: String) -> Result<(String, String), ChatError> { let (mcp_server_name, tool_name) = name - .split_once("::") + .split_once(":") .ok_or(ChatError::FunctionParseError(name.clone()))?; Ok((mcp_server_name.to_string(), tool_name.to_string())) @@ -161,7 +161,7 @@ impl AgentChat { /// # Arguments /// /// * `mpc_server_data`: - /// * `name`: must be in the format "mcp_server_name::tool_or_resource_name". + /// * `name`: must be in the format "mcp_server_name:tool_or_resource_name". /// /// returns: Option<&RestrictedTool> pub fn get_tool( @@ -352,7 +352,7 @@ mod tests { assert_eq!(tools.len(), 1); assert_eq!(tools[0].permission, ToolPermission::Ask); - chat.set_permission("test::echo".to_string(), ToolPermission::Allowed).unwrap(); + chat.set_permission("test:echo".to_string(), ToolPermission::Allowed).unwrap(); let tools: Vec = chat.get_all_tools().cloned().collect(); assert_eq!(tools[0].permission, ToolPermission::Allowed); diff --git a/own_mcp/src/mcp/translation.rs b/own_mcp/src/mcp/translation.rs index 45aa128..4f0f4a1 100644 --- a/own_mcp/src/mcp/translation.rs +++ b/own_mcp/src/mcp/translation.rs @@ -32,7 +32,7 @@ fn tool_info_from_mcp_tool(mcp_tool: &rmcp::model::Tool, server_name: &String) - ToolInfo { tool_type: ToolType::Function, function: ToolFunctionInfo { - name: format!("{}::{}", server_name, mcp_tool.name), + name: format!("{}:{}", server_name, mcp_tool.name), description: mcp_tool .description .clone() @@ -70,7 +70,7 @@ fn tool_info_from_mcp_resource( ToolInfo { tool_type: ToolType::Function, function: ToolFunctionInfo { - name: format!("{}::get_{}", server_name, mcp_resource.name), + name: format!("{}:get_{}", server_name, mcp_resource.name), description: format!( "type: {} - {}", mcp_resource diff --git a/src/config.rs b/src/config.rs index dac2d99..b4df196 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2,6 +2,7 @@ use own_assist_common::config_loader::ConfigLoadingError; use std::collections::HashMap; use ollama_rs::headers::{HeaderMap, HeaderValue}; use ollama_rs::Ollama; +use own_mcp::audio::{AudioClient, AudioClientTrait}; use rmcp::model::Implementation; use serde::Deserialize; use url::Url; @@ -12,10 +13,18 @@ use own_assist_common::config_from_file; #[derive(Debug, Deserialize)] pub struct Config { ollama: OllamaConfig, + #[serde(rename = "audio-server")] + audio_server: AudioServerConfig, #[serde(rename = "mcp-servers")] mcp_servers: Vec } +#[derive(Debug, Deserialize)] +pub struct AudioServerConfig { + url: Url, + authorization: Option, +} + impl Config { pub fn from_file() -> Result { config_from_file("assist.toml") @@ -39,6 +48,15 @@ impl Config { ollama } + pub fn audio_client(&self) -> AudioClient { + let mut audio_client = AudioClient::new(self.audio_server.url.clone()); + if let Some(authorization) = &self.audio_server.authorization { + audio_client = audio_client.with_authorization(authorization.clone()); + } + + audio_client + } + pub async fn mcp_clients(&self) -> HashMap { let mut mcp_clients: HashMap = HashMap::new(); diff --git a/src/main.rs b/src/main.rs index 8367d7d..da5a5c8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ mod config; mod model; +use own_mcp::audio::{AudioClientTrait}; use own_mcp::mcp::chat::{PermissionAnswer, ToolPermission}; use crate::config::Config; use crate::model::create_model_from_config; @@ -18,6 +19,12 @@ async fn main() { std::process::exit(1); }).unwrap(); + let audio_client = config.audio_client(); + log::debug!("Audio server status {:?}", audio_client.status().await.inspect_err(|e|{ + log::error!("audio server error {}", e); + std::process::exit(1); + }).unwrap()); + let ollama = config.ollama_instance(); let mcp_clients = config.mcp_clients().await; @@ -29,7 +36,8 @@ async fn main() { std::process::exit(1); }).unwrap(); - let system_prompt = "Du bist ein Assistent, der per Sprache bedient wird. Du erhälst die Transkription. \ + let system_prompt = "Du bist ein Assistent, der per Sprache bedient wird. Du erhälst die Transkription \ + und dein Output wird per Sprache ausgegeben und sollte dementsprechend auch kurz sein. \ Wichtiger als deine Antworten sind deine Aktionen.\ Nutze bitte die tools, falls du sie brauchst um Informationen zu bekommen (z.B: über das aktuelle Datum oder den aktuellen Wochentag). \ Du bist in einem Agent Loop und kannst mehrere Tools hintereinander nutzen. \ @@ -44,10 +52,12 @@ async fn main() { log::info!("all tools: {:#?}", agent_chat.get_all_tools().collect::>()); - agent_chat.set_permission("mcp-playground-echo-server::echo".to_string(), ToolPermission::Allowed).unwrap(); + agent_chat.set_permission("datetime:get_weekday".to_string(), ToolPermission::Allowed).unwrap(); + agent_chat.set_permission("datetime:get_utc_datetime".to_string(), ToolPermission::Allowed).unwrap(); + agent_chat.set_permission("datetime:get_local_datetime".to_string(), ToolPermission::Allowed).unwrap(); + agent_chat.set_permission("datetime:get_week".to_string(), ToolPermission::Allowed).unwrap(); - let answer = agent_chat.message("Welche Tools kannst du benutzten?".to_string(), permission_callback).await; - let answer = agent_chat.message("Teste beide Server aus. Nutze bei fetch https://example.com/. Melde mir die Ergebnisse zurück.".to_string(), permission_callback).await; + let answer = agent_chat.message("In welcher Kalenderwoche befinden wir uns gerade?".to_string(), permission_callback).await; dbg!(answer); }