add logging config to audio server
add audio client change delimiter for mcp tools from "::" to ":" to save tokens
This commit is contained in:
milan
Generated
+302
@@ -11,6 +11,28 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "alsa"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "812947049edcd670a82cd5c73c3661d2e58468577ba8489de58e1a73c04cbd5d"
|
||||
dependencies = [
|
||||
"alsa-sys",
|
||||
"bitflags",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "alsa-sys"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ad7569085a265dd3f607ebecce7458eaab2132a84393534c95b18dcbc3f31e04"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "android_system_properties"
|
||||
version = "0.1.5"
|
||||
@@ -164,6 +186,15 @@ dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block2"
|
||||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5"
|
||||
dependencies = [
|
||||
"objc2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.20.2"
|
||||
@@ -282,6 +313,50 @@ version = "0.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
||||
|
||||
[[package]]
|
||||
name = "coreaudio-rs"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "16dd574a72a021b90c7656c474ea31d11a2f0366a8eff574186e761e0b9e3586"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"libc",
|
||||
"objc2-audio-toolbox",
|
||||
"objc2-core-audio",
|
||||
"objc2-core-audio-types",
|
||||
"objc2-core-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpal"
|
||||
version = "0.17.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d8942da362c0f0d895d7cac616263f2f9424edc5687364dfd1d25ef7eba506d7"
|
||||
dependencies = [
|
||||
"alsa",
|
||||
"coreaudio-rs",
|
||||
"dasp_sample",
|
||||
"jni",
|
||||
"js-sys",
|
||||
"libc",
|
||||
"mach2",
|
||||
"ndk",
|
||||
"ndk-context",
|
||||
"num-derive",
|
||||
"num-traits",
|
||||
"objc2",
|
||||
"objc2-audio-toolbox",
|
||||
"objc2-avf-audio",
|
||||
"objc2-core-audio",
|
||||
"objc2-core-audio-types",
|
||||
"objc2-core-foundation",
|
||||
"objc2-foundation",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
"windows",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
@@ -344,6 +419,12 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dasp_sample"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f"
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
@@ -354,6 +435,16 @@ dependencies = [
|
||||
"crypto-common",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dispatch2"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"objc2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "displaydoc"
|
||||
version = "0.2.5"
|
||||
@@ -658,6 +749,12 @@ version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "hound"
|
||||
version = "3.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f"
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "1.4.0"
|
||||
@@ -1080,6 +1177,15 @@ version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
|
||||
|
||||
[[package]]
|
||||
name = "mach2"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a1b95cd5421ec55b445b5ae102f5ea0e768de1f82bd3001e11f426c269c3aea"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.0"
|
||||
@@ -1092,6 +1198,16 @@ version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
|
||||
|
||||
[[package]]
|
||||
name = "mime_guess"
|
||||
version = "2.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
|
||||
dependencies = [
|
||||
"mime",
|
||||
"unicase",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "1.2.0"
|
||||
@@ -1120,6 +1236,35 @@ dependencies = [
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ndk"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"jni-sys 0.3.1",
|
||||
"log",
|
||||
"ndk-sys",
|
||||
"num_enum",
|
||||
"thiserror 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ndk-context"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b"
|
||||
|
||||
[[package]]
|
||||
name = "ndk-sys"
|
||||
version = "0.6.0+11769913"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee6cda3051665f1fb8d9e08fc35c96d5a244fb1be711a03b71118828afc9a873"
|
||||
dependencies = [
|
||||
"jni-sys 0.3.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nix"
|
||||
version = "0.31.2"
|
||||
@@ -1132,6 +1277,17 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-derive"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
@@ -1141,6 +1297,28 @@ dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_enum"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26"
|
||||
dependencies = [
|
||||
"num_enum_derive",
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_enum_derive"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8"
|
||||
dependencies = [
|
||||
"proc-macro-crate",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "oauth2"
|
||||
version = "5.0.0"
|
||||
@@ -1160,6 +1338,95 @@ dependencies = [
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f"
|
||||
dependencies = [
|
||||
"objc2-encode",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-audio-toolbox"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"libc",
|
||||
"objc2",
|
||||
"objc2-core-audio",
|
||||
"objc2-core-audio-types",
|
||||
"objc2-core-foundation",
|
||||
"objc2-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-avf-audio"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be"
|
||||
dependencies = [
|
||||
"objc2",
|
||||
"objc2-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-core-audio"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2"
|
||||
dependencies = [
|
||||
"dispatch2",
|
||||
"objc2",
|
||||
"objc2-core-audio-types",
|
||||
"objc2-core-foundation",
|
||||
"objc2-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-core-audio-types"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"objc2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-core-foundation"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"block2",
|
||||
"dispatch2",
|
||||
"libc",
|
||||
"objc2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-encode"
|
||||
version = "4.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33"
|
||||
|
||||
[[package]]
|
||||
name = "objc2-foundation"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"block2",
|
||||
"libc",
|
||||
"objc2",
|
||||
"objc2-core-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ollama-rs"
|
||||
version = "0.3.4"
|
||||
@@ -1274,7 +1541,10 @@ dependencies = [
|
||||
name = "own_mcp"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"cpal",
|
||||
"env_logger",
|
||||
"hound",
|
||||
"log",
|
||||
"ollama-rs",
|
||||
"rand 0.10.0",
|
||||
@@ -1353,6 +1623,15 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro-crate"
|
||||
version = "3.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"
|
||||
dependencies = [
|
||||
"toml_edit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.106"
|
||||
@@ -1635,6 +1914,7 @@ dependencies = [
|
||||
"js-sys",
|
||||
"log",
|
||||
"mime",
|
||||
"mime_guess",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"quinn",
|
||||
@@ -1643,6 +1923,7 @@ dependencies = [
|
||||
"rustls-platform-verifier",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
@@ -2290,6 +2571,18 @@ dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_edit"
|
||||
version = "0.25.11+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"toml_datetime",
|
||||
"toml_parser",
|
||||
"winnow",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_parser"
|
||||
version = "1.1.2+spec-1.1.0"
|
||||
@@ -2393,6 +2686,12 @@ version = "1.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
|
||||
|
||||
[[package]]
|
||||
name = "unicase"
|
||||
version = "2.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
@@ -2979,6 +3278,9 @@ name = "winnow"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen"
|
||||
|
||||
@@ -16,4 +16,4 @@ RUN apt -y install ffmpeg
|
||||
WORKDIR /app
|
||||
|
||||
EXPOSE 8000
|
||||
CMD ["uvicorn", "audio_server:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
CMD ["uvicorn", "audio_server:app", "--host", "0.0.0.0", "--port", "8000", "--log-config", "logging_config.yaml"]
|
||||
@@ -0,0 +1,42 @@
|
||||
version: 1
|
||||
disable_existing_logger: false
|
||||
formatters:
|
||||
main:
|
||||
format: "\x1b[1;32m%(name)s \x1b[0m- %(levelname)s - %(message)s"
|
||||
default:
|
||||
format: "\x1b[1;93m%(name)s \x1b[0m- %(levelname)s - %(message)s"
|
||||
use_colors: yes
|
||||
handlers:
|
||||
main:
|
||||
formatter: main
|
||||
class: logging.StreamHandler
|
||||
stream: ext://sys.stdout
|
||||
default:
|
||||
formatter: default
|
||||
class: logging.StreamHandler
|
||||
stream: ext://sys.stdout
|
||||
access:
|
||||
formatter: default
|
||||
class: logging.StreamHandler
|
||||
stream: ext://sys.stdout
|
||||
loggers:
|
||||
audio_server:
|
||||
level: INFO
|
||||
handlers:
|
||||
- main
|
||||
propagate: no
|
||||
uvicorn.error:
|
||||
level: INFO
|
||||
handlers:
|
||||
- default
|
||||
propagate: no
|
||||
uvicorn.access:
|
||||
level: INFO
|
||||
handlers:
|
||||
- default
|
||||
propagate: no
|
||||
root:
|
||||
level: INFO
|
||||
handlers:
|
||||
- default
|
||||
propagate: no
|
||||
@@ -5,4 +5,5 @@ uvicorn~=0.44.0
|
||||
starlette~=1.0.0
|
||||
python-multipart~=0.0.24
|
||||
aiofiles~=25.1.0
|
||||
piper-tts~=1.4.2
|
||||
piper-tts~=1.4.2
|
||||
PyYAML~=6.0.3
|
||||
@@ -5,13 +5,14 @@ from wave import Wave_write
|
||||
from pathlib import Path
|
||||
|
||||
import aiofiles
|
||||
from piper import PiperVoice, SynthesisConfig
|
||||
from piper import PiperVoice
|
||||
from piper.download_voices import download_voice
|
||||
import psutil
|
||||
from fastapi import FastAPI, UploadFile
|
||||
from fastapi import FastAPI, UploadFile, HTTPException
|
||||
import whisper
|
||||
from pydantic import BaseModel
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from .models import TranscriptionResponse, VoiceRequest, StatusResponse
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
@@ -28,26 +29,28 @@ download_voice(piper_model_name, piper_path)
|
||||
temporary_audio_path = Path("temporary_audio")
|
||||
temporary_audio_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger = logging.getLogger("audio_server")
|
||||
|
||||
|
||||
@app.get("/")
|
||||
async def index():
|
||||
return {
|
||||
"memory_usage": psutil.Process(os.getpid()).memory_info().rss,
|
||||
"piper_model": piper_model_name,
|
||||
"whisper_model": whisper_model_name,
|
||||
}
|
||||
async def index() -> StatusResponse:
|
||||
return StatusResponse(memory_usage=psutil.Process(os.getpid()).memory_info().rss, piper_model=piper_model_name,
|
||||
whisper_model=whisper_model_name)
|
||||
|
||||
|
||||
@app.get("/transcribe")
|
||||
async def transcribe(audio_file: UploadFile):
|
||||
async def transcribe(audio_file: UploadFile) -> TranscriptionResponse:
|
||||
file_extensions = {
|
||||
"audio/wav": "wav",
|
||||
"audio/wave": "wav",
|
||||
"audio/mpeg": "mp3",
|
||||
}
|
||||
|
||||
if audio_file.content_type not in file_extensions.keys():
|
||||
logger.error("%s is not supported", audio_file.content_type)
|
||||
return JSONResponse({"detail": "wrong file type"}, status_code=400)
|
||||
print("CONTENT TYPE:")
|
||||
print(audio_file.content_type)
|
||||
raise HTTPException(status_code=400, detail="file type not supported")
|
||||
|
||||
file_path = temporary_audio_path / f"{uuid.uuid4()}.{file_extensions[audio_file.content_type]}"
|
||||
|
||||
@@ -58,14 +61,11 @@ async def transcribe(audio_file: UploadFile):
|
||||
# noinspection PyArgumentList
|
||||
transcription = whisper_model.transcribe(file_path.as_posix())
|
||||
|
||||
return JSONResponse(transcription)
|
||||
return TranscriptionResponse.model_validate(transcription)
|
||||
|
||||
class VoiceRequest(BaseModel):
|
||||
text: str
|
||||
config: SynthesisConfig | None
|
||||
|
||||
@app.get("/tts")
|
||||
async def tts(voice_request: VoiceRequest):
|
||||
async def tts(voice_request: VoiceRequest) -> FileResponse:
|
||||
voice = PiperVoice.load(piper_path / f"{piper_model_name}.onnx")
|
||||
|
||||
audio_file_path = temporary_audio_path / f"{uuid.uuid4()}.wav"
|
||||
@@ -73,4 +73,4 @@ async def tts(voice_request: VoiceRequest):
|
||||
with Wave_write(audio_file_path.as_posix()) as writer:
|
||||
voice.synthesize_wav(voice_request.text, writer, syn_config=voice_request.config)
|
||||
|
||||
return FileResponse(audio_file_path, media_type="audio/wav")
|
||||
return FileResponse(audio_file_path, media_type="audio/wav")
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
from piper import SynthesisConfig
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class StatusResponse(BaseModel):
|
||||
memory_usage: int
|
||||
piper_model: str
|
||||
whisper_model: str
|
||||
|
||||
|
||||
# noinspection SpellCheckingInspection
|
||||
class TranscriptionSegment(BaseModel):
|
||||
id: int
|
||||
seek: int
|
||||
start: float
|
||||
end: float
|
||||
text: str
|
||||
tokens: list[int]
|
||||
temperature: float
|
||||
avg_logprob: float
|
||||
compression_ratio: float
|
||||
no_speech_prob: float
|
||||
|
||||
class TranscriptionResponse(BaseModel):
|
||||
text: str
|
||||
segments: list[TranscriptionSegment]
|
||||
language: str
|
||||
|
||||
|
||||
class VoiceRequest(BaseModel):
|
||||
text: str
|
||||
config: SynthesisConfig | None
|
||||
@@ -67,7 +67,7 @@ async fn main() {
|
||||
|
||||
let router = mcp_router(&config.servers).route("/", axum::routing::get(|| async { Json(config.servers) }));
|
||||
|
||||
let bind_address = config.bind_address.unwrap_or(url::Url::parse("localhost:8000").unwrap());
|
||||
let bind_address = config.bind_address.unwrap_or(url::Url::parse("http://localhost:8000").unwrap());
|
||||
println!("binding address at {bind_address}");
|
||||
|
||||
let tcp_listener = tokio::net::TcpListener::bind(bind_address_format(bind_address)).await.inspect_err(|e|{
|
||||
|
||||
+5
-2
@@ -5,7 +5,7 @@ edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
ollama-rs = {version = "0.3.4", features = ["macros", "headers"]}
|
||||
reqwest = "0.13.2"
|
||||
reqwest = { version = "0.13.2", features = ["stream", "multipart", "form"] }
|
||||
tokio = { version = "1.50.0", features = ["rt", "rt-multi-thread", "macros"] }
|
||||
rmcp = {version="1.3.0", features = ["transport-streamable-http-client-reqwest", "reqwest", "client", "auth", "transport-child-process"]}
|
||||
log = {version = "0.4.29"}
|
||||
@@ -13,4 +13,7 @@ env_logger = "0.11.10"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
thiserror = "2.0.17"
|
||||
url = "2.5.8"
|
||||
rand = "0.10.0"
|
||||
rand = "0.10.0"
|
||||
cpal = "0.17.3"
|
||||
hound = "3.5.1"
|
||||
bytes = "1.11.1"
|
||||
@@ -0,0 +1,71 @@
|
||||
use std::io::ErrorKind;
|
||||
use std::path::Path;
|
||||
use reqwest::Client;
|
||||
use reqwest::multipart::Form;
|
||||
use thiserror::Error;
|
||||
use crate::audio::models::{AudioServerStatus, TranscriptionResponse, VoiceRequest};
|
||||
use url::Url;
|
||||
|
||||
pub struct AudioClient{
|
||||
client: Client,
|
||||
authorization: Option<String>,
|
||||
base_url: Url,
|
||||
}
|
||||
|
||||
pub trait AudioClientTrait {
|
||||
fn new(base_url: Url) -> Self;
|
||||
fn from_client(base_url: Url, client: Client) -> Self;
|
||||
|
||||
fn with_authorization(self, authorization: String) -> Self;
|
||||
|
||||
fn status(&self) -> impl Future<Output = Result<AudioServerStatus, AudioError>>;
|
||||
|
||||
fn transcribe(&self, audio_file_path: impl AsRef<Path>) -> impl Future<Output=AudioResult<TranscriptionResponse>>;
|
||||
|
||||
fn tts(&self, voice_request: VoiceRequest) -> impl Future<Output=AudioResult<bytes::Bytes>>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum AudioError {
|
||||
#[error(transparent)]
|
||||
RequestError(#[from] reqwest::Error),
|
||||
#[error(transparent)]
|
||||
UrlParseError(#[from] url::ParseError),
|
||||
#[error(transparent)]
|
||||
IOError(#[from] std::io::Error),
|
||||
}
|
||||
|
||||
type AudioResult<T> = Result<T, AudioError>;
|
||||
|
||||
impl AudioClientTrait for AudioClient {
|
||||
fn new(base_url: Url) -> Self {
|
||||
AudioClient { client: Client::new(), authorization: None, base_url }
|
||||
}
|
||||
|
||||
fn from_client(base_url: Url, client: Client) -> Self {
|
||||
AudioClient { client, authorization: None, base_url }
|
||||
}
|
||||
|
||||
fn with_authorization(mut self, authorization: String) -> Self {
|
||||
self.authorization = Some(authorization);
|
||||
self
|
||||
}
|
||||
|
||||
async fn status(&self) -> AudioResult<AudioServerStatus> {
|
||||
let response = self.client.get(self.base_url.clone()).send().await?.error_for_status()?;
|
||||
Ok(response.json().await?)
|
||||
}
|
||||
|
||||
async fn transcribe(&self, audio_file_path: impl AsRef<Path>) -> AudioResult<TranscriptionResponse> {
|
||||
let form = Form::new().file("audio_file", audio_file_path).await?;
|
||||
|
||||
let response = self.client.get(self.base_url.join("transcribe")?).multipart(form).send().await?;
|
||||
Ok(response.json().await?)
|
||||
}
|
||||
|
||||
async fn tts(&self, voice_request: VoiceRequest) -> AudioResult<bytes::Bytes> {
|
||||
let response = self.client.get(self.base_url.join("tts")?).json(&voice_request).send().await?.error_for_status()?;
|
||||
|
||||
Ok(response.bytes().await?)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
pub mod recording;
|
||||
pub mod client;
|
||||
pub mod models;
|
||||
pub use client::{AudioClient, AudioClientTrait};
|
||||
@@ -0,0 +1,60 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct AudioServerStatus {
|
||||
#[serde(rename = "memory_usage")]
|
||||
pub memory_usage_bytes: usize,
|
||||
pub piper_model: String,
|
||||
pub whisper_model: String,
|
||||
}
|
||||
|
||||
//noinspection SpellCheckingInspection
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct TranscriptionSegment {
|
||||
pub id: u32,
|
||||
pub seek: u32,
|
||||
pub start: f32,
|
||||
pub end: f32,
|
||||
pub text: String,
|
||||
pub tokens: Vec<u32>,
|
||||
pub temperature: f32,
|
||||
pub avg_logprob: f32,
|
||||
pub compression_ratio: f32,
|
||||
pub no_speech_prob: f32
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct TranscriptionResponse {
|
||||
pub text: String,
|
||||
pub segments: Vec<TranscriptionSegment>,
|
||||
pub language: String
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct SynthesisConfig {
|
||||
pub speaker_id: Option<u32>,
|
||||
pub length_scale: Option<u32>,
|
||||
pub noise_scale: Option<u32>,
|
||||
pub noise_w_scale: Option<u32>,
|
||||
pub normalize_audio: bool,
|
||||
pub volume: f32,
|
||||
}
|
||||
|
||||
impl Default for SynthesisConfig {
|
||||
fn default() -> Self {
|
||||
SynthesisConfig {
|
||||
speaker_id: None,
|
||||
length_scale: None,
|
||||
noise_scale: None,
|
||||
noise_w_scale: None,
|
||||
normalize_audio: true,
|
||||
volume: 1.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct VoiceRequest {
|
||||
pub text: String,
|
||||
pub config: Option<SynthesisConfig>,
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
use cpal::traits::{DeviceTrait, StreamTrait};
|
||||
use cpal::{BuildStreamError, Device, SampleFormat, SizedSample, Stream, SupportedStreamConfig};
|
||||
use std::fmt::Debug;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::{path, thread};
|
||||
use std::thread::JoinHandle;
|
||||
use std::time::Duration;
|
||||
use hound::WavSpec;
|
||||
use log::{debug, error, info, trace};
|
||||
use thiserror::__private18::AsDisplay;
|
||||
use thiserror::Error;
|
||||
|
||||
fn generic_sample_to_i16(sample: impl SizedSample + Into<f64>, format: SampleFormat) -> i16 {
|
||||
match format {
|
||||
SampleFormat::I8 => (sample.into() * 2.0) as i16,
|
||||
SampleFormat::I16 => sample.into() as i16,
|
||||
SampleFormat::I32 => (sample.into() / 2.0) as i16,
|
||||
SampleFormat::F32 => (sample.into() * (i16::MAX as f64)) as i16,
|
||||
_ => panic!("Unsupported sample format {:?}", format),
|
||||
}
|
||||
}
|
||||
|
||||
pub type SampleArc = Arc<Mutex<Vec<i16>>>;
|
||||
|
||||
fn stream_callback<T: SizedSample + Debug + Into<f64>>(
|
||||
input: &[T],
|
||||
samples: SampleArc,
|
||||
sample_format: SampleFormat,
|
||||
) {
|
||||
if let Ok(mut guard) = samples.lock() {
|
||||
for &sample in input {
|
||||
guard.push(generic_sample_to_i16(sample, sample_format));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_stream<T: SizedSample + Debug + hound::Sample + Send + Into<f64>>(
|
||||
mic: &Device,
|
||||
device_config: SupportedStreamConfig,
|
||||
samples: SampleArc,
|
||||
sample_format: SampleFormat,
|
||||
) -> Result<Stream, BuildStreamError> {
|
||||
mic.build_input_stream(
|
||||
&device_config.into(),
|
||||
move |input: &[T], _info| stream_callback::<T>(input, samples.clone(), sample_format),
|
||||
|e| {
|
||||
error!("a stream error occurred while trying to record: {:?}", e);
|
||||
},
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct RecordingHandler {
|
||||
pub samples: SampleArc,
|
||||
pub spec: WavSpec,
|
||||
thread_handle: JoinHandle<Result<(), RecordingError>>,
|
||||
should_stop: Arc<Mutex<bool>>,
|
||||
}
|
||||
|
||||
impl RecordingHandler {
|
||||
pub fn stop_recording(self) -> Result<(), RecordingError> {
|
||||
info!("Stopping recording");
|
||||
*self.should_stop.lock().unwrap() = true;
|
||||
self.thread_handle.join().unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum RecordingError {
|
||||
#[error("unsupported sample format")]
|
||||
UnsupportedSampleFormat(SampleFormat),
|
||||
#[error(transparent)]
|
||||
PlayStreamError(#[from] cpal::PlayStreamError),
|
||||
#[error(transparent)]
|
||||
BuildStreamError(#[from] BuildStreamError),
|
||||
#[error("thread poisoned")]
|
||||
ThreadPoison,
|
||||
#[error("recording was already stopped")]
|
||||
RecordingAlreadyStopped,
|
||||
}
|
||||
|
||||
fn start_recording_blocking_with_parameters(
|
||||
mic: &Device,
|
||||
device_config: &SupportedStreamConfig,
|
||||
sample_format: SampleFormat,
|
||||
samples: SampleArc,
|
||||
should_stop: Arc<Mutex<bool>>,
|
||||
) -> Result<(), RecordingError> {
|
||||
let stream = match sample_format {
|
||||
SampleFormat::I8 => {
|
||||
build_stream::<i8>(mic, device_config.clone(), samples.clone(), sample_format)?
|
||||
}
|
||||
SampleFormat::I16 => {
|
||||
build_stream::<i16>(mic, device_config.clone(), samples.clone(), sample_format)?
|
||||
}
|
||||
SampleFormat::I32 => {
|
||||
build_stream::<i32>(mic, device_config.clone(), samples.clone(), sample_format)?
|
||||
}
|
||||
SampleFormat::F32 => {
|
||||
build_stream::<f32>(mic, device_config.clone(), samples.clone(), sample_format)?
|
||||
}
|
||||
sample_format => {
|
||||
return Err(RecordingError::UnsupportedSampleFormat(sample_format));
|
||||
}
|
||||
};
|
||||
|
||||
stream.play()?;
|
||||
|
||||
loop {
|
||||
thread::sleep(Duration::from_millis(100));
|
||||
if let Ok(guard) = should_stop.lock() {
|
||||
if *guard {
|
||||
debug!("Stopping recording gracefully");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
drop(stream);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn start(
|
||||
microphone: Device,
|
||||
device_config: SupportedStreamConfig,
|
||||
) -> Arc<Mutex<Option<RecordingHandler>>> {
|
||||
let sample_format = device_config.sample_format();
|
||||
|
||||
let samples: SampleArc = Arc::new(Mutex::new(Vec::new()));
|
||||
let recording_should_stop = Arc::new(Mutex::new(false));
|
||||
|
||||
info!("started recording");
|
||||
debug!("sample format: {:?}", sample_format);
|
||||
|
||||
let thread_samples = samples.clone();
|
||||
let thread_recording_stop = recording_should_stop.clone();
|
||||
let thread_device_config = device_config.clone();
|
||||
let record_thread_handle = thread::spawn(move ||
|
||||
start_recording_blocking_with_parameters(
|
||||
µphone,
|
||||
&thread_device_config,
|
||||
sample_format,
|
||||
thread_samples,
|
||||
thread_recording_stop,
|
||||
)
|
||||
);
|
||||
|
||||
let spec = WavSpec {
|
||||
channels: device_config.channels(),
|
||||
sample_rate: device_config.sample_rate(),
|
||||
bits_per_sample: 16,
|
||||
sample_format: hound::SampleFormat::Int,
|
||||
};
|
||||
|
||||
debug!("device spec: {:?}", spec);
|
||||
|
||||
Arc::new(Mutex::new(Some(RecordingHandler {
|
||||
spec,
|
||||
samples,
|
||||
thread_handle: record_thread_handle,
|
||||
should_stop: recording_should_stop,
|
||||
})))
|
||||
}
|
||||
|
||||
pub fn stop_and_take_data(handler: Arc<Mutex<Option<RecordingHandler>>>) -> Result<(Vec<i16>, WavSpec), RecordingError> {
|
||||
let handler = handler.lock().map_err(|_|RecordingError::ThreadPoison)?.take().ok_or(RecordingError::RecordingAlreadyStopped)?;
|
||||
|
||||
let samples = handler.samples.clone();
|
||||
let spec = handler.spec.clone();
|
||||
|
||||
handler.stop_recording()?;
|
||||
|
||||
Ok((samples.lock().map_err(|_|RecordingError::ThreadPoison)?.to_vec(), spec))
|
||||
}
|
||||
|
||||
pub fn samples_to_wav(
|
||||
samples: impl IntoIterator<Item = i16>,
|
||||
spec: &WavSpec,
|
||||
filepath: impl AsRef<path::Path>
|
||||
) -> Result<Vec<u8>, hound::Error> {
|
||||
let wav_bytes = std::io::Cursor::new(Vec::<u8>::new());
|
||||
|
||||
trace!("creating a wav file writer to {}", filepath.as_ref().display());
|
||||
let mut file_writer = hound::WavWriter::create(&filepath, *spec)?;
|
||||
|
||||
trace!("writing samples...");
|
||||
for sample in samples {
|
||||
file_writer.write_sample(sample)?;
|
||||
}
|
||||
|
||||
trace!("finalizing for file {}", filepath.as_ref().display());
|
||||
file_writer.finalize()?;
|
||||
|
||||
Ok(wav_bytes.into_inner())
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
pub mod mcp;
|
||||
pub mod audio;
|
||||
|
||||
pub use mcp::chat::AgentChat;
|
||||
@@ -132,7 +132,7 @@ impl AgentChat {
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `name`: must be in the format "mcp_server_name::tool_or_resource_name".
|
||||
/// * `name`: must be in the format "mcp_server_name:tool_or_resource_name".
|
||||
///
|
||||
/// returns:
|
||||
/// Ok(tuple) => tuple of the server name and the tool or resource name
|
||||
@@ -141,12 +141,12 @@ impl AgentChat {
|
||||
/// # Examples
|
||||
///
|
||||
/// ```ignore
|
||||
/// let result = AgentChat::parse_tool_name("example::get_foo".to_string());
|
||||
/// let result = AgentChat::parse_tool_name("example:get_foo".to_string());
|
||||
/// assert_eq!(result.unwrap(), ("example".to_string(), "get_foo".to_string()))
|
||||
/// ```
|
||||
fn parse_tool_name(name: String) -> Result<(String, String), ChatError> {
|
||||
let (mcp_server_name, tool_name) = name
|
||||
.split_once("::")
|
||||
.split_once(":")
|
||||
.ok_or(ChatError::FunctionParseError(name.clone()))?;
|
||||
|
||||
Ok((mcp_server_name.to_string(), tool_name.to_string()))
|
||||
@@ -161,7 +161,7 @@ impl AgentChat {
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `mpc_server_data`:
|
||||
/// * `name`: must be in the format "mcp_server_name::tool_or_resource_name".
|
||||
/// * `name`: must be in the format "mcp_server_name:tool_or_resource_name".
|
||||
///
|
||||
/// returns: Option<&RestrictedTool>
|
||||
pub fn get_tool(
|
||||
@@ -352,7 +352,7 @@ mod tests {
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
assert_eq!(tools[0].permission, ToolPermission::Ask);
|
||||
chat.set_permission("test::echo".to_string(), ToolPermission::Allowed).unwrap();
|
||||
chat.set_permission("test:echo".to_string(), ToolPermission::Allowed).unwrap();
|
||||
|
||||
let tools: Vec<RestrictedTool> = chat.get_all_tools().cloned().collect();
|
||||
assert_eq!(tools[0].permission, ToolPermission::Allowed);
|
||||
|
||||
@@ -32,7 +32,7 @@ fn tool_info_from_mcp_tool(mcp_tool: &rmcp::model::Tool, server_name: &String) -
|
||||
ToolInfo {
|
||||
tool_type: ToolType::Function,
|
||||
function: ToolFunctionInfo {
|
||||
name: format!("{}::{}", server_name, mcp_tool.name),
|
||||
name: format!("{}:{}", server_name, mcp_tool.name),
|
||||
description: mcp_tool
|
||||
.description
|
||||
.clone()
|
||||
@@ -70,7 +70,7 @@ fn tool_info_from_mcp_resource(
|
||||
ToolInfo {
|
||||
tool_type: ToolType::Function,
|
||||
function: ToolFunctionInfo {
|
||||
name: format!("{}::get_{}", server_name, mcp_resource.name),
|
||||
name: format!("{}:get_{}", server_name, mcp_resource.name),
|
||||
description: format!(
|
||||
"type: {} - {}",
|
||||
mcp_resource
|
||||
|
||||
@@ -2,6 +2,7 @@ use own_assist_common::config_loader::ConfigLoadingError;
|
||||
use std::collections::HashMap;
|
||||
use ollama_rs::headers::{HeaderMap, HeaderValue};
|
||||
use ollama_rs::Ollama;
|
||||
use own_mcp::audio::{AudioClient, AudioClientTrait};
|
||||
use rmcp::model::Implementation;
|
||||
use serde::Deserialize;
|
||||
use url::Url;
|
||||
@@ -12,10 +13,18 @@ use own_assist_common::config_from_file;
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct Config {
|
||||
ollama: OllamaConfig,
|
||||
#[serde(rename = "audio-server")]
|
||||
audio_server: AudioServerConfig,
|
||||
#[serde(rename = "mcp-servers")]
|
||||
mcp_servers: Vec<MCPServerConfig>
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct AudioServerConfig {
|
||||
url: Url,
|
||||
authorization: Option<String>,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn from_file() -> Result<Self, ConfigLoadingError> {
|
||||
config_from_file("assist.toml")
|
||||
@@ -39,6 +48,15 @@ impl Config {
|
||||
ollama
|
||||
}
|
||||
|
||||
pub fn audio_client(&self) -> AudioClient {
|
||||
let mut audio_client = AudioClient::new(self.audio_server.url.clone());
|
||||
if let Some(authorization) = &self.audio_server.authorization {
|
||||
audio_client = audio_client.with_authorization(authorization.clone());
|
||||
}
|
||||
|
||||
audio_client
|
||||
}
|
||||
|
||||
pub async fn mcp_clients(&self) -> HashMap<String, MCPClient> {
|
||||
let mut mcp_clients: HashMap<String, MCPClient> = HashMap::new();
|
||||
|
||||
|
||||
+14
-4
@@ -1,6 +1,7 @@
|
||||
mod config;
|
||||
mod model;
|
||||
|
||||
use own_mcp::audio::{AudioClientTrait};
|
||||
use own_mcp::mcp::chat::{PermissionAnswer, ToolPermission};
|
||||
use crate::config::Config;
|
||||
use crate::model::create_model_from_config;
|
||||
@@ -18,6 +19,12 @@ async fn main() {
|
||||
std::process::exit(1);
|
||||
}).unwrap();
|
||||
|
||||
let audio_client = config.audio_client();
|
||||
log::debug!("Audio server status {:?}", audio_client.status().await.inspect_err(|e|{
|
||||
log::error!("audio server error {}", e);
|
||||
std::process::exit(1);
|
||||
}).unwrap());
|
||||
|
||||
let ollama = config.ollama_instance();
|
||||
|
||||
let mcp_clients = config.mcp_clients().await;
|
||||
@@ -29,7 +36,8 @@ async fn main() {
|
||||
std::process::exit(1);
|
||||
}).unwrap();
|
||||
|
||||
let system_prompt = "Du bist ein Assistent, der per Sprache bedient wird. Du erhälst die Transkription. \
|
||||
let system_prompt = "Du bist ein Assistent, der per Sprache bedient wird. Du erhälst die Transkription \
|
||||
und dein Output wird per Sprache ausgegeben und sollte dementsprechend auch kurz sein. \
|
||||
Wichtiger als deine Antworten sind deine Aktionen.\
|
||||
Nutze bitte die tools, falls du sie brauchst um Informationen zu bekommen (z.B: über das aktuelle Datum oder den aktuellen Wochentag). \
|
||||
Du bist in einem Agent Loop und kannst mehrere Tools hintereinander nutzen. \
|
||||
@@ -44,10 +52,12 @@ async fn main() {
|
||||
|
||||
log::info!("all tools: {:#?}", agent_chat.get_all_tools().collect::<Vec<_>>());
|
||||
|
||||
agent_chat.set_permission("mcp-playground-echo-server::echo".to_string(), ToolPermission::Allowed).unwrap();
|
||||
agent_chat.set_permission("datetime:get_weekday".to_string(), ToolPermission::Allowed).unwrap();
|
||||
agent_chat.set_permission("datetime:get_utc_datetime".to_string(), ToolPermission::Allowed).unwrap();
|
||||
agent_chat.set_permission("datetime:get_local_datetime".to_string(), ToolPermission::Allowed).unwrap();
|
||||
agent_chat.set_permission("datetime:get_week".to_string(), ToolPermission::Allowed).unwrap();
|
||||
|
||||
let answer = agent_chat.message("Welche Tools kannst du benutzten?".to_string(), permission_callback).await;
|
||||
let answer = agent_chat.message("Teste beide Server aus. Nutze bei fetch https://example.com/. Melde mir die Ergebnisse zurück.".to_string(), permission_callback).await;
|
||||
let answer = agent_chat.message("In welcher Kalenderwoche befinden wir uns gerade?".to_string(), permission_callback).await;
|
||||
|
||||
dbg!(answer);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user