From ff27dc5c7eed4a1ace442a46c0338ab03cb7fd0e Mon Sep 17 00:00:00 2001 From: Liam Pettigrew Date: Wed, 4 Feb 2026 15:44:25 +1100 Subject: [PATCH] V1.0: 2026-02-04 --- .gitignore | 3 ++ app.py | 5 +- core/__init__.py | 7 ++- core/asr-tiny.py | 64 ++++++++++++++++++++++++ core/asr.py | 100 ++++++++++++++++++++++++-------------- core/assistant.py | 14 +++--- core/slm.py | 4 +- core/tts.py | 11 ++++- data/config.example.yml | 4 ++ launch.sh | 43 +++++++++------- pyproject.toml | 8 +-- readme.md | 2 +- requirements.txt | 4 +- searxng_data/settings.yml | 2 +- 14 files changed, 193 insertions(+), 78 deletions(-) create mode 100644 core/asr-tiny.py mode change 100644 => 100755 launch.sh diff --git a/.gitignore b/.gitignore index 778b665..ac21808 100755 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ parts/ sdist/ var/ wheels/ +media/ *.egg-info/ .installed.cfg *.egg @@ -33,6 +34,7 @@ wheels/ htmlcov/ .tox/ .nox/ +archives/ # ============================================================================= # IDE and Editors @@ -87,6 +89,7 @@ Thumbs.db # Old/Archive Folders # ============================================================================= 0ld/ +dev/ # ============================================================================= # Keep Example Configs (negation patterns) diff --git a/app.py b/app.py index cb1346e..a6f6b16 100644 --- a/app.py +++ b/app.py @@ -38,12 +38,11 @@ from core.assistant import Assistant # Configuration WAKEWORD = config['general']['wakeword'] -SLM_MODEL = "./data/models/Qwen3-4B-Instruct-2507-Q4_K_M.gguf" - +USE_AI = config['general']['use_ai'] def main(): """Main entry point for the voice assistant.""" - assistant = Assistant(wakeword=WAKEWORD, slm_model_path=SLM_MODEL) + assistant = Assistant(wakeword=WAKEWORD, use_ai=USE_AI) assistant.run() diff --git a/core/__init__.py b/core/__init__.py index b8e7ecc..e836fbc 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -15,8 +15,8 @@ from .audio import ( SAMPLE_RATE, SILENCE_THRESHOLD, ) -from .asr import load_moonshine, stream_generator -from .tts import speak_stream, kpipeline +from .asr import load_asr_model, stream_generator +from .tts import speak_stream from .slm import load_slm, generate_slm from .assistant import Assistant @@ -27,11 +27,10 @@ __all__ = [ "SAMPLE_RATE", "SILENCE_THRESHOLD", # ASR - "load_moonshine", + "load_asr_model", "stream_generator", # TTS "speak_stream", - "kpipeline", # SLM "load_slm", "generate_slm", diff --git a/core/asr-tiny.py b/core/asr-tiny.py new file mode 100644 index 0000000..03fcacb --- /dev/null +++ b/core/asr-tiny.py @@ -0,0 +1,64 @@ +""" +Automatic Speech Recognition module using Moonshine ASR. + +Handles loading and running the Moonshine speech recognition model. +""" + +import logging +from typing import Generator + +import torch +from transformers import AutoProcessor, MoonshineForConditionalGeneration, pipeline + +logger = logging.getLogger(__name__) + +# Model configuration +ASR_MODEL_NAME = "UsefulSensors/moonshine-tiny" + +# Device configuration +DEVICE = "cuda" if torch.cuda.is_available() else "cpu" +DTYPE = torch.float16 if DEVICE == "cuda" else torch.float32 + + +def load_asr_model(): + """ + Load the Moonshine ASR model and create a pipeline. + + Returns: + A Hugging Face pipeline configured for automatic speech recognition + """ + logger.info(f"Loading {ASR_MODEL_NAME} on {DEVICE}...") + + processor = AutoProcessor.from_pretrained(ASR_MODEL_NAME) + asr_model = MoonshineForConditionalGeneration.from_pretrained(ASR_MODEL_NAME).to( + device=DEVICE, + dtype=DTYPE, + ) + + asr_pipe = pipeline( + task="automatic-speech-recognition", + model=asr_model, + tokenizer=processor.tokenizer, + feature_extractor=processor.feature_extractor, + device=-1 if DEVICE == "cuda" else 0, + dtype=DTYPE, + ) + + return asr_pipe + + +def stream_generator(queue) -> Generator: + """ + Generator that yields audio from a queue for the ASR pipeline. + + Args: + queue: Queue containing audio numpy arrays. None signals stop. + + Yields: + Audio numpy arrays until None is received + """ + while True: + audio = queue.get() + if audio is None: + break + yield audio diff --git a/core/asr.py b/core/asr.py index 7ffe09e..8809347 100644 --- a/core/asr.py +++ b/core/asr.py @@ -1,61 +1,89 @@ -""" -Automatic Speech Recognition module using Moonshine ASR. - -Handles loading and running the Moonshine speech recognition model. -""" - import logging -from typing import Generator - import torch -from transformers import AutoProcessor, MoonshineForConditionalGeneration, pipeline +import numpy as np +from typing import Generator, Optional, Union +from qwen_asr import Qwen3ASRModel logger = logging.getLogger(__name__) # Model configuration -ASR_MODEL_NAME = "UsefulSensors/moonshine-tiny" +ASR_MODEL_NAME = "Qwen/Qwen3-ASR-0.6B" +SAMPLE_RATE = 16000 # Qwen3-ASR standard sample rate # Device configuration DEVICE = "cuda" if torch.cuda.is_available() else "cpu" -DTYPE = torch.float16 if DEVICE == "cuda" else torch.float32 +DTYPE = torch.bfloat16 if DEVICE == "cuda" else torch.float32 -def load_moonshine(): +class QwenASRPipelineWrapper: """ - Load the Moonshine ASR model and create a pipeline. - - Returns: - A Hugging Face pipeline configured for automatic speech recognition + Wrapper for Qwen3-ASR to mimic basic HF pipeline behavior for streaming. """ + def __init__(self, model): + self.model = model + + def __call__(self, audio_input: Union[np.ndarray, Generator], batch_size: int = 1, generate_kwargs: Optional[dict] = None, **kwargs): + logger.info(f"[Batch size {batch_size}] and {generate_kwargs} are not being used") + + if isinstance(audio_input, Generator): + for chunk in audio_input: + if chunk is not None: + if isinstance(chunk, np.ndarray): + audio_tuple = (chunk, SAMPLE_RATE) + elif isinstance(chunk, torch.Tensor): + audio_tuple = (chunk.cpu().numpy(), SAMPLE_RATE) + else: + audio_tuple = (np.array(chunk), SAMPLE_RATE) + + try: + results = self.model.transcribe( + audio=[audio_tuple], + return_time_stamps=False + ) + except TypeError as e: + logger.warning(f"Transcribe argument error: {e}. Retrying.") + results = self.model.transcribe( + audio=[audio_tuple], + return_time_stamps=False + ) + + if isinstance(results, list): + for res in results: + yield {"text": getattr(res, 'text', str(res))} + else: + yield {"text": getattr(results, 'text', str(results))} + + else: + if isinstance(audio_input, np.ndarray): + audio_tuple = (audio_input, SAMPLE_RATE) + else: + audio_tuple = (np.array(audio_input), SAMPLE_RATE) + + results = self.model.transcribe(audio=[audio_tuple]) + + out_text = [] + if isinstance(results, list): + out_text = [{"text": getattr(r, 'text', str(r))} for r in results] + else: + out_text = [{"text": getattr(results, 'text', str(results))}] + return out_text + + +def load_asr_model(): logger.info(f"Loading {ASR_MODEL_NAME} on {DEVICE}...") - processor = AutoProcessor.from_pretrained(ASR_MODEL_NAME) - asr_model = MoonshineForConditionalGeneration.from_pretrained(ASR_MODEL_NAME).to( - device=DEVICE, + model = Qwen3ASRModel.from_pretrained( + ASR_MODEL_NAME, + device_map=DEVICE, dtype=DTYPE, ) - asr_pipe = pipeline( - task="automatic-speech-recognition", - model=asr_model, - tokenizer=processor.tokenizer, - feature_extractor=processor.feature_extractor, - device=-1 if DEVICE == "cuda" else 0, - dtype=DTYPE, - ) - - return asr_pipe + return QwenASRPipelineWrapper(model) def stream_generator(queue) -> Generator: """ - Generator that yields audio from a queue for the ASR pipeline. - - Args: - queue: Queue containing audio numpy arrays. None signals stop. - - Yields: - Audio numpy arrays until None is received + Generator that yields audio from a queue. """ while True: audio = queue.get() diff --git a/core/assistant.py b/core/assistant.py index d08bb8a..8202cc3 100644 --- a/core/assistant.py +++ b/core/assistant.py @@ -11,9 +11,9 @@ import threading from typing import Optional from .audio import AudioCapture -from .asr import load_moonshine, stream_generator +from .asr import load_asr_model, stream_generator from .tts import speak_stream, remove_emoji -from .slm import load_slm, generate_slm, SLM_MODEL +from .slm import load_slm, generate_slm from utils.system_prompts import getIntentSystemPrompt, getChatSystemPrompt from utils.intent_catch import catchAll @@ -34,7 +34,7 @@ class Assistant: grammar: JSON grammar for structured output """ - def __init__(self, wakeword: str, slm_model_path: str = SLM_MODEL): + def __init__(self, wakeword: str, use_ai: bool): """ Initialize the assistant. @@ -43,7 +43,7 @@ class Assistant: slm_model_path: Path to SLM model (empty to disable AI) """ self.wakeword = wakeword.lower() - self.slm_model_path = slm_model_path + self.use_ai = use_ai self.audio_capture = AudioCapture() # Models loaded lazily in transcriber thread @@ -55,10 +55,10 @@ class Assistant: def _load_models(self): """Load ASR and optionally SLM models.""" - self.asr_pipe = load_moonshine() + self.asr_pipe = load_asr_model() - if self.slm_model_path: - self.grammar, self.slm_model = load_slm(self.slm_model_path) + if self.use_ai: + self.grammar, self.slm_model = load_slm() self.intent_prompt = getIntentSystemPrompt() self.chat_prompt = getChatSystemPrompt() diff --git a/core/slm.py b/core/slm.py index ebca97b..b4bbb68 100644 --- a/core/slm.py +++ b/core/slm.py @@ -14,7 +14,7 @@ from llama_cpp import Llama, LlamaGrammar logger = logging.getLogger(__name__) # Model configuration -SLM_MODEL = "./data/models/Qwen3-4B-Instruct-2507-Q4_K_M.gguf" +MODEL_PATH = "./data/models/Qwen3-4B-Instruct-2507-Q4_K_M.gguf"#"./data/models/Qwen3-0.6B-Q4_K_M.gguf" GRAMMAR_FILE = "./data/models/grammars/json.gbnf" N_CONTEXT = 8192 @@ -26,7 +26,7 @@ DEVICE = "cuda" if torch.cuda.is_available() else "cpu" def load_slm( - model_path: str = SLM_MODEL, + model_path: str = MODEL_PATH, grammar_path: str = GRAMMAR_FILE, n_ctx: int = N_CONTEXT, n_threads: int = N_THREADS, diff --git a/core/tts.py b/core/tts.py index 92dba06..deb2d42 100644 --- a/core/tts.py +++ b/core/tts.py @@ -32,6 +32,9 @@ EMOJI_PATTERN = re.compile( flags=re.UNICODE, ) +# Thinking removal pattern +THINK_PATTERN = r".*?" + # Create global pipeline (loads model once) logger.info(f"Loading {TTS_MODEL_NAME} on {DEVICE}...") kpipeline = KPipeline( @@ -41,8 +44,12 @@ kpipeline = KPipeline( ) -def remove_emoji(text: str) -> str: - """Remove emoji characters from text.""" +def remove_emoji(text: str, rem_think: bool = True) -> str: + """Remove emoji characters and thinking from text.""" + if rem_think: + text = re.sub(THINK_PATTERN, "", text, flags=re.DOTALL) + text = text.strip() + return EMOJI_PATTERN.sub("", text) diff --git a/data/config.example.yml b/data/config.example.yml index 781bf61..6230dee 100644 --- a/data/config.example.yml +++ b/data/config.example.yml @@ -9,6 +9,10 @@ general: # Wakeword to activate the assistant (case-insensitive) # Other examples: "alexa", "jeeves", "mycroft" wakeword: "computer" + # Use a small language model for extra capabilities + use_ai: True + # Use the Moonshine Tiny ASR if running on very small edge devices + use_tiny_asr: False # ============================================================================= # Spotify Integration diff --git a/launch.sh b/launch.sh old mode 100644 new mode 100755 index 0c281fe..8e5bd92 --- a/launch.sh +++ b/launch.sh @@ -11,8 +11,9 @@ QWEN_REPO="unsloth/Qwen3-4B-Instruct-2507-GGUF" QWEN_FILE="Qwen3-4B-Instruct-2507-Q4_K_M.gguf" # Define the cache folders -KOKORO_DIR="$HUB_DIR/models--hexgrad--Kokoro-82M" -MOONSHINE_DIR="$HUB_DIR/models--UsefulSensors--moonshine-tiny" +ASR_TINY_DIR="$HUB_DIR/models--UsefulSensors--moonshine-tiny" +ASR_DIR="$HUB_DIR/models--Qwen--Qwen3-ASR-0.6B" +TTS_DIR="$HUB_DIR/models--hexgrad--Kokoro-82M" # 1. Ensure Dependencies are installed if ! command -v huggingface-cli &> /dev/null; then @@ -22,16 +23,15 @@ fi # 2. Create Directory Structure echo "📂 Checking directory structure..." -mkdir -p "$GRAMMAR_DIR" mkdir -p "$HUB_DIR" # 3. Check and Download json.gbnf if [ ! -f "$GRAMMAR_DIR/json.gbnf" ]; then - echo "⬇️ Downloading json.gbnf..." - wget -q --show-progress -O "$GRAMMAR_DIR/json.gbnf" \ - "https://raw.githubusercontent.com/ggml-org/llama.cpp/master/grammars/json.gbnf" +echo "⬇️ Downloading json.gbnf..." +wget -q --show-progress -O "$GRAMMAR_DIR/json.gbnf" \ +"https://raw.githubusercontent.com/ggml-org/llama.cpp/master/grammars/json.gbnf" else - echo "✅ json.gbnf exists." +echo "✅ json.gbnf exists." fi # 4. Check and Download Qwen3 GGUF @@ -45,21 +45,30 @@ else fi # 5. Check and Download Kokoro-82M (TTS) -if [ ! -d "$KOKORO_DIR" ]; then - echo "⬇️ Downloading Kokoro-82M..." - huggingface-cli download hexgrad/Kokoro-82M \ - --cache-dir "$HUB_DIR" +if [ ! -d "$TTS_DIR" ]; then +echo "⬇️ Downloading Kokoro-82M..." +huggingface-cli download hexgrad/Kokoro-82M \ +--cache-dir "$HUB_DIR" else - echo "✅ Kokoro-82M exists." +echo "✅ Kokoro-82M exists." fi -# 6. Check and Download Moonshine Tiny (STT) -if [ ! -d "$MOONSHINE_DIR" ]; then - echo "⬇️ Downloading Moonshine Tiny..." - huggingface-cli download UsefulSensors/moonshine-tiny \ +# 6. Check and Download Qwen3 ASR model +if [ ! -d "$ASR_DIR" ]; then + echo "⬇️ Downloading Qwen3 ASR..." + huggingface-cli download Qwen/Qwen3-ASR-0.6B \ --cache-dir "$HUB_DIR" else - echo "✅ Moonshine-tiny exists." + echo "✅ ASR model exists." +fi + +# 6a. Check and Download Moonshine Tiny ASR model +if [ ! -d "$ASR_TINY_DIR" ]; then +echo "⬇️ Downloading Moonshine Tiny..." +huggingface-cli download UsefulSensors/moonshine-tiny \ +--cache-dir "$HUB_DIR" +else +echo "✅ Moonshine-tiny exists." fi # 7. Prompt the user diff --git a/pyproject.toml b/pyproject.toml index c3f2183..7f5987d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,10 +100,10 @@ all = [ fulloch = "app:main" [project.urls] -Homepage = "https://github.com/liampetti/fulloch" -Documentation = "https://github.com/liampetti/fulloch#readme" -Repository = "https://github.com/liampetti/fulloch" -Issues = "https://github.com/liampetti/fulloch/issues" +Homepage = "https://github.com/yourusername/fulloch" +Documentation = "https://github.com/yourusername/fulloch#readme" +Repository = "https://github.com/yourusername/fulloch" +Issues = "https://github.com/yourusername/fulloch/issues" [tool.setuptools.packages.find] where = ["."] diff --git a/readme.md b/readme.md index 7ef50bb..78dcf30 100644 --- a/readme.md +++ b/readme.md @@ -68,7 +68,7 @@ A privacy-focused voice assistant that runs speech recognition, text-to-speech, ### 1. Clone and Install ```bash -git clone https://github.com/liampetti/fulloch.git +git clone https://github.com/yourusername/fulloch.git cd fulloch pip install -r requirements.txt ``` diff --git a/requirements.txt b/requirements.txt index 7bf9d77..3efc0e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,10 +12,12 @@ torchaudio==2.8.0 # AI and ML torch==2.8.0 -transformers==4.57.1 +transformers==4.57.6 kokoro==0.9.4 accelerate==1.12.0 llama_cpp_python==0.3.16 +qwen-tts==0.0.5 +qwen-asr==0.0.6 # Smart home integration phue==1.1.0 diff --git a/searxng_data/settings.yml b/searxng_data/settings.yml index a374ea0..7f7e350 100644 --- a/searxng_data/settings.yml +++ b/searxng_data/settings.yml @@ -95,7 +95,7 @@ server: # If your instance owns a /etc/searxng/settings.yml file, then set the following # values there. - secret_key: "changeme" # Is overwritten by ${SEARXNG_SECRET} + secret_key: "JFHpQvFGOR9gpXzAuR0FBUNBOvCjaNi7" # Is overwritten by ${SEARXNG_SECRET} # Proxy image results through SearXNG. Is overwritten by ${SEARXNG_IMAGE_PROXY} image_proxy: false # 1.0 and 1.1 are supported