Nighthawk/fulloch
1
0
Fork 0
mirror of https://github.com/Nighthawk42/fulloch.git synced 2026-08-30 10:12:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

voice clone config and improved installation/launch

Browse Source
This commit is contained in:
Liam Pettigrew
2026-02-06 12:44:13 +11:00
parent b1c26f5ce0
commit becd782f2f
15 changed files with 215 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/assistant.py
+10 -5
View File
@@ -31,7 +31,7 @@ class Assistant:
grammar: JSON grammar for structured output
"""
def __init__(self, wakeword: str, use_ai: bool, use_tiny_asr: bool = False, use_tiny_tts: bool = False):
def __init__(self, wakeword: str, use_ai: bool, use_tiny_asr: bool = False, use_tiny_tts: bool = False, voice_clone: str = "cori"):
"""
Initialize the assistant.
@@ -40,11 +40,13 @@ class Assistant:
use_ai: Whether to use the SLM for intent detection
use_tiny_asr: Whether to use Moonshine Tiny ASR instead of Qwen ASR
use_tiny_tts: Whether to use Kokoro TTS instead of Qwen TTS
voice_clone: Name of the voice clone wav/txt file to use for TTS
"""
self.wakeword = wakeword.lower()
self.use_ai = use_ai
self.use_tiny_asr = use_tiny_asr
self.use_tiny_tts = use_tiny_tts
self.voice_clone = voice_clone
self.audio_capture = AudioCapture()
# Models loaded lazily in transcriber thread
@@ -54,6 +56,7 @@ class Assistant:
self.grammar = None
self.intent_prompt = None
self.chat_prompt = None
self.voice_prompt = None
def _load_models(self):
"""Load ASR, TTS and SLM models."""
@@ -71,8 +74,10 @@ class Assistant:
from .tts_tiny import speak_stream, remove_emoji
logger.info("Using Kokoro TTS")
else:
from .tts import speak_stream, remove_emoji
logger.info("Using Qwen TTS")
from .tts import speak_stream, remove_emoji, set_voice, warmup_model
self.voice_prompt = set_voice(self.voice_clone)
logger.info(f"Using Qwen TTS with voice clone: {self.voice_clone}")
warmup_model(self.voice_prompt)
self.remove_emoji = remove_emoji
self.speak_stream = speak_stream
@@ -142,7 +147,7 @@ class Assistant:
"Just a second.",
"Got it, let me think.",
"Let's see."
]))
]), self.voice_prompt)
logger.info(f"AI chat query: {user_prompt}")
answer = generate_slm(
@@ -198,7 +203,7 @@ class Assistant:
# Process and respond
answer = self._handle_wakeword(user_prompt)
cleaned = self.remove_emoji(answer.replace('"', '').replace('*', ''))
self.speak_stream(cleaned)
self.speak_stream(cleaned, self.voice_prompt)
# Resume transcription
self.audio_capture.transcribing = True
core/tts.py
+44 -27
View File
@@ -8,13 +8,16 @@ import queue
import re
import threading
import torch
# Set precision before any CUDA operations
torch.set_float32_matmul_precision('high')
import sounddevice as sd
from qwen_tts import Qwen3TTSModel
logger = logging.getLogger(__name__)
REF_AUDIO = "./data/voices/cori.wav"
REF_TEXT = "./data/voices/cori.txt"
VOICES_DIR = "./data/voices"
# Device configuration
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
@@ -43,10 +46,7 @@ def remove_emoji(text: str, rem_think: bool = True) -> str:
return EMOJI_PATTERN.sub("", text)
# Get reference audio text
with open(REF_TEXT) as f:
ref_text = f.read()
# Load model
model = Qwen3TTSModel.from_pretrained(
"Qwen/Qwen3-TTS-12Hz-1.7B-Base",
@@ -62,35 +62,52 @@ model.enable_streaming_optimizations(
compile_mode="reduce-overhead", # Includes CUDA graphs automatically
)
# Create voice clone prompt from reference audio
prompt = model.create_voice_clone_prompt(
ref_audio=REF_AUDIO,
ref_text=ref_text,
)
def set_voice(voice_name: str):
"""
Load voice clone prompt from audio/text files to use for TTS.
Args:
voice_name: Name of the voice (matches wav/txt files in data/voices)
"""
ref_audio = f"{VOICES_DIR}/{voice_name}.wav"
ref_text_path = f"{VOICES_DIR}/{voice_name}.txt"
with open(ref_text_path) as f:
ref_text = f.read()
logger.info(f"Setting voice clone to: {voice_name}")
return model.create_voice_clone_prompt(
ref_audio=ref_audio,
ref_text=ref_text,
)
# Warmup: run dummy generation to initialize torch.compile and CUDA graphs
logger.info("Warming up TTS model...")
for _ in model.stream_generate_voice_clone(
# Reference text must be longer to properly initialise model
text="A rainbow is a meteorological phenomenon that is caused by reflection, refraction and dispersion of light in water droplets resulting in a spectrum of light appearing in the sky.",
language="english",
voice_clone_prompt=prompt,
overlap_samples=512,
emit_every_frames=12,
decode_window_frames=80,
first_chunk_emit_every=5,
first_chunk_decode_window=48,
first_chunk_frames=48,
):
pass # Discard output
logger.info("TTS model ready")
def warmup_model(prompt):
logger.info("Warming up TTS model...")
for _ in model.stream_generate_voice_clone(
# Reference text must be longer to properly initialise model
text="A rainbow is a meteorological phenomenon that is caused by reflection, refraction and dispersion of light in water droplets.",
language="english",
voice_clone_prompt=prompt,
overlap_samples=512,
emit_every_frames=12,
decode_window_frames=80,
first_chunk_emit_every=5,
first_chunk_decode_window=48,
first_chunk_frames=48,
):
pass # Discard output
logger.info("TTS model ready")
def speak_stream(text: str, voice: str = "cori", speed: float = 1.0):
def speak_stream(text: str, prompt, voice: str = "cori", speed: float = 1.0):
"""
Generate speech from text using stream optimised Qwen3 TTS from rekuenkdr
Args:
text: Text to synthesize
prompt: Prepared voice cloning prompt
voice: Not used
speed: Not used
"""
core/tts_tiny.py
+2 -1
View File
@@ -53,12 +53,13 @@ def remove_emoji(text: str, rem_think: bool = True) -> str:
return EMOJI_PATTERN.sub("", text)
def speak_stream(text: str, voice: str = "af_bella", speed: float = 1.2):
def speak_stream(text: str, prompt=None, voice: str = "af_bella", speed: float = 1.2):
"""
Generate speech from text using Kokoro and stream to speakers.
Args:
text: Text to synthesize
prompt: Not used
voice: Voice model to use (default: af_bella)
speed: Speech speed multiplier (default: 1.2)
"""