mirror of
https://github.com/Nighthawk42/soprano-factory.git
synced 2026-08-30 04:30:21 +00:00
29 lines
856 B
Python
29 lines
856 B
Python
# utils/text_normalizer.py
|
|
import re
|
|
import unicodedata
|
|
|
|
def normalize_text(text: str) -> str:
|
|
"""
|
|
Standardizes text for training and inference.
|
|
- Lowercases everything.
|
|
- Removes accents (e.g., é -> e).
|
|
- Strips unknown symbols.
|
|
- Collapses extra whitespace.
|
|
"""
|
|
if not isinstance(text, str):
|
|
return ""
|
|
|
|
# 1. Standardize Case
|
|
text = text.lower()
|
|
|
|
# 2. Decompose characters and remove non-spacing marks (accents)
|
|
text = unicodedata.normalize('NFD', text)
|
|
text = "".join([c for c in text if unicodedata.category(c) != 'Mn'])
|
|
|
|
# 3. Filter characters: keep only basic English letters, numbers, spaces, and core punctuation
|
|
text = re.sub(r"[^a-z0-9\s.,!?'-]", "", text)
|
|
|
|
# 4. Collapse multiple spaces into one
|
|
text = re.sub(r"\s+", " ", text)
|
|
|
|
return text.strip() |