mirror of
https://github.com/Nighthawk42/MioTTS.git
synced 2026-08-30 08:42:27 +00:00
118 lines
3.4 KiB
Python
118 lines
3.4 KiB
Python
import os
|
|
import shutil
|
|
import csv
|
|
import soundfile as sf
|
|
from tqdm import tqdm
|
|
from pathlib import Path
|
|
|
|
# === SETTINGS ===
|
|
DATASET_ROOT = Path("datasets/mio_dataset")
|
|
METADATA_FILE = DATASET_ROOT / "metadata.csv"
|
|
WAVS_DIR = DATASET_ROOT / "wavs"
|
|
BACKUP_DIR = DATASET_ROOT / "wavs_backup"
|
|
NEW_WAVS_DIR = DATASET_ROOT / "wavs_clean"
|
|
# ================
|
|
|
|
def clean_text(text):
|
|
"""Removes unsupported characters from IPA/Text."""
|
|
# Remove the syllabic marker (combining vertical line below)
|
|
text = text.replace("̩", "")
|
|
# Replace ellipsis with standard dots (optional, or just remove)
|
|
text = text.replace("…", "...")
|
|
# Replace em-dash with comma or space (pauses)
|
|
text = text.replace("—", ", ")
|
|
return text
|
|
|
|
def main():
|
|
if not METADATA_FILE.exists():
|
|
print(f"ERROR: Could not find {METADATA_FILE}")
|
|
return
|
|
|
|
print(f"Reading {METADATA_FILE}...")
|
|
|
|
valid_entries = []
|
|
poison_pills = []
|
|
|
|
with open(METADATA_FILE, "r", encoding="utf-8") as f:
|
|
reader = csv.reader(f, delimiter="|")
|
|
data = list(reader)
|
|
|
|
print("Scanning for poison pills and scrubbing text...")
|
|
|
|
for row in tqdm(data):
|
|
if not row: continue
|
|
|
|
file_id = row[0]
|
|
# Clean the text fields immediately
|
|
text = clean_text(row[1])
|
|
norm_text = clean_text(row[2]) if len(row) > 2 else text
|
|
|
|
wav_path = WAVS_DIR / f"{file_id}.wav"
|
|
|
|
if not wav_path.exists():
|
|
continue
|
|
|
|
try:
|
|
# Check duration
|
|
info = sf.info(str(wav_path))
|
|
if info.duration > 15.0:
|
|
poison_pills.append(file_id)
|
|
continue
|
|
|
|
# Keep valid entry with CLEANED text
|
|
valid_entries.append({
|
|
"old_path": wav_path,
|
|
"text": text,
|
|
"norm_text": norm_text
|
|
})
|
|
|
|
except Exception as e:
|
|
print(f"Error reading {file_id}: {e}")
|
|
|
|
print(f"\n[!] Found and removing {len(poison_pills)} poison pills.")
|
|
print(f"[+] Keeping {len(valid_entries)} valid files.")
|
|
|
|
# Create new clean structure
|
|
if NEW_WAVS_DIR.exists():
|
|
shutil.rmtree(NEW_WAVS_DIR)
|
|
NEW_WAVS_DIR.mkdir()
|
|
|
|
new_metadata_rows = []
|
|
|
|
print("\nRenaming files and writing clean metadata...")
|
|
|
|
for index, entry in enumerate(tqdm(valid_entries)):
|
|
# Generate new sequential ID
|
|
new_id = f"mio_dataset_{index:05d}"
|
|
new_filename = f"{new_id}.wav"
|
|
new_path = NEW_WAVS_DIR / new_filename
|
|
|
|
# Copy file
|
|
shutil.copy2(entry["old_path"], new_path)
|
|
|
|
# Add to new metadata list using the CLEANED text
|
|
new_metadata_rows.append(f"{new_id}|{entry['text']}|{entry['norm_text']}")
|
|
|
|
print("\nSwapping folders...")
|
|
if BACKUP_DIR.exists():
|
|
shutil.rmtree(BACKUP_DIR)
|
|
|
|
WAVS_DIR.rename(BACKUP_DIR)
|
|
NEW_WAVS_DIR.rename(WAVS_DIR)
|
|
|
|
print("Writing new metadata.csv...")
|
|
shutil.copy2(METADATA_FILE, str(METADATA_FILE) + ".bak")
|
|
|
|
with open(METADATA_FILE, "w", encoding="utf-8") as f:
|
|
f.write("\n".join(new_metadata_rows))
|
|
|
|
print("\n" + "="*40)
|
|
print("COMPLETE")
|
|
print("="*40)
|
|
print("1. Long files deleted.")
|
|
print("2. Text cleaned (syllabic markers removed).")
|
|
print("3. Metadata re-indexed.")
|
|
print("You can now train with aggressive settings.")
|
|
|
|
if __name__ == "__main__":
|
|
main() |