mirror of
https://github.com/Nighthawk42/MioTTS.git
synced 2026-08-30 09:42:27 +00:00
95 lines
2.9 KiB
Python
95 lines
2.9 KiB
Python
import argparse
|
|
import os
|
|
from typing import List, Tuple
|
|
|
|
|
|
def _process_line(line: str) -> Tuple[bool, str]:
|
|
stripped = line.strip()
|
|
if not stripped:
|
|
return False, ""
|
|
|
|
parts = stripped.split("|")
|
|
if len(parts) != 3:
|
|
return False, stripped
|
|
|
|
file_id, normalized_text, original_text = parts
|
|
file_id = file_id.strip()
|
|
normalized_text = normalized_text.strip()
|
|
original_text = original_text.strip()
|
|
|
|
cleaned = "|".join([file_id, normalized_text, original_text])
|
|
return True, cleaned
|
|
|
|
|
|
def _fix_csv(input_path: str, output_path: str, skip_invalid: bool, keep_invalid: bool) -> List[str]:
|
|
invalid_lines: List[str] = []
|
|
|
|
with open(input_path, encoding="utf-8-sig") as fin, open(
|
|
output_path,
|
|
"w",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
) as fout:
|
|
for raw_line in fin:
|
|
ok, cleaned = _process_line(raw_line)
|
|
if not ok:
|
|
if keep_invalid:
|
|
invalid_lines.append(raw_line.rstrip("\n"))
|
|
if skip_invalid:
|
|
continue
|
|
# write the original line (normalized only for newlines)
|
|
fout.write(raw_line.rstrip("\r\n") + "\n")
|
|
continue
|
|
|
|
fout.write(cleaned + "\n")
|
|
|
|
return invalid_lines
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(
|
|
description=(
|
|
"Clean and normalize an LJSpeech-style metadata.csv file. "
|
|
"Removes BOM, normalizes EOLs, and trims surrounding whitespace."
|
|
)
|
|
)
|
|
parser.add_argument("--input", required=True, help="Path to the original metadata.csv file.")
|
|
parser.add_argument("--output", required=True, help="Path to write the cleaned metadata.csv file.")
|
|
parser.add_argument(
|
|
"--skip-invalid",
|
|
action="store_true",
|
|
help="Skip lines that do not have exactly 3 '|' separated fields.",
|
|
)
|
|
parser.add_argument(
|
|
"--keep-invalid",
|
|
action="store_true",
|
|
help="Write invalid lines to a separate '<output>.invalid' file.",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
input_path = os.path.abspath(args.input)
|
|
output_path = os.path.abspath(args.output)
|
|
|
|
if not os.path.isfile(input_path):
|
|
msg = f"input metadata file not found: {input_path}"
|
|
raise FileNotFoundError(msg)
|
|
|
|
os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
|
|
|
|
print(f"Reading : {input_path}")
|
|
print(f"Writing : {output_path}")
|
|
|
|
invalid_lines = _fix_csv(input_path, output_path, args.skip_invalid, args.keep_invalid)
|
|
|
|
if args.keep_invalid and invalid_lines:
|
|
invalid_path = output_path + ".invalid"
|
|
with open(invalid_path, "w", encoding="utf-8", newline="\n") as f:
|
|
for line in invalid_lines:
|
|
f.write(line + "\n")
|
|
print(f"Invalid lines written to: {invalid_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|