From 9606162d99521b32457beb42188b22003a814c10 Mon Sep 17 00:00:00 2001 From: Nighthawk Date: Tue, 1 Apr 2025 12:58:13 -0400 Subject: [PATCH] Add files via upload --- export_onnx.py | 457 ++++++ infer/__init__.py | 0 infer/lib/infer_pack/__init__.py | 0 .../__pycache__/__init__.cpython-311.pyc | Bin 0 -> 173 bytes .../attentions_onnx.cpython-311.pyc | Bin 0 -> 21305 bytes .../__pycache__/commons.cpython-311.pyc | Bin 0 -> 10597 bytes .../__pycache__/models_onnx.cpython-311.pyc | Bin 0 -> 38568 bytes .../__pycache__/modules.cpython-311.pyc | Bin 0 -> 30254 bytes .../__pycache__/transforms.cpython-311.pyc | Bin 0 -> 7609 bytes infer/lib/infer_pack/attentions.py | 459 +++++++ infer/lib/infer_pack/attentions_onnx.py | 459 +++++++ infer/lib/infer_pack/commons.py | 172 +++ infer/lib/infer_pack/models.py | 1223 +++++++++++++++++ infer/lib/infer_pack/models_onnx.py | 818 +++++++++++ infer/lib/infer_pack/modules.py | 615 +++++++++ infer/lib/infer_pack/onnx_inference.py | 149 ++ infer/lib/infer_pack/transforms.py | 207 +++ 17 files changed, 4559 insertions(+) create mode 100644 export_onnx.py create mode 100644 infer/__init__.py create mode 100644 infer/lib/infer_pack/__init__.py create mode 100644 infer/lib/infer_pack/__pycache__/__init__.cpython-311.pyc create mode 100644 infer/lib/infer_pack/__pycache__/attentions_onnx.cpython-311.pyc create mode 100644 infer/lib/infer_pack/__pycache__/commons.cpython-311.pyc create mode 100644 infer/lib/infer_pack/__pycache__/models_onnx.cpython-311.pyc create mode 100644 infer/lib/infer_pack/__pycache__/modules.cpython-311.pyc create mode 100644 infer/lib/infer_pack/__pycache__/transforms.cpython-311.pyc create mode 100644 infer/lib/infer_pack/attentions.py create mode 100644 infer/lib/infer_pack/attentions_onnx.py create mode 100644 infer/lib/infer_pack/commons.py create mode 100644 infer/lib/infer_pack/models.py create mode 100644 infer/lib/infer_pack/models_onnx.py create mode 100644 infer/lib/infer_pack/modules.py create mode 100644 infer/lib/infer_pack/onnx_inference.py create mode 100644 infer/lib/infer_pack/transforms.py diff --git a/export_onnx.py b/export_onnx.py new file mode 100644 index 0000000..ae28450 --- /dev/null +++ b/export_onnx.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Exports RVC (Retrieval-based Voice Conversion) models from PyTorch (.pth) +to ONNX format, simplifying the model optionally. +Prompts the user for the input model path and optionally verifies the output. +Logs output to console and 'conversion.log'. Keeps window open on exit. +Includes workaround for onnxsim dynamic shape issue and weight_norm FutureWarning. +""" + +import logging +import os +import sys +import argparse +import time +import warnings # Added for warning suppression + +# Third-party libraries +import onnx +import onnxsim +import torch + +# Optional verification library +try: + import onnxruntime as ort + ORT_AVAILABLE = True +except ImportError: + ORT_AVAILABLE = False + + +# --- Configure Logging (File and Console) --- +log_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') +root_logger = logging.getLogger() +root_logger.setLevel(logging.INFO) + +# Console Handler +console_handler = logging.StreamHandler(sys.stdout) +console_handler.setFormatter(log_formatter) +root_logger.addHandler(console_handler) + +# File Handler (log to conversion.log in script's directory) +try: + script_dir = os.path.dirname(os.path.abspath(__file__)) + log_file_path = os.path.join(script_dir, "conversion.log") + # 'a' appends to the log, 'w' overwrites + file_handler = logging.FileHandler(log_file_path, mode='a', encoding='utf-8') + file_handler.setFormatter(log_formatter) + root_logger.addHandler(file_handler) + # Use print only for the initial notification about the log file location + print(f"Logging detailed output to: {log_file_path}") +except Exception as e: + print(f"Warning: Could not configure file logging. Error: {e}", file=sys.stderr) + + +# --- RVC Model Import --- +# Suppress the specific FutureWarning during import and model loading if needed +# Note: Ideally, the RVC library itself should be updated. +try: + # This context manager is generally better applied specifically around the + # code that triggers the warning (model instantiation/loading). + # Placing it here might suppress warnings during import of the module itself. + # with warnings.catch_warnings(): + # warnings.filterwarnings("ignore", category=FutureWarning, message=".*`torch.nn.utils.weight_norm` is deprecated.*") + from infer.lib.infer_pack.models_onnx import SynthesizerTrnMsNSFsidM +except ImportError as e: + logging.error(f"Error importing SynthesizerTrnMsNSFsidM: {e}", exc_info=True) + logging.error("Please ensure the script is run from within the RVC " + "project structure or that the RVC library's root " + "directory is in your PYTHONPATH.") + input("Import error occurred. Press Enter to exit...") + sys.exit(1) +except Exception as e: + logging.error(f"An unexpected error occurred during import: {e}", exc_info=True) + input("Import error occurred. Press Enter to exit...") + sys.exit(1) + + +class RvcOnnxExporter: + """Handles the export of an RVC model to ONNX format.""" + + INPUT_NAMES = ["phone", "phone_lengths", "pitch", "pitchf", "ds", "rnd"] + OUTPUT_NAMES = ["audio"] + + def __init__(self, opset_version: int = 18, simplify: bool = True): + """ + Initializes the exporter. + + Args: + opset_version (int): The ONNX opset version for export. + simplify (bool): Whether to simplify the exported model using onnxsim. + """ + self.opset_version = opset_version + self.simplify = simplify + self.config = None + self.model_version = "v1" + self.vec_channels = 256 + self.n_spk = 1 + self.posterior_channels = 192 # Default value used in error case before + self.dummy_input_shapes = {} # Store shapes for simplify step + + def _load_checkpoint(self, model_path: str) -> dict | None: + """Loads the PyTorch checkpoint file.""" + if not os.path.exists(model_path): + logging.error(f"Model file not found: {model_path}") + return None + try: + logging.info(f"Loading checkpoint: {model_path}") + checkpoint = torch.load(model_path, map_location=torch.device("cpu")) + + required_keys = ['config', 'weight'] + if not all(key in checkpoint for key in required_keys): + missing = [k for k in required_keys if k not in checkpoint] + logging.error(f"Checkpoint missing required keys: {missing}. " + f"Found keys: {list(checkpoint.keys())}") + return None + if "emb_g.weight" not in checkpoint["weight"]: + logging.error("Speaker embedding 'emb_g.weight' not found in " + "checkpoint weights.") + return None + logging.info("Checkpoint loaded successfully.") + return checkpoint + except Exception as e: + logging.error(f"Failed to load checkpoint '{model_path}': {e}", + exc_info=True) + return None + + def _prepare_config(self, checkpoint: dict) -> bool: + """Determines model version, channels, and speaker count from config.""" + try: + logging.info("Preparing model configuration...") + self.config = list(checkpoint["config"]) + self.model_version = checkpoint.get("version", "v1") + self.vec_channels = 768 if self.model_version == "v2" else 256 + logging.info(f"Detected model version: {self.model_version}, " + f"Feature channels: {self.vec_channels}") + + # Use config[2] for posterior_channels if valid, otherwise log error + if len(self.config) > 2 and isinstance(self.config[2], int) and self.config[2] > 0: + self.posterior_channels = self.config[2] + logging.info(f"Using posterior encoder channels (config[2]): " + f"{self.posterior_channels} for 'rnd' input.") + else: + # Log the error but might proceed if the old default was somehow correct (unlikely) + # Consider returning False here for stricter validation + config2_val = self.config[2] if len(self.config) > 2 else 'N/A' + logging.error("Could not determine valid posterior encoder channels " + f"from config[2] (value: {config2_val}). " + f"Using fallback: {self.posterior_channels}") + # return False # Uncomment for stricter validation + + n_spk_inferred = checkpoint["weight"]["emb_g.weight"].shape[0] + if len(self.config) < 3: + logging.error(f"Config list has fewer than 3 elements " + f"({len(self.config)}), cannot access speaker " + f"count at index -3.") + return False + + n_spk_config = self.config[-3] + if n_spk_config != n_spk_inferred: + logging.warning(f"Mismatch: Config speaker count ({n_spk_config})" + f" != Inferred count ({n_spk_inferred}). " + f"Updating config.") + self.config[-3] = n_spk_inferred + + self.n_spk = n_spk_inferred + logging.info(f"Using speaker count (n_spk): {self.n_spk}") + self.config = tuple(self.config) + logging.info("Model configuration prepared successfully.") + return True + + except (IndexError, KeyError, TypeError, ValueError) as e: + logging.error(f"Failed to prepare config from checkpoint: {e}", + exc_info=True) + return False + + def _create_dummy_inputs(self, seq_len: int = 200) -> tuple | None: + """Creates dummy input tensors for ONNX tracing and stores their shapes.""" + try: + logging.info(f"Creating dummy inputs (sequence length: {seq_len})...") + # Create tensors + dummy_phone = torch.rand(1, seq_len, self.vec_channels, dtype=torch.float32) + dummy_phone_lengths = torch.tensor([seq_len], dtype=torch.long) + dummy_pitch = torch.randint(low=5, high=255, size=(1, seq_len), dtype=torch.long) + dummy_pitchf = torch.rand(1, seq_len, dtype=torch.float32) + dummy_ds = torch.tensor([0], dtype=torch.long) # Speaker ID + dummy_rnd = torch.rand(1, self.posterior_channels, seq_len, dtype=torch.float32) + + inputs_tuple = (dummy_phone, dummy_phone_lengths, dummy_pitch, + dummy_pitchf, dummy_ds, dummy_rnd) + + # Store shapes mapped to names for use in simplification step + self.dummy_input_shapes = { + name: list(tensor.shape) + for name, tensor in zip(self.INPUT_NAMES, inputs_tuple) + } + logging.info(f"Dummy input shapes: {self.dummy_input_shapes}") + logging.info("Dummy inputs created successfully.") + return inputs_tuple + except Exception as e: + logging.error(f"Failed to create dummy inputs: {e}", exc_info=True) + return None + + def _instantiate_and_load_model(self, checkpoint: dict) -> SynthesizerTrnMsNSFsidM | None: + """Instantiates the RVC model and loads weights, suppressing specific warnings.""" + try: + logging.info("Instantiating model SynthesizerTrnMsNSFsidM...") + # Suppress the specific weight_norm FutureWarning during model init and loading + # Note: The ideal solution is to update the RVC library code itself. + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=FutureWarning, message=".*`torch.nn.utils.weight_norm` is deprecated.*") + net_g = SynthesizerTrnMsNSFsidM( + *self.config, + is_half=False, + version=self.model_version + ) + logging.info("Loading state dictionary into the model...") + net_g.load_state_dict(checkpoint["weight"], strict=False) + + net_g.eval() + logging.info("Model instantiated and weights loaded successfully.") + return net_g + except Exception as e: + logging.error(f"Failed to instantiate or load model: {e}", + exc_info=True) + return None + + def _simplify_model(self, onnx_path: str) -> bool: + """Simplifies the exported ONNX model using onnxsim, providing input shapes.""" + try: + logging.info("Simplifying ONNX model using onnxsim...") + if not os.path.exists(onnx_path): + logging.error(f"ONNX file not found at {onnx_path} before simplification.") + return False + if not self.dummy_input_shapes: + logging.error("Cannot simplify model: Dummy input shapes were not generated.") + return False # Treat as failure if shapes are missing + + logging.info(f"Providing input shapes to onnxsim: {self.dummy_input_shapes}") + onnx_model = onnx.load(onnx_path) + # Provide the input shapes derived from dummy data + model_opt, check_ok = onnxsim.simplify( + onnx_model, + input_shapes=self.dummy_input_shapes, # Pass the shapes here + check_n=3, + perform_optimization=True + ) + + if check_ok: + onnx.save(model_opt, onnx_path) + logging.info("Simplified ONNX model saved successfully.") + return True + else: + logging.error("ONNX simplification check failed. Keeping the " + "original (unsimplified) model.") + return True # Treat simplification check failure as non-critical + except Exception as e: + # Log the specific error from onnxsim + logging.error(f"Error during ONNX simplification: {e}", exc_info=True) + logging.warning("Proceeding with the unsimplified ONNX model.") + return True # Simplification process failed, but export is usable + + def export(self, model_path: str, exported_path: str) -> bool: + """ + Executes the full export process. + + Args: + model_path (str): Path to the input PyTorch model checkpoint (.pth). + exported_path (str): Path to save the exported ONNX model. + + Returns: + bool: True if base export was successful, False otherwise. + """ + checkpoint = self._load_checkpoint(model_path) + if not checkpoint: return False + + if not self._prepare_config(checkpoint): return False + + dummy_inputs = self._create_dummy_inputs() # Generates shapes needed later + if not dummy_inputs: return False + + net_g = self._instantiate_and_load_model(checkpoint) + if not net_g: return False + + dynamic_axes = { + name: {1: "sequence_length"} for name in ["phone", "pitch", "pitchf"] + } + dynamic_axes["rnd"] = {2: "sequence_length"} # Dim 2 for rnd noise + dynamic_axes[self.OUTPUT_NAMES[0]] = {1: "audio_length"} # Output audio + + try: + logging.info(f"Exporting model to ONNX format at: {exported_path} " + f"(Opset: {self.opset_version})") + torch.onnx.export( + net_g, dummy_inputs, exported_path, + input_names=self.INPUT_NAMES, output_names=self.OUTPUT_NAMES, + dynamic_axes=dynamic_axes, do_constant_folding=False, # Be cautious with folding + dynamic axes + opset_version=self.opset_version, verbose=False, + ) + logging.info(f"Initial ONNX model exported successfully to: {exported_path}") + + except Exception as e: + logging.error(f"ONNX export failed: {e}", exc_info=True) + if os.path.exists(exported_path): + try: os.remove(exported_path) + except OSError as oe: logging.error(f"Could not remove intermediate ONNX file {exported_path}: {oe}") + return False + + if self.simplify: + # _simplify_model now uses self.dummy_input_shapes generated earlier + self._simplify_model(exported_path) + + logging.info("ONNX export process finished.") + return True + + +def get_model_path_from_user() -> str | None: + """Prompts the user to provide a valid .pth model file path.""" + while True: + # Use print for direct user interaction + print("\nPlease provide the path to the RVC model checkpoint file (.pth).") + print("You can type the full path or drag and drop the file onto " + "this window and press Enter.") + print("Type 'quit' or 'exit' to cancel.") + raw_path = input("Model Path: ").strip() + + if raw_path.lower() in ['quit', 'exit']: + logging.info("Operation cancelled by user.") + return None + + cleaned_path = raw_path.strip("'\"") + if not cleaned_path: + print("ERROR: No path provided. Please try again or type 'quit'.") + continue + + try: absolute_path = os.path.abspath(cleaned_path) + except Exception as e: print(f"ERROR: Could not resolve path '{cleaned_path}'. Error: {e}"); continue + + if not os.path.exists(absolute_path): print(f"ERROR: File not found at '{absolute_path}'. Check the path."); continue + if not os.path.isfile(absolute_path): print(f"ERROR: Path '{absolute_path}' is a directory. Need a file."); continue + if not absolute_path.lower().endswith(".pth"): print(f"ERROR: File '{os.path.basename(absolute_path)}' needs .pth extension."); continue + + logging.info(f"Input model accepted: {absolute_path}") + return absolute_path + + +def verify_onnx_model(onnx_path: str): + """Attempts to load the ONNX model using onnxruntime for verification.""" + # Use print for separators in direct user feedback section + print("-" * 30) + logging.info(f"Attempting to verify ONNX model: {onnx_path}") + + if not ORT_AVAILABLE: + logging.warning("ONNX Runtime (onnxruntime) library not found.") + logging.warning("Verification skipped. Install with: pip install onnxruntime") + print("-" * 30) + return + + if not os.path.exists(onnx_path): + logging.error(f"Verification failed: ONNX file not found at {onnx_path}") + print("-" * 30) + return + + try: + providers = ['CPUExecutionProvider'] + ort_session = ort.InferenceSession(onnx_path, providers=providers) + inputs = ort_session.get_inputs() + outputs = ort_session.get_outputs() + logging.info(f"ONNX Runtime loaded model successfully.") + logging.info(f"Detected {len(inputs)} inputs: {[inp.name for inp in inputs]}") + logging.info(f"Detected {len(outputs)} outputs: {[out.name for out in outputs]}") + logging.info("Verification successful.") + + except ort.OrtLoadError as load_error: + logging.error(f"ONNX Runtime failed to load the model: {load_error}", exc_info=True) + logging.error("Verification failed: Model loading error.") + except Exception as e: + logging.error(f"An unexpected error occurred during ONNX verification: {e}", exc_info=True) + logging.error("Verification failed.") + finally: + # Ensure separator is printed even on error during verification + print("-" * 30) + + +def main(): + """Main execution function.""" + # Use print for initial user-facing messages + print("-" * 30) + print("RVC ONNX Exporter") + print("-" * 30) + logging.info("Exporter script started.") # Log that script has begun + + checkpoint_path = get_model_path_from_user() + if not checkpoint_path: + return 0 # User quit + + input_dir = os.path.dirname(checkpoint_path) + base_name = os.path.splitext(os.path.basename(checkpoint_path))[0] + onnx_output_path = os.path.join(input_dir, f"{base_name}.onnx") + + # Log essential paths + print("-" * 30) + logging.info(f"Input PyTorch Model: {checkpoint_path}") + logging.info(f"Output ONNX Model: {onnx_output_path}") + print("-" * 30) + + # --- Initialize and Run Exporter --- + exporter = RvcOnnxExporter(opset_version=18, simplify=True) + export_successful = exporter.export(checkpoint_path, onnx_output_path) + + # --- Final Status & Optional Verification --- + print("-" * 30) # Use print for visual separation for user + if export_successful: + logging.info(f"Successfully exported model to: {onnx_output_path}") + + while True: + # Use print for direct user interaction prompt + verify_choice = input("Verify the exported ONNX model using ONNX Runtime? (y/n): ").lower().strip() + if verify_choice in ['y', 'yes']: + verify_onnx_model(onnx_output_path) + break + elif verify_choice in ['n', 'no']: + logging.info("Skipping ONNX model verification.") + print("-" * 30) # Print separator after skipping + break + else: + print("Invalid input. Please enter 'y' or 'n'.") # Use print for error feedback + else: + logging.error("Model export failed. See log messages above for details.") + + logging.info("Exporter script finished.") + return 0 if export_successful else 1 + + +if __name__ == "__main__": + exit_code = 1 # Default to error + try: + # Start logging session info + logging.info(f"--- Starting new conversion run ---") + logging.info(f"Python version: {sys.version}") + logging.info(f"Torch version: {torch.__version__}") + logging.info(f"ONNX version: {onnx.__version__}") + logging.info(f"ONNXSim version: {onnxsim.__version__}") + logging.info(f"ONNX Runtime Available: {ORT_AVAILABLE}" + (f" (Version: {ort.__version__})" if ORT_AVAILABLE else "")) + + exit_code = main() + + except Exception as e: + logging.error(f"An unexpected critical error occurred in main execution: {e}", exc_info=True) + # Ensure the error is logged before the exit prompt + + finally: + # This block ensures the prompt appears even if main() raises an exception + logging.info(f"--- Ending conversion run (Exit Code: {exit_code}) ---") + print("\nScript execution complete.") # Use print for final user message + input("Press Enter to exit...") # Keep window open + sys.exit(exit_code) \ No newline at end of file diff --git a/infer/__init__.py b/infer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/infer/lib/infer_pack/__init__.py b/infer/lib/infer_pack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/infer/lib/infer_pack/__pycache__/__init__.cpython-311.pyc b/infer/lib/infer_pack/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..60d695717b8409f81cd2b3a9d8432ec4c1647848 GIT binary patch literal 173 zcmZ3^%ge<81jqQ^q=V?kAOZ#$p^VRLK*n^26oz01O-8?!3`I;p{%4TnF9&CTZlX-=wL5i8I@kd?*!K;i>4BO~Jn1{hJq3={(Z%!ez2 literal 0 HcmV?d00001 diff --git a/infer/lib/infer_pack/__pycache__/attentions_onnx.cpython-311.pyc b/infer/lib/infer_pack/__pycache__/attentions_onnx.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4bdf5919f6afd6f33e12faf74b2882ce8fb6f0dd GIT binary patch literal 21305 zcmd^nX>c27mRL8?xEmJ_h?hw4K152QB+D`-OQH@+rexYW?D23kgN4{21qlSG2B?Dr zdgPR67jl>#TE&gvEo}|7rSVc@CE+BqwVbWlU2fK?s3!S=MmizVRjR0xIEnv3B~R7V zCfVe@uYqnf20{{;hu%aDI~`gibf zo8l;r4pHOsogSwltPAO;_2YV4j$_6RB#&|22zPzRIBgm?ku)Y`p0s(N$=~r+SL|4F2V7+)Z*>AeZ$SxvEGm8{}f2 zk*k{IvO_M%Gji2%^ckwx`6V`Bte2L|Z_UJlk+44m&$@6JpmlOKJduynNv4U&^mHT~ zl}ytSZZ;H%LJrGG|MdVr7U8ES@*N?!LbsmwL-6-thyxT3>$uKB;g2&Ws7ab*KD3PM zIm3t4IFq2R(c=baj!7~d4NpY40RJToW&e8t>z6p26Xrq??igW%|EK$*wm+b6Q!y-% zzhb(=6Gg?TPa*V~5|dJ%)I1dRDFi-KB4{d3$0=W-A5>fiA*PTV*F&hVh^Uyo5D#@R zpF%4>Q(6M~Ooi0A0YXX-;zkHrlr$5B$b!X~*rq}}MO|bI&vD}h+)B-w5{^O%ZJ(e9 zXCdVU^DB$>1@pUK5`UGFzq{~+^L0@fGi{HLP~Tpjw?Mkmk5r9XPJN1PQJ-NRX_zl# z-1LC{6z1$RW!|*7Va6$yI4j^CD8{QTuMH7w%4k%;wyhW&v{DBv#P~q>+*T^1Rsruz zv~uA-M*-F8$9i>=WyZ(xk(tPBOtM`KhPlX9Uo4V{zWDfg6xM@Ts+ob>x zB}gr#4j$JVk7Op9j^vxdV4EcCn`Fy58H_^X{IQsV!ybmcCy&rK2s_qf`J^3^f%k`} z0+P``GZP4NQq`NYp;+)Zv}-sP3xsi(Qe{m95zuT5g2fzTw6%Zsl?;6YG_pdJAZ z32ClmymZyiPemmGkKNk#@`-0Jaq1zDD?W=Sa9k>?1KO5rE^CD z(Mz$&%(+N7eC=E?JQ?86g@WhhC*O>J;?gN6@xz1Mu;yG7-kK9_A1>7*A zzX#jSuWnGk^iUSZKbl&M3Faoz+_Y})%$Pd`bGK;jPMf>4En5=|L@W1A%?W0KeN@+& zJpPIOjvWH_Y-1PZsms>4LC{eMeo^CD8ckjoYI?<*UP!P!YTSJ1^-m7mIY4qh+T3#I z+n@OE_%PJ)sG;f3z$e@8Y=ex>k|eihF}5@>xI0C6XO60|9HM2I7|yaTk!@OM+cRwY zBUf9trk(sXW}CaS-Yq$s(Ova~0`L`rCwjBp@f8IiM?e*2cZ+QEI@_6HJAV{haSB}r z#jb+_J1nxpX%>+$ytU4@W!SbyuId7AsG$YW|C+b~eXdvS$y1mQYC<_aAYg=l3_tyS zn5ElrprjzM@#1LhMp7}KPMdEIwzYXZZX)D~juq7n8=G?E!LyPA&+3szT~s$li2Y1S znP=h*Y`l}_AZFe$Z=5&Hn{^ayRb*q;axu6$r?96RuDC_p-gyh)Fai$C9ONxYqan?N zebFA=FNxh;3H!nn;+DrLjy#Zl>`NS=FaHl*z=2q1!_mvLfxuiq($58F_|1^UHzR04 z;6=~~px4TG!cC&DNyclwX@B&RWFdPKc{D=NnLydn5`8_2yQs21c==vLWSWfdu$gdk z&H{bsBSs)K`ab~Npt3a$>opyjnvT>dp{84`>0a(y?;FbW4Xy4F`u2-``-PhQ2}_o> zFB)%-Cg?}*8kwe&oY=g5rQ@gD)^>@9-+7|fRaw8H03?h#BQ-*w1QIk^fEob+!6Z&3 zH!oXP-pslh7j4Pm=KK?&5QP=Kh^q4KyY=4u8zd; zV^_nXH944^6kMI6tMf&QAm!&dX_hmQ*{@1(}P7Rg)MzYy(#Y!1l!hXli zz<>kh&njRqVZh5{9#kufmz zIoqp~vl=J-vr$kgH1IfZ{Obs?)6e0pL1#np2NA%0P<#UdJaqXc0Fn#2b6-V4V7pWm z2v2Z4Ggw#_Bnd0WT5fxFx8 z0pl*3TT0Sin5l0fOg%z8+JUKWU1xhTY)>gu-?lB=wk>BjAzu&RD+EvU7Ub&z8N8(V(OR-?r5nmSB` zGni0#a99|)PaGXaJV2}R0PKlS zhpIrEH^iCpTz+Y-&^l$eFRqHyYuiJ3ag+uM(u;D!5Ot8k6=+MrdGzn*O&}{!7gbKd zgyOk;0X{lzD3MTrlSnCwwn9$4$IV*oM9QHqBHu)=xE8ayg4|N9u+x@+Q}EYpi=ys( z;kGEeLP?3z8GYGNQRb%vZ`n~<5Y?F_@)k3<&sx|buMK%&pGJkRHyECojVTEihP<$c zlLSyn6{qm1Rlcv2j9lPKa3UZXxY%`2OrhGV*Tv(4lbq4n^U(kZap9OOm@%lzl60{M ze+<(3;|P%1=E*5Q^g4$yNRF7p_;dt;R_g1u$S1>gOv2++qAy6s3vvw(^++P}pR4!; z!Wj9C*agL+xLgYZAj^J4UYTJ*;Fg=08VL`B;5|(611Kjo zdK$ByhFtY)L~wj2yh3k6!SPu@796q83HVp8!cO+|KSNYG!I79-YE|J8L`e^43^3~M z&Y3hJb3l`-?xvGQ&hnvQ+(a-~HDzp?5UUB@6I9i5$35-6-`B z@PK(iE75O8^o@gp&8Hkm92|W<91C(mTl90#Hi%OWirD5tDp-=#O&i+H>9mItlv1rd zFH;JceR922U5XG@v6QY%3l(8WQ%VtZ%2?k}cTcXk;e+3e8+pJQhH)ZdtTIZRehyfS zd7B2SB90Y{vlsECknvgL2980-h0`G`08EI2uViFxm`lrCz-TL!_4isU%2DKsp&XUJ zF5;<2u2fNMA&&G-SsNFXwV~}1O}!*WW!?P7Fe+=RGRC-tzoD5=JmAUoDZtP2&lL3v zrBxuP7*nVN-f+R}zrYAVF16R*0+JWNsN^jja}FI8O6=R{d~k+prv|Agb5(au|4r&D zO;g{59?2?0T|E3^x_o5N0Dr zK40{LpASgv`A}ryk}nhpPsJ|qiZCxRD6>n}ODYx=?bf zI0zipv`;c#4WKzrGKC|)Dc;XXhKb0`b)UqbTBFx4vwNqpi)Rqvo}ilk__G*%2Z7o_ zTi)1-O^~eEj%W<*qgQ0*0B#Wq;|CAL+JVN67D+`>@Xw%J^dG~F6%+;k-r-Gk$VR<& z^Y)B+yI|fSns=nlJF?DA$!@{fnmUzncBKn{kMoLxrXCdQ>$9@euf)JtF!W^$xk~g1 z*_t{u`#o-GOHBz41H`iDsap!B%*#D1!-8kG=-CaCwnr}=*}qB)o*~gQM2eT%86Vp` zX7fb(A3{Bg@tOODdnUn2kvE=x9Ga@Sow8DP3=9yE%* zdj;1%(X}t_+LurDuDiN2uC7$<-gm|BR|VG|(X}V-+LJRBun<%r71&<`sUVM8t)tx2 zuYUEba?wR`dT20dbrkoBhpofaqqP1PG&5>2{lZKG z@BrpeDpg3)+)I_bV2q?v$cb;bz*q?Js^<+TDX8+QkqtS)6Q_bggh6F%Tn~zCXuGNu zJ_m$2PtSviB30ZcrA-@J4g3}`uR8RyVo((9HSnnVCfm%_IH4lIevc5&_Td=8e5Y>ST5DZse#D%C{IgEhFq!hwZS!9H%* zPJ8*u=8%0ekuAo{up>bnDe5hb(G=^S_%Ns6MZd;#$BxidaKzA zQwnRU=7FACRAv;O73TSYQQMb%+n5g)oJnpeXV#V_eNx1;4N2QB5paw;(du9zT$XSVk6mf6^1-au;UrG@M zH8r^6Zf)Dh8gj;+6)1kg4w_QLvSPg2)lzZSGlC``_8rx?dLC+yJ0^8-1lz`ljPoV# zOJDvSxcd;T@!Zb0^X5!1^8Mam)Efzhu6zAg{K1g_d??_B6V!|P-2ufTZ!13nIEWA> z%TI~FS(|(1WMsl0@+z5L3=?n#dM-HK+rp0mN{Imx<*F=dd;`Ol7ikJDAEP&PdrUEgPJN&WWm4FX~p8&@N!&6cIcd%Ro=&7-w?`kl1 zLCpq-b5!b9!%$HWy&-+Eh|hmM8VSwD0=_~d5p<+FG^@(o3bFtC3f5r$1XQGI*d-e> zI|0rI9-$#3C`pz`I1u%Pf|mjk6XS!k1coGfL^4Gqld)<4HKKxt25qPt8>DnjX%K2( zrKa*-BP>T$vVu?<3r@{OW~04TGGE1GJEI6<2#_n0=<^cIN&1)%993h!m}HDjMEF2d z+5{t{R;-SyEZv17Q^8nNV&%*}kb3%-qh!I*c%2o`?rSw6TJ)=@RX=k}A zqPQy;m^}3^vwPE!HrG^!-BTtf2RWh^SnI&>xv1K@+sAJm|9Et1^ri**w!}o%QT>C9 zA6-nkZ%!{vC-hlY?QQEV^xWd^gqDvycx&~H;2IWP!|SdS8P|!2!w)YDu2Z7xRKk#T zxPNfrqYI0dlO1>Zmzf97)oQ`w$kckbe7g`s};Hp>K$Xz9G2Z6W#A6j9Hsgv^Az}-Px|b^{&I2 zuET3bJ|9b;{f5x>p4jzXnr#O;alL*|rhX6f?vI!|mQ=^RZ9?4+v2KUp+L7LM0zBaw zKqLf^7)3*gcfIjgrt#QAtI&8xY&?@VnXPS1hJ@O_#0aTuGZ>F8j%>9j*?wzi5q{~G zvmXyF!2@RG)|>Zcn)j|A7Mc%=%?H<;4`-SWXE$wLnqMAya1u;2EyOg_f;F}{KpgWp z5+hJ8>q^@i0l|80cc!*`nOUKKY+2bQ)b0^$_dvzYYSHOUJNqBkY)bCDHM$7DZ1w09 zJyqMX1kb-TQnd|BZ>2_rnk{0@mj6^+f9q^=m)O{!Zs-?k2gKR|IjTc!*do;Si?#hZ z6XkBqfl0}`UcV(%zh!xJwe`nut)3O?4~g}M7LAYGRZI2B-6=lp?i1X7qPq{q+Fg6Q z{uaav?l#fgmVit+U6ZMR;O-aQ{l&A<(!SnuBGYo>;fT<3Mr=8gW}C7-{p&qPGd)L9 zVs)hSFz~%N484vt+X+M9X~NMvkZC@Ub$Qcmdm+7Kn9P*DdYx^`uuaKRDOzCLM7C|6 z9muc)0=rFQx2?0gGVHFEQ>*mqPJuljvIk(mEl$Ag@+@9Syq`92maXgxR|@ojCn|=Y zz6+qDl^r>H;PE$JrE-S$+yjqXJqmtv?5|MSaF%+6l@Z>Q(5VY>#G2CWl8>i2npY$%K3lqUHu( zctY^h-j36O6I@|B)|+={ns=}8;2GS!Uu@nlI1Y%80~MzSc}GGCEZ~K=;=r_png{t( z4IT6PV&C#2$r)aeQuE73kvZTiiLaVwLgtwY7&ep#e~|JR3}Af8dyC+JLT!+5-WbCr zmcNR9SwN=Lg8`1g`&5a118N)-D$&1BJnO!%o75?+I|W(;GtjR=<{D+gFrhi%*rtPF zP~)IzC9`D)c=cm?hW}lNkn}TtF7GTLRV(Lx5$j--72{GZFg)tPUtnq^E0oXc&ZU}Y z2%H>1i|prP3Lhp_$+7SziyGGer8wZW%2il58!eww&KCjcb8btywOE0{d<|FP&!H6f zr+ZsdN5qa@f_K--Wx=}#HeEQLJdI1=N_m!diCgxr`h_hAgw}&={&ekeq4u~~dpu!% z)Y5V1yURz!{(Y;Lh5mzUBf^%^^xLP>EvJQ+(_+i%G`z5~d2nTT)%cT9Ve|eqx3Kw8 z8eTH#?7H_(mTh@#cP!G2L&-VfO~7t{Vq&_$`vE}O+y=*7$9}N%Cx#QlX=i)7V|UuQ zJI(I?O0VmHbT6I{J@96XykX(=7A}P%P7;c!q-)a3CFDH`-MoIDnKx8e=kyd%*B{d* z8xtrA%WrAm^9idL);O3E-`!prt~s5I@NjQ4~-cP?T}*fO7I_PQ)F3p@G4QXJ1H70Szfm zl}eLxki?aMe$1z29)t676l0K5`5z*<4&Z)~##J($_s7tpL@ZkTUCeVC0KVoi3DT8h zf_o6&lr5u#y#e{=hRV^7LY4(GFDqew6W&9YTlpc-S+{jrt2-~k4sG~4~epvMbpN{So8m^Io*6*sCq-JdLvI+iOcv~>|Wfx*bR*1@FzpbeaU@*5y-6CnlK@i zZAv&c5*aG#$gy+!H%w%3Kp@fJ0Ya<4sZtfybI49!_T=y&xR3~uAYPmVp%qH~8xEi6 z5g(8**0_P-fV_a;J_B zdJflr7LYuD{cl*m%k=VKnjL^OjN||S$ib{3hvmt;^D9@@JP%v{xnGCt@u<&|S0j{Y>7 zoE7tx@Xv&`(wEQu0QhrsWINn9yt0Zl?{|fVI=Mx-<-j2{ZTkAynIKRML54 z5#mHT%loGSMQS;So(O?L&0E-tqg$2ZXc%7R9a2JGugs|pdHrE

ltqz)>{{Ivzi- z7C9Y=&GKPQDZlq@ARGm84wY?Q@QIiSc)tar9sJ1%3-_WEjh5X~6>94><`sV6>42{; zn=q7EUHTKbpNapJrbX*jGgM6?nN2M8NOCyMe8OvqN-tKd5ajnDcntwsBJ#r{k2H@73q*?wtU#s4*eIY+Luee)(zd;kBcmpA`3=QuC3_GH9cE`g68d49z(T zz+)Kkc+P-9Bju=GIF~cwvzc<$Lc9f^t(0@q;_OmfbabfYNMRX-X_KKE-t#PgD&>j9 zRXGhk_zq(pwAC3l1n02|#`IXlM$ z^pm>0`>K7M0oPT7YIp?K)#8x3I7Hl4nQ_yzN=!1G^+N_+P~n3r;DX8mv=UcS8|*3$ zz5^O1ufh?@>W2@ZTmi*;B&_)g3GQRK@SfuC_FM5f{cZb~8Veq+vTq~^4A1`(34T$7 z(HdDaUZ+LowxaKs=s}R#SV}Q@YF=7Y44d!*2aK$GJ{0?@5>GZ}Ok$=2q1m$|y>8ia zjm#VF50WjYewRY_;6kHf(H9*49D~D{)gFxc)H#-4Iz#vFyzALu{=P$XK`Xos@E3^V z41!9?W#2D6=1~EMSFzt03m=?N9KF5|osIqz$O${oFKh6Pu!+T8g1HX9#;|T~&6ryS zbGvA6Pn+8}IBOL?>`?OYhK3IG)5?09(6uVtxou^4rt`IQ=WEHc$+N4o2o zU)ms6GI&`$if@+ZgM~9y?UX5>QgX2VE5JEt!#T{S-^D4!DnJ=xb1v*f52;t+Re*I* zU&e#)J6yRxl=qc8mN@np6M8Zpe4FCk`wfC;m+0A*I99|e;-D2S+%U-?jzA^PaGQF@ zaBA79l7U5?pE9g`BX~oU)u@4JMWa}vEsZhrbht)S1~*7@m2rxy^sR`JH+$!yRjHzO z6flA|k{uRVl~UN<&ne$X_|?fstzK&kLofSt!V<^K}FcL0ECGB`CI333txGW@Lk zj?fw;z*6eOOv)OG;@kjv7x`n6m63PATE>;}4Boo6`Ee5reCb-hZkv3ne_#DJc70EWBU^5wh-YP zAHA{Yzj<=uWP&a@WmUrObS>__IhQuqZxqNCaU={a)e@?DRG>p!F$r@t5febjsHh)? z5s&LZJx_ic1Yy}lPCuq9bnffrjat#ufZ|3OZpi@D<_pO;zJw(I1{ywBO}14njsFXr zfYb5>5X)ima>nLO+r04hRN^=orbL%FH3;9a01KzfoiHex7JJp=QCS{L!IxyZGzu3h z1zF`UR!k_g19d3l_~A>BBd9F^(|oB$9pfr#-{sLI8on|*K6!MR{OGC*v0p&b_+H8l($_E~NfTw8Du$%w++U7asKm_S`b z1Q>xAvbD^=q>sjURRZ8wu@{79(FKyn>F2Qr<+=P0OhND=_6A3wn9Fw{m*3Ewb=KtE z4TioPg#bPTQkQcoK@wL2_81JClC^i5auhr)&pwEghc(0JEO{se4vV2d&JGW9c6dNm zx)^wDC^#sfL3vW+v`_LujGgPPl^FD8=i?BNjPQmec!CfaNiu|Ir)RFiH~wbG_9lOD z%S@c;w}!*K0Sd@wj2iM$s;%%*s;%USe3Xj3V9DclEH#x&-2KWvPy|_U|z@m@fC1rFNvXe>vj;8s=joNSt2?KEe2})WjpYGc@Kd z59Jb=_Em9?knW70MvKXjK@fgi2k4NVa_8o5M6k-K7Ld zprcfb9;RcKVGE7-mMAmKkenU1lH4|IgPe)lV)kJ>>BC03SjDgd%G(9|Eqa(2b_f-a zI|aurYS<<404oJ2K)0|{sDzd($lZ{8gk3@v;jmJQyAH z)4C-d2k01`h=+{IFAEcV`Qw_uIs30uxB5a%p$c*q{&@e*Jb=4Yin;>1k*)xaj5N)p z7{yY!E6)Y`W9aosv8Q0UN=BEq!$`$>uHY;rDb_-%JWtUn%f~SOC&f7s&zH#G&kXA9 zsYLwxSAT-!x9JsuKqwlN8^0)coiCPIlFjl>1rc0-JW$C5fL1d~DC9#2TIU^L)o zbvmZAF)B0fsiPw=$)Y5W4B`-D!5ddcPK)x@WMX_I5sy!f zMB<~OG!l(mHcEldC=rV#;_|-no4PX+k0b-e0%Qrv2iZVA4(=ykT5tLIZw_D?OXA8eIhT?WY zzi(hT>MPvtP;sx|rijIExVcLui};`bqbYg{b{!xrNT8>9iZbI2D9SSU6%GfK!$jgq zpe+8XD80nWI_s!>5Moz?ziepEHJLyqKU8!Iow>!-(1rzSEDk_L+CHRr>?8f zfDpMZN?|b`5~bbHx&{97s{qo}V@K88!Q1;Ym(+@u^vN7+%W?cKCuVPEsx#N$->SOX zGW|Wy!?+%YTqkb-zzHj%GZp=YD=G7vr*2< z^m7J2%1DO7RS|sr913P~D(!u0-JSty!E=HI_j}rmuOg|Ma8RsGtrH^rx{-Mh1%77g z*ri}>JSw(~N{LuYa!hPFI}y7qMq30Bf>Px21XAeUmT@VO6hld)R|2+jG@7^}?+YLK z&-u!K`O|;(g})yA{mkco__tm^BelX(byf<3=wJXth&y>KI4-QrMlx$BGf8hQzssdGl#5LHfHuqH zX*xw;qV8J;@53Gm$(yi8HbJ}|35k9y*#n&=qQU~Sk1|L)008!rp492EY4@tib}tZ( zfSfsuUEwd|Gax&->bVnl``_(<=iK}`m21+trWLMbnQKw5eRyN>+Gmqlu0`dJYuxel zi5%}tpCeoM6)xSR+3Zk3_D_StlZhB?3+iW<0!&$cW*KElI^4X6(@YXsBcA{ZIR1vT zUMHHzI8D9Y3L=e8v87q2ttl&rEEs9p2EvOqRs(f5^J+javd5s;?DiBxupuE2-DeF# zc1(#OqVzO2+ogk$>Gp^q2Y^#_r!0nJVmxUGcSaJEIvv60NzzOTVqB1M z13@)nG9-Sb90i02@{h|>KXgF)mwyTX5a)1fjxCw4%n)7*O|EB^=apx(O`Td}m+I)& z9NpT1$lO=}cm?I2)=&UqU$t@9l-b}T%vdUr#~1JBF_ z@ygUGb(wA1>fM^hKf}zNxNV!U<@n0^i(nv@KQXhbgXmXp;)GeKZZilezbpRXi$lCq*$ z%IkXCV4q|e{;!lY3=>|bV>V}9lL)k!@t?Axe%t)HWN{{nx>6Jo4;b?Z)Al6T6omwg zZU?g$*853u&C^_pOW8-!$Y%#7q(nbWMCpB+h*iIxur%R!sRNp&XAq#^)Y)hP3^C+M ze}%#E6_bIe7!N1MWa$LrMe&(+k3@FapN?iLPpg$L zYn3mj&#pS^GoH*@WuzcSfa&M#Gsm)Q9pU^hVKI=_`i#QNKO8sNm|zJS>fkcq>dCSo zNd_@MiGDVyLlEF#>A*d-H-&<+&g7|LceuyPe$cT>n-k}sTMItQi?Or zWa@=PyJ@em1y%?cDmj(Y6-fw46w(#gPa~x&1QsuXQ+U_#hUZchinGwWJQu)I0Xx{t z!xgCt^ce=F0q9D+1H#EfSNmOtuC@hN=pg z4y5~YuIjg6yYm`|v~$X+>Uv6ZJq4A#Cw<{t?3llVef#ruWw}lS>jI5W%f~Em2K|WZ)B^|s z?=bd93kMX@md0PV{i^N#-S>8@Y#nH#$vNMut5s=R*u8k+;j^l1NOKM0COc=YzkT!0 z&CIoXHx{lvn4Gz(x({pa!zzC`%N{nizqmCpGKwcu(ck<%fIn;d!BCmDnYP~s2-vv& z_Hva_ikqQ{pp>r!<~I@;Dn-!`<)VOIuc*S76k`^G1r+5i3FJn@oOl&W#GEK0iz9|a z30XQDoQ%kuq70>V!FVv-g&L4t8*vtges92Eei6X;#@Q-gHG2eXf8Vbge$@a5L*0_+ zi~3B1+Hg{9IGL?I_3%ZNAJF)LESq-&iYgFh!QCPFQ=sr9)o)TNpf-#I7@*AiXI4cD zj=LeP_W-|NFT=+s=Dk1f(y5VNp>N8b}6IN@9HslHCl*QxSd8sD|T_bl^0OD9ymSL1tE_;btrIgnxe zd5u4xKAq#;>GK-}QjsJ676u27un2U?ZGvbqMjN4JnlUI9i%zzBxf zAVM%9#Y08(!K#q#2}}JQ6rRLV>m!7l>zD7zEbHuKWmB>EF|6hj_#0{S!v=F7j4Tck z(@f&wDZedmpC!e(oRHvLAsQLi?UG2gLfQ&lbXzhJg^(k7W@)&Uaa-~Rk0%7@6rx1J z;>PAlzkt>sW4C?)1{)sBRH*!Rjo-e)w=eVU3$)62Xne;C-@VLtFLtZ^L5)AS!XI7c zkE;AJjX$=+pIYWmsr(s@Ka*w880=V%4YYv?HjwF_jE!<7BhUg37P0{$4_m_P>~yo( zD+I{$tOjgb&T7*jCJmnzxzjLr$#yc*2v)p~0Jj1*4TDkTo5smwkO!FL5xo0)**`**}4?W~M_3+}u zQ%ezzJ69ecGz6yutzpAtHXWd#G9^3s2nrz3=yc8Zi%3AE8=uRklE;XCLAfF?d4`~( z;fXSz-;%OyPN~8=nuiq?m40DH%!PHBL)n407{(B~pk$CYi6+?K2m*ZmR;+yqK~dKm z`h_u!ushg95QQ%XYTeN2EBy>ns4jTHC<-r=}F zt@f#-4zwVJ35lFG9&LH;^4^xNy)DVW_})#|6T>Zj7rKd#@oQtw}`_b;?AgzmSi_1#)M z95mIg^7YwUJgr1HJi z`CS?Icg_33%otkqEwwHUE%_d{J{)@J%XXeqo1fE~pHsa9ns?w)7r8=@q(`Ag2Q}|S zmA`0=+k2-s)2i}YHGb;~-?q%RDc2U@ey_&wT}Z0@fpQCkAkcK6V}lJK+&@pazuYKH zQKkrNQD&H_O#;I-Oixqg5xZoOZ}4#Z)_0iu@N60VgBJ}eDSiBaGUzS0smHpg^E5;a z(L9Djpa7n0mmIf}R8Y`j+>kcRWd*5ZRmxUhdc6-1IJ-^~?UUoJs14_^0K-Sr;I{P$dU)I_8hn%NTVU);k+^VM& zCWU9%T7od}aZy7`$Od3860?Ptj^LeTFrO0~N|&&XDU}Vo^Pce>ewE>G1!Mi@IDVjl zf9CeceYjV&+ zPS(hAh!N_#ik&gQLA-CBxe@_7l0#BtJV~57;y(~OnAp0~Utw3GfKbX49N*7Vtq&@I z2Vx0fA}aPupTjN48cO~jfNKm*(>ba&yV);C@o92Bro7q0FGo4ksZt2oz7f5kE UsC45pieRUk_TlZOfbhuw0)Gp*EdT%j literal 0 HcmV?d00001 diff --git a/infer/lib/infer_pack/__pycache__/models_onnx.cpython-311.pyc b/infer/lib/infer_pack/__pycache__/models_onnx.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..33eb5462848cf1672941a9600a6b62430e7dec5c GIT binary patch literal 38568 zcmeHw4R9OReJ2*53jzcIeuEzpD3Y=$QxqvnvP@HwB~h{@OZ0~%CsE{}5CKw@NRV29 z`e4Auy*dq~GBxGUXV_sbNNPKj?e^%qw4=7kjoPMddUJD!Wj7UOXEdXm8#kj~?}i#V zGoIW`dcXf$>|z%nXgP6umuVKjAG>eg_rCZ4djI#|@06F9nQ+|Lvv2qx&Y4XAgD%v= zmOHucwwO$>nZl+pH)=Ym{BtKcvnj%zHLH;$mbW>R>0SJk`I~ zG5#daQ6832=UCavvN6|5*O>dHd#wCqISaQU+;g&mo!dseW4@EVvC5N`V^t@sSeSjZ zdaUMT%~}oe}=* zTD6XEqh0Ufm#cAr#VSKA*DA4ESS&YUm9G+O9gF2btcq1)wX#@V#PUTPXL+?IC2QEm z;#VSm)dR(EXYs2MzvhABuV?XV5x?$%;y)Cw5BtLn;l_E($qg)R6Vf)X(&7#lYYk$p zeV`IHviJeSZ+W2j53~5|5Wn?-;&-z6ZHV9gK=C()xnrh|^>^t}4Rmm_IU4P-%FaC( zhMyT9AMFXt_I=~g3+n%tuv|gc$AwTdHZdNH^n{V7Y-ln%7#|srhDJLqa-|R%8@~_< zUW|+kpNR*foCiX!SS)N@n}qTjE#pUMp<#2_@}9LQRk?aU!G|s9Sbjo-AqS5dW5rS{ z)VuK(^$a!jspseu3)*f=*b|Po(c*XWtu_TK)mVn|v=$>R7q(5I*TQxq21E~4t7*d~ zJb1^zT@#+zkKcHmjz8Ib{5mH)Mxqm=p~1*qYSdkNp4V$+TWoS7BFKC&$cA1}c8^cS zgM(*6(P(5eCRdyp35O$5UBJ-DXgnh1102!dnMf!cllf?HG;}#4#AMgGh=A0=*vM2w zc1{Gt!uSMo6=*2P_9G+FNJx;Kk+IXki8JF-L^%=(oxA*8ve8NN7Tk&`e+r0U{ql zfB-!jVI6>M9_p5z!QkL%C>9F_V>F@y>ha=F_wDOFbs~l~of@DyekOGB+^POZ>|A_& z;?#IFdg;_ibSNU68XY;UoM9plo;!ufjlK&~gszFpLL>5=sx!{RF15d@iueB>;G-8! zKdCi2Jy#qvj$3^F5??R!4HDmw;Ty6|8XkGY{*t}b6-kt298Ms~7BGq+d-HoZ>!tmmyn~gslxo0ugmwjjgKw|e#>?UM- zC^f#2xDgUJ@4o31tM^IO`*6>h#OFR*W*XFHJuWny6U8um&6F^QO}SZCn=qv*deM>K zj2z0eHTv*_#zj0D=|ubjhQ(BiUUs%Fw zj6H3QYxo8E=W(1h3yB;%9(B&I@q`cdVyS zTYOvq3UY$BD-40`oQ%fKPevkB5rO&)q(6eo=wKu`7K)ui288|0@p0J}j$a0jeV9t< zWO>CVMn~cdJ6(4QT`Wk!gtiHW=p3Vw=y3c@Oy(}h_Df2BwpctAN646L9UUKz#fXlp zh$*ay?#z0tU-P`;Nll90cFEhG z+>_jM#~8F={r8@CJd2a!hMm%eouYSVa!=M>{;GA(nQD_7xQ>Zwl7}1$^DJ}FYGtZ|GD!A zPO;^%)N=TTdqm%Wka`i~cmwNv7hV=8I zyHj#^qFhwi`HC~ua?L&GPF+AtlC}>j>*gL#MbgKm_U$+JWGZ)wmAjv%v5fq{2nhoM zF_OoH8bE+pQPM30XuAEHAwVavqkzc&njug- zhW!_^PoFkJAZ$Z1!ea#b0al=?4AmRxDUIqJJsqZm0|aPL3r`bxhQJ8`Jtg=o1rj(+ zC(jdLDJW=Vluc8`DCI-W=b4;uVRF7RY=4P^s?Gs<-^S#9JH);+x#qb@Y$ObNwQqb9 zTt;+w@96m8xw}>>6AY1QP~#(UL5Pj0#Qi@7+4@zM|KCO<-o;NjfZ8-pI8I0_cO>ja z{<7cn!f${-G@aFAr!_8DO+){25+QTPSq)i3*P+Vt4sf65w=sF&)h4j9BxpuHSt%Nq zYiwCzt}`Jf?OO(pLzCnl9?=Qv&ap@+3K?|ta)(c7K9iG~I0z2Xsg zgtP*(^}y)JL`NB8cZCpT$^>(&fN+{XfB?+~R;0|0C=8WRBG-|cH7kyEjpC3%7yIu3 zBv@CM5Ug{Ovnp4s5_I!MlB63{!8#{q=cMotAn?9$EjSk>;Ta6=?8R$q=GH8m+|Dir zh&}es9KPjlUUE0z_SQ+>P1)+EY;|k4dTqA4h0a?xW!J1*uHb!@_e=mE61Zn6EBAb8 z0$66C7R;`i%QuGSM_-T5M@9E0$-ODVzRP~KK!!^O4h-rgP#TA9X8&89K+3kpnlS=# zt{fQo)zdaJ>D_Xp_&x4z;D~n#Pq2eYPu#S9rY71Er<`(!J{RL?{3LlMNEd4!xNerY zVc8)>Eu`rd87&79DjXs(Mj%R&tTX^+E*PVh=Hwt{a2!RzRP)M% zBZs)>5ha!YfFM*+yKJ&KcV_FG=F65%T=`B0X06$}#@RTemkSHN#V0d$ePUgoRM&US zI?K)W%=Ro>5EIG|e@nXh%?7c)Q>yQL@YIk$lDxtRaj1oQQ0KFUb_pj&Wh}Z~i&@m| zZPQS@<`;CJB;<(W%VWN@?g6TW6)Jc+(@Z-WnN-0Q`9^0z4Yet;KEG=t)E3DMUk`P$Hh&PN!?&7#@!sh(?-lDqkq zyL-vqExNZz?kySi#UseYWj%?Sy?_iEk;BkBaNQw1OV`g4I7WaNR7THPWrgE(SxBi* zQ6PcK_{By5K2jyN7fs7vldm#cQ@iX7*f7^ifqu?m^DUcaEjUw;-O5Kx*Vz(RW)Wgor|R&7T>NEl@u(6;p?zRuTx2d`wthR@KZsmiqj{<{Ld?)lR8uCkQQn zyQ*QX7mDbtyDsbYXWi9VfBUk_UR`$21n?mNiuM!F>iTGfR<106-ofZMV%Euj|1 zf=pT?ndl7dz$ip+dG)eU1qxL!aA$t8s)X{GJ8M@nH0H)ss2Z!O#-aa0Re8cnW0UAZ zTF>}vNibuq3CnvHJ9_ ze9zqS9b57pE1_6Dc=h1yvsa#)c`CJDbO#h|E(=T9(=Ew^qPtCUx2>#fy}hckwcPV+ zjpS=dwe$&do`3OS6rISl1WN$i02l@K1ZjL@1Q0vJWOY3w(^}N!#XJmke2gAGm`L6| z!{6dtm-tqZZT5cFUdF@uWF#7aKshcb9ETpC(A;Iuk_YxI z~FF*lf+G(+l84nLl zxe{dw{HD;-u$^Lo8W{S|SUcktq?6A)V82wUwTgGBXHem2PX@KqwbEFM@e2L(*=grW zZJBnL>~k!`eoUWlV!d6?pljNL+GsUixz8b{n-gyIoySO}jkQG&bR z31OJZB=w;{HJ6=2BzBqTNMN!JI{*tkt<+{Wx!ZZ`+gYsBVmskxhJ<8Ifl0so(A-nMxDM$64- zmn~*Ln128)5BrJvFUJCLN6+HEcPbVtUVT)mYRgo$eg8Ak&VeNV0Yr+O*LKhCW}<|D z-CK{Oe>3yw5wZ2C)Ou9(4@mxjWs}PZ)f>Q-L&-f?E|8vw6nO_a@V%c?wLK0p7Pbq zd2V^zm%Q!iGwCx6!eY;2NL;@|TE9c|J|=k|%dqdV1L=NJW@-qe&FMYq^9wELffE`3 zKGDBV@{^g5hcQ1dd#-w3O^AG(#J6SmwmYu6RL_#DDdTFoUEeV8Om}9w5B{)4tUn^v zAAw=fp(H-b7L&UfF>*2)!yy-$GAbcokW$WNiVA9` zm0eV`!_&5e^+lL2BY%yPfNny^Z;(`k^)pm>l~S!U>l4;rG?yvx5YyI~r^v)J&+kkD zOHOk$zh9~cjcsBQDB;zjqqwK^OFVi0r8pibof!`fS?_DeSC}U-MBo(wnL8ktjglo| zkZceYt%~c&bNi2+2p&7~^fUVf(xnU62)s_<4FY`tvNaxxp6RgbwAEvDk0Da?rI_#? zT*ZhUDp*ser}pJhW=^PcK$eNE1AtfodzzzHkEU8gca!99f{f&Hlf{m2w*R$5uN+E+ zt{s^>B6Yf2ob;uo*y8&=eE}eo0wnv+Yr(g0T-vl#?AUpe6Fc@u z4SR3)hzMuCn798@JnEHI98AVn?4A@^X?BXwA2P3kA?X8LhE z7R+-C<+PIo`&yj(9RuOAh;zsqcD-ahS;iL9T~HI1%Z_8PQ^vCi&0s+EM}`;&H69I& z4+Ubl6Bt&d+cKKexyT!c!DP%=XW&dofUW zj*rA5!B{-Z0=l~&F-E^IJ~9{y2jgeJ`kfgcWwDGghDJlfLEwA9_Y^H zj?SQl^ahBx9*IweBWfRQx){0;84||Fbcwoplw?|aQ7JSUWjz_1jKxMmQKc*NNzfZg zBCQw3bKTGzIM5x)cM6&xII;Qp0JG*E85$X6tJ=D(lx&P$)r$;9V)O*@v;#Psj77re z8<BI`(CXhev`1M!iu2$~w1ptgnr zN>j8U5sM6u!L*$chDI-jF2`tqY=};DO^kE|(5k`l*oIt(E4n1I()#gFfAj43;$J(n z`@o<>n;Rw*%?@H>?*A#kr>|TgwnDe714KTrP=TPTQ=0&pYAh0#BvBjO&5MP!S468f zsfzQvJeaBu3y~xvLY3NpKjDuEF!7G32}om3c51^&cB^BR=9@q|LRjRoAsHQJGg9_y z!zVX-I(P*wJGB`i{23MUCj_YTgzr-}9^=DRL?Pnha&uWP0(ozWz6?zcYJ>}=(er0xfH<2{iyB2Nl9_gx=tCT9vU&1Cj9Q9^!T2iK9vtf4GI?Rjqdf|C-A@mq9}!EoubADEvP$)vmZ~<1xYg!>L&Hpn=Rf{HIJ}F|1UeWaXd@L%E5d+klzi@*Rs6NOXDL3}f2D9te93lm~Iq z3AWw{=|IcZAd6VT*ah}w=qJNgKwN#egYGuk8Yub*@8JeaHEzZk1_Aqo%@0A&~V~P|~ zYEuOIT$9+suDnEq{wi3n;OswFmRz-}X4u!#X{W7dRYs*)rs!}Ip2 zJ&!bonixityI?f&VHfmuaI(wI?1>%Rq0FF*`e$9LZJ{3!aZN;l#wQ zm;%A;?hR}>9m19^QcHCPl*O;k0Crj^1UQE{iF67P3I@3i6SW3^fi0F{7~x0bq0s<=GF7U@D}+tSrm@C2BSxYzG`xDtTvj?qqqlz*hIH3e_YO$x^aQ7{g&J8DzIO z_I6rn*OsDBrJL-gN2C>f>$T8GrnVguAxsYrTkOAU`w;-EP zI2alnl)2N;6CyQj+e4lph%uv8b`BfzgbnHTRGu{DLXKc8$A$nvnRpX$B5kbLmu`9e z$%QTN?YOb`dxtXa1ETwY2>&> zP7kIBQ|F|Hjkg-Qmm0dohAmRVmc?UY!;TxRV#O}0V%NhmsHtxtFmXQvS)FfSh-EA+?I4?dDmCZ&rHsKCN;X)kg4bs zEBYirzAtIXdVDXRyLv9QZhk|$Xa3=|u+SrVHc6gMs3LZtP^0?i52nw*eq_=7UH)d} z5BwSTQ=)$SypZEQ;Nv$Y~N+#&jzw zY!QIf^3(?G8z@kz&20tpGcc64dZgEa0S9Y3FRfsl&>EVN!szjP;Re01^*x&r!#`-Q zw5<=V(&zYJt7~jr=nPC=fJm5A4s969 z2?2ybv7L7^4gm-P42zmaxjP|H>X7cvoXn-7UFCnMn{o|IMisW0$tDmx&Wuk&ygVHV zbZfX4dv~cMU9CR0$UL6UC7!&ndKz6786wYbJAo-O_6tX1aWZj^4TeS|!J*I~Ei3$siYNwO3#ly zbyr7HT&m~F_{{jS%~W22^#Ffk>iGN@#QKL9&M$7ce&I&-_nO7713&bMU5CZ`!^uP0 z>gKm>QlNXWK@9Xsfj+UiFS$SKZ%vMP|zRi}u+-%+Vh~qD}bATN-g-DY;Ec`J6!iWbj zAdJ?ZvLhG_j}HceiWw*y5Nv!fsl$Gp%^=x=(V>`#lG3fnOoNG_1X}4)zJZgER8<(P zL(Szje%VBz4QjEhuR2>zbA;Dr|VhrNBD7t#a9<0k79qvux5po6pvsYM+Nb z7AFhG*xv&Hh1^hO2l3Mml?vP^ZfOm+KWhUB4N3P%Q^*$Qjjdit`~-z=<9 zOG5fFA;Of7@$o<`5@#y5VN@@=Vt)|U3n~rQ)S5~+>`REc9Cmm4T-@y$;4ZUKHdZqT(kFpFqP-9G?~@Gq=uBQGgn zWt2nDdJ$8{7z&%wz(%icGZ-If3yW#6M8$`yv;bR)NuE~-AU};MDv_TiWb1P^GP1}; zV;k8L$_>QD5?urSgnuMJO252P4t`8x!80Ma_aNjZJIT*TGzfJ(V*wO{jaItDk3x^c ze2rL-V|`Axj|SN~ouc()f?HnSCzl(hqFiB|24Uf!s1n@(u@b7khN8i(DFI(1a`8Jz z3v$7ffIsrAO~VXgy~MA-#cy8XH;a6a#P?+Qo;$fMNq*@0u(qyjJ^H{OfYEcRZ=rs1 z=grMn+HWAEhXyiw@L+M?zh<8QX2ly73+_d%$L+X*b@o1~z7KcHZ`U@?oq28Sm9cB% zbK?kbCGlaTVZb1O&^Th_m^hr(7TgQPg;Pj1ZJD-$UDj<;5vmiGl|>WdvuDBum0NT` z=+MLv{mnw=yFdwHhp7>aiHMO_mU?Z%|3Y&1^kgd=HOqAUzX4QM?;r*I2tamXB*C9k z=z_xO3Fj$L;s0RtGaMhCmvyLsWV*z%av@|alD zE7kNS%d!>z)Yf!$rh<%3*%Muv6*{i;L(y)Sf1p;gnRS%TR^)fs>|=o1QMmYLt{{Gg zz=j;ZQ`<;3Fy89riW;&vEd{!4boXO{!og7SgZOEOI$VBUyHqz=a>XtcY|DW`4TGTo z!b6n({XY9I1DhL+5I+sGR2_EB`;u8%EY^$Gc~Ka?S@2}v#ZNh~3(#r3sVU(M zn-gV)+XnQQJgzAcp$gEHmB!hR_*yKz>jTg)I~1|DupJ6zU?zi!t)_?sRpN7N5AJxI z+5(iODT!fXs}Cm;fgLch)yiG9b~S2>_$KuXrlceq=8gw2vAwStdNSyqE=S9p374W^ zS50G?5vW#Hnijo^j`$<_p!l}@xsson(@MD~%3(Z;A2YmlGQ-<~UNt^*;}ujBx!M{= zZ^5`Z$0{T|S89~0d4x@0Hs^+Gf@J{1 zh%uG{6OX_B*{O3!=RTXM8M zsByASF;Si-a~Q40D?brtob1K;s{EK9BI9J;(@S_Uzu6oQyWcA>GEQz*dj~bp`XfzV z+=`AkZQ02k#y-N><-_KjadM;K32Dad%s9C~u?V~rP;)FbqKzdXh1ND^%ItxQN#PDM zVbTK@ENPiQ;Wil~d4(=%qBH%K@Vj*ORRV0W!6Hy@vON|TAgVEYXJkiV2ZY}P>{zF8 z#~)D&M#E?6>>2@D@Xd3=GJhgx^ep@xZZQ9NVc9v1F}nK^&N`}4jyI4Id16BH)=ECszu;4KQb4F(4%;ZIQMEG%~jL}i%Gov=V> zjArus#`MdDj-@f(-;jUwME@tqmI z^J6WTKLDd%f6Kpd$-nXY=I=Rfdc+-1OFN#<_&19FXC(hK#LnNY@5nYZE%Yo7uVCPe z4d+d3=6Alndme_&aP{{~H;MK;55cmFpw1||x^vM78|o!Vt8<#3JW-2z`Mvpw19)S7s`S-UiKDzj*)b_a8)F(Cd%{sw~O5VUN z??X%8hi+HHp95^Po3m}37P)U7&92>$4RkC#`z=pNOYLRy%bCFJ%lb$0M5;DQRq&cI!zb~{37A&QIrY#KCI18> zy^4Sd1ETz{Tcf&xiJA~J$r4~?;gWW{7}KzwvZ}HMue1dg=gxxm5XxsqQF{(gmnEDzi*t>=$1Yo)O$1}i$|&zu>gt5Wxs%oR z7s!R#F*spy?lhE--h##Xe=61Q#x}7vQjPu0HJkkv_phFj$y-Jg$760zEQ%SMLfI>Y zpCM1CZBSO9zDAex1Q@Yo)cYN}e3t+tx$n^#qqhG{XN=0fPG^j=_R-lN0Ccz%ddk!p zB%`p+#&Q|#;&RkoE(2j#Dg9ql3?ij6e@PAdLkjy2K#T@jo=PgV1Z-hss@gy$b9#>d zNko!%(L&GB-tnCmzWu`Dmc{sYFNy6>NbOIE{JLb>?BTRITeJ37O~+DA2ecs1iZz?1 zn$1vT{M=UVF0yh5cyKHCx{YKe|Ih*l^0Ck>)@_&SwkHpgMg75R?q5AC_>c1=6US5h ze`W-_sMzk^f)v7kr1T{AUgr+1Vxi8MU;@8SgK!il z;MCO5a=pn{yIi-Qvpu|QS``kMYso_02)b7@PG_d;ijsJb}49^5UOiI$C7Q zBU{eNmaW|ccAOhT9POYUq2vA*0R1u?TXy4K;^E5O3|n=W?*T#+61HIp4*p)?*xdmm z0f$_7>~a*=#bjY692cTTW6);7WclczrshD-ggKI`!4%Akk5F0! zbt)-bY1BO7!&Q}5x))d4vq@AzFlB>4XpBFs#v$8Iw3RuBN8O!9mvTs7tHz8r7}QV? zm@3|*re`J5er!?=L5+gCRwk;$Iy+J9&PLC@o9{hSPqF^M=FvuGqx6ApGKErw5E8djlP9ik#0x{2w48Ed7Kn zF+q9Fgt}oE`+k{Gi*~)b%lp-3ZJ5|7TM^h^V@2}DOD%NOVvdGo8(a-x@APM>Nre62 zM>{geTv#Z#&NDAs!UWwX{#$s6BH764|FG=BTQWk}cnr_h6IhlTA0CE-KwCI+dUCj< zR^ei*sT1*@7?V&#fCfZKC;qFqNT{O>v?n7w$k`m69m@7|aF#tBQ}hEgvvdqk{#36a z2$aCNnzEo0Q#$s4P##FoPr6oiI(XYvF>AiEV`j&z@zjB9iMhm*t2N_lB|m7woF7&{ zs#l~noh0NpYTo`;FNLdl{Z}^;wD(flb=ik@(SKW3&1=IGblsj+@2j4fa{qVKvR#$4 z;j6ooyKnm+%6iviy=&RGF6*tw=E1Djf3JdXfH4sOcJm%i9)=63h7Au86RV%qSj%i6 z2Ttymth<$+x6!%39j5i!>W0)~Qg!=sjf-460enb+1WW9Zu3gW3KOr^zK$Wwvhx|ZQ z@U@5xfEv|O4eb}Np{3IrisXbkrM4b#L}-)NZCxA{ zYxYSsw4=EUOLf#s^|yJ?%kHc0touajRK|S*-<#Fao@2@>R$-l#l@neA4S;b3gBSza zL3p0#z8cv$>E?BL$Vnjt474+r0?Ke}0-VL`9YP+d!8u;gUFf zlb(;;SZYEPvvd0_-=kSp;O ztdBjYV<`(-@{Gu=<+YSU1-KpbLhCpEPUS+`4ifz%*veAEGS*->s&PCC^DsADu)t>I zp!82IVeUhdX140XO6-GE-ZK- z8IKMt264=eGPm6Zc8|dv-d5Z{Qbz$#D!Xo~pe+V9oE(fJ(z&sbOF->Y50@GqD@2~8 z@$uWp63(m|1M`nBILRZyjcVz!!=X9k>%w&lVz7wD zXQ*ItjVzHd3Qz&%VD`o~{c>F1Vmo1jm$d?`YPMrXM-PQ9y`>oxvWspKju7eze2Fd@herehCR<$;u%D8Yv-g>3Zto6I zm<^_&Se+T8bBlnDFOIPg^Nc53#)1kT7z0arh~1yU*>BMqHy$I+V4fkPGo1&8wWV-O zPBryK!YjK`KhPalh2(0?xWLl**b75?<{!nDk>2$AH!m)ne{+f<9ABp4anai+dHaA- z$kl>W-6ne5NXNu>ccgt8Z>#8SmAtJi!fUQq;6))NC^px(yq!zlPVgLN(ff$xedLz+ z@g?u$H};6$U6OZ~@^%%Uk|gJN0q-s;d!-CsZp-J&m6Zy(SLhMFo8UJZ?_ycA?we=d zIJ;1@Xj_~X*X)

=*gQq$T)?YiDhFd;-o~6$MzfiZpr@N$@p2cl9 z_K?fJU6OkjQfST=6k8&`z_wahf@J){dKwo;@w`?xD8h>(q0|KM(Nm$J^O35lAp`1D zoDJMm379ZjY#ZRmMg?V7+he4H$pN>A)Hfds6nzaX>tj}ZxnBKI zY=j?qhP>;B`$uAf!pPW2lsw)YFl@%9`Oav^-{a&}?kfc^+WW=*-3Ktz<|F+7ZCu3KbtS2tr?gvSsgw_jM1?0#C-HwusTcr)X~uBmOM9;hba zb@jo$7kn%1!ik#fYZ%P;x1xd=%n%0rBoX95_Q``^g%6WV_nwSNb-03z~3D=ZDM_){nHQd{!Zae~L|Y>Lf!_!Ol=iah zr&>O?p@t8Vklc0flCMAG-H+{WUhHR3dnA9BJ#wsIjT_>buPkXpE^Axn#}@XBYqm&hwh&|1 zm|))us%M$EJYKP!GV{0x3&!9yXAau{a@xcNzD@`BfeVJ!-4V#fRI z9ykMbuV7ZOdoED&5(cl(;-S?1EQC=?iPg)5t0<;z<#o3Em8W`T)ln#2ajBTNEec%5N* z65Iq*e(BIWA$J(uGQ;3ED0iz)_^?OEg|Xm}vYtkKIaoZs%u#2M%8qH|=Whw~6Ae=Y zjju*iVUcf@_|{u|=Mvv3@?8?&mEpUxfu6+^OM%`@p!e0r*~Y&rPug$OySVOnYiI4R zUVxvD%Jg#!zQsM6y6uW-kLZ0=@;;i)-9HT%C8y!QcFzqVQ@2;F+baPwXSw+pt=Mee zv-sT2%1m9qSl2HBdiP7-{c4QC^wxz+sCR^$TQYSA#JU3#p!Z41`=lBpobF#(w-CBv zx%phC?x0wAPy+NGlDvoXF<}36gUi&x1gTd7^gb?mA73talzZ-3oV1JuKr2yju-Fht z)xKp<$CY)g8=*|#2{G`51nA!b&zp$lN#es$ph4?E&x+xo0J@e*Mr2!wK;?kUvu7H1 zHjoAiq#dJ|%)F)=mWrCpk8LybgxO)#48xADgd?!RamF^3U_c6h_MpEtaN_yb20aKE8W1TwxO zZ$P05fy9(pZW`znmbC@bAVZ2-Q{d#Kv70=vn|)GbJh|D zZ_xNXb@izfcV%E^Ac+sD01Ylwv7q*{>#A#Z&lL}i8gh>V@3Qb1VHVx3lDjp@!LRCU zTk>LVJgq`f8J-~UL#p?4sJB5}DYxy#y3DF@jmuUC^SxDoLe{B}2PD$U`>?F+1(`Oo zo?`Y=HnNuzcG44NIFFc2yQ~vb$}@HJM4-Gw=sVUUi0wF z%-Lk%~L$1@f1HMtaJ~FKaQuU zZ2|>#Wm}k*am}hMhPs2KA?t5ThcfjWvi`McU&?j6rE}r)*mU30C$;o30$qAXNu|kG zH*04(VLM9Wr;`(QU4MExHGMjDYQhUOwVdkxpQMkT;<|_r>Lxx2fbl`6IXxp}zro$y zo(UWl1BWF*|5K9xsnU!P4LOG0e~R&tV}!V~2viQ82Lpo-gDIo~Zl;tbVY6O&%$DPk zmvT@%<`~niH-OEX-;l5d*ItBuk(Yt_fT8CUGQmQ1iPy;GRJA(AHG92|?z&(>nOs2< z;Q{Lb2E43UsYhF8;r%%#1b_&{TG&#WASu!Eg!ou?kCCB z@S+(FqoJoE$QJdo%HrYGZ-lhS730t-twAw7#u&bGW!;-$pmz{Q_^%YvdSN7TQNdI1 zQXr|Wh4%;;xj^9z21Cx7us|H3FoFO<#+l5)RU=;h5)Hx$q{0%CObE${9Xj2&ku7{i zbsDx7Vn1!yU3>z7*`qz7bRwfc1pa|g|1sP(Fr?dUN5GJ- z3Mb&B9vtavK;cNKl#VW;7iGu}vR;KJE7>_5i63E>tjg+mGa})+PmWDoW?OIddvdWt zJb0!n8Wo5QRCd_X6lCuZ5@>uVzUn+`yspXk$Y@Nl$Yb=B7y-F@Bsvrkx<*G%cd;`J zvB7i7y9;Tg%55v%?_%N#z3e-NmxD(y;ay21r{%JI269C~41p91a?^^LVbv5q?qdo; zCuM|eSA2poBVoll_!NSfVK94*fa1pcA1RDRwNOPfg_#Ag1rVmGW3mNx^7Wi%kIdG8 z%J;#?>f~r-m+)_J3txC(!{osG3^)#QGUtBLMW3w6^CJ7)aW*9DBxgfvhv;mRoNX`K z@0cnx>Njh0zsNpWQ+3Ap&6;*+KJJ?}wPlRotm#mu^!JXbC$rKwYkDm6ao?|%4$Do&Ahmb?&^-$OFn{wjw~@m>tYQFj z=F9ge{KNdsliVSW!=l{EaMm`*&sE%GcR$p}`KcYh%1`J$COyeT)m1Mh>fZ#UM2~e`)?p_vp@t`P+TP5n1AN&71Xa ru3aHE@LMBS2xd>reReKLvFj-B-0ePeHcnTDY7)B3x_pJ03|anP_Ry+; literal 0 HcmV?d00001 diff --git a/infer/lib/infer_pack/__pycache__/modules.cpython-311.pyc b/infer/lib/infer_pack/__pycache__/modules.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a91bc9b7488229aff901c5cbeaa9f4b34d4f846d GIT binary patch literal 30254 zcmeHwdvF`andjii0Wl;265vC80Te|EdQ(r5)+t-5EnhZwi{VX*xVht&u5!AR|D#UIl{a-) zcfYR(GnfHDnsU5OQngL+Lr+gnPd~r@eP4gw_)%F|DF=7E<52I9pX9h-(uL+6SW`whmYaZC$oOdzam!hC8~PEM7@h3H#=|czoLi_(4~fi^Z`I zln%PP+=D`wFj&@AHdx+O&cYoCujukHzjMGlSlLxMSk+a zgSB0??AkR@H@Ko}1^lI5^;Yf}7j?hFMTMB_LW5S2cWfN@9{yF=)yQI%Ay)Ykv6@({ z3dHg(5v!TS@*-AcjKAR0-m&Pr11x?O;`^2rKgi(EfhuDCj0=>6jG@IZT1aUL2TdO`c%5>+bb`YACo6dxIm$J(QKrnF~t zs9Q}4iB^iS!QmHT;TL25y?u%B(6BhDxD6M0prm_vaBz4ij@a(rSRy4*8%B z7vIJJi*Yolx~x%qlIyWVt*^McY*E`QTo6rpQzA$2<0{ZnB^Vh>peYh<}0MC52apYE^N4F(O6=V~LokI1
C;cVm}{CM9@X;@z_9*Sc?b6I`(up+&vJ9$HU<`)mR`9 zc=MXW(eS1a`{u~EyvAK~#{kdI&H=J~Sk^iTUFzDeer=J?gK{A!60$$Tiy zhcdN+qP>fGC_-da2JoYb;IZrLIEcgp^qxK%putDZXbw*Req+zRfH zgF7VOPT5DdTuJ;rz^k}!V-G4p0PBbE!}B^?CWY@p7ezle_rq~ZLhoPbF?yQR^DgLj zdgw~;dGKqK6$iiG_ufJ1d-|Pm%kp!9v1aT1h`Q%T{|_!yqPW;_CZ3{Ljw+=CGz=N@ zP}{f>QN?B=)Y>7d7{Hfexg4hg1Oj3UU6z0wyciLqSG~C=%LdW&@y9`4<}yOrcOQTC z@oCHS7D;H5g{GusUMQP5HSL{#>c;cwhV?Vin-^}keh|9z)K8z2gicxLOj->@VgckG^ptEUz3s$FN*8;Z~@z@+K3*1tK<((zp zjg)n#b%zJ72r@EF#xgCh@L-u%6U($I0?i0#VgsYZHM^-}YK|BOb*TsI(Um@5W;td9 zlheEAm>pMow;glfO3!D)t7R7q&bc_F9CJi17f}v0M(IglGc)V{h9yBvA?q&ajL;kl z?-jH-%Zy*9#mB(YpG}o(p~i1hiu$yGfo(LxxAQwxC{Cl z7kLYH)T|q`nx8jcSvXUU^m;ah(!{`=xu#5UXn--dR9l(fZDx9?cN#@%8Z`W}j@tAp zda3!@!`$=d#!EhReOrHb^*!;tsorvZz&d7&TFf{dbyBU@19w69U!X~kxeGMAGM6Hz zTj0zs#`;b!l@(g_M@@yGL&(u7u0jG=L!G08=VM}ExF^77gm@s{H#|BJ4V;f5KyJibh5%uBG?BaQz8Dh$Pr~v3 zt1*Qi3ac+y96bZWk%ZzJ2}i}@5j>$Hm(nK~WH{9yPbltga6>J=o6xUz;f#!o#D=0G zaUK;9m7a}V8jTGl`hmdhXJV(GR!R=5=_)P}43)h_aRBG`#uWRAIDA2Isz_R~M8sos|48BD;y~A+ndqV<3tptWGq9|A3L0ds&@u7})U}nYoU4S`j;b z7n2jFpkG&VuJZ2;Odpo`pv(v7__eeAT8UpT^Xt?6`b>RW(k}D0ndXi3-H>TmL*I3J zxQdA8>zbyoNOd8(E(DBIo~fvr4!$~;#9zAc__eVKc<$9V-gxYd12+z^^yW=h8D79U z=J@7WzIk3KpE!~6)K7EHO1Mc~{BaRlsyE0(PzzbS}@zNxSR`q`b0cWvXMw`NNZvXOryNRB%i3 zw8@?}3iiET{#yC;CCL+%J;7uL^-%BR;nz>Rc49hm{nXScNeD2gJGtZ9)ihtHg2eU6 zjMz@uGN4n&R*9^n8^=LR=ybIML!k&cV<~a0K;K@1s4u0YyYa6(t(_mxyNt**Y5^uK zVffSrk=`EVqmGy(=DeUIyj-8_u|}P*IJ$Ty@VhY2xWI7_pQU!V+FnZSa~Qlh5sL;Q zL(#zD*hr#p!`avfeII9vCG>1dIDCEpy$*Zbi_Ky6=OhK!2B@H=zSL3<$e< z=qv_+b1Jr_o-RmB*AN&1+Oj~f4hkWQE)JUYk}NZQ9Ud<;z7CWUMsRC0mlLq?&bR~M z&T$obmT@P1PGVr7C_jKC*jyfLG?Y&3D;#IalMa8uZV0Gb#++j%H?8kD0n+iWI&(hq zK>8!TBWg3lz#?huwU;WM_G&F(g^adHYI@!N_Nh($gn>hQr??$i2-y^8uQ)t95?AawHa~Xizb4Wt$BC-ZVvG+o%62C;(UncS_2!LIH`~=*@P9ikSL>j1y z5P&wfkbDftZL;IKBiWIM-gsB_0tZ4^EI4VO z0u}=hO9fLsKU^z$Hpre0bf+12gy|=zDpD57(;|BS_U1j_Dd*&+>#iwR{XP=uGd`C!Rx}5Ko4Xy32%kR2>GDXJd47pxVKc?K>m$HHi0N5;c#@gI~-QIDnT8w zp2D3^#fIX;BC8>BGX=MU7(pZB#YCvy<3k`|WZK_?)#~6EI3j$#13Y9gvPU5F2$DKJ z^YqPga_b&;wG5&{OUi%=T55gnfrvXsXzAiySoX}V7U9Yrwa4taF$Ab-ymH2z#%lxc zbVf@6JWHZHl&5^u1?plTv=kszP^`~(iX+k zr>Zoc@q4T!YQO1NrdBjhSOD<-c$`Pu^9=a962uU)4v^Kwa_h>1V1RgYt&Eoz&v9`$ zIA+nykCewsZ#wf9ncT%w`^55N;O3Wq+-=f!8(Lp09RRM|TpECH9=dcZmG&ZwCOWko zj|)Qp@Z&=994=PgF`=N1%ZjJ)iDfJ+QpPfrQSafY;$)RvEn{@ios)=NK`zo zf^fvjs$MI^7|{5|{*i0|1MF-9XTV!=KrSDMi^MpTQegL}8joQC#nwNR5Ib-yRH07x zdhWztdLz@1_K}a}Vt{;Rq^|H$X=8UIdI+EHXHMQS5E;a4D9l39_t{^?^it^;i zUpqOxq3_1fT+^=Erd_vJe7Gjvv`cC_D>t1@9-cUTFDFO%U0C1*0XnH^O+}=d)yd=Y z)pgT*XKJN-O1pj^;QopZsrsN?eK2`!zNTU7^6M|X_7VUTL7UT(XC>6anYv)6wt2zL z)vmtJIct#-kmT`Smvgm^QxnPmCTkYIVEj@YSeC&Qw*wFgEc^Jo;3-yAhxgj7gfII z=}n+8)-^lOe&~}nm@Ai4(!sCcusq8dGjqaZB%ee1#+{6ojPsCMN(xHlLJrrCfi<(q z$xs%$9xugm7!2yZWgROWD+T+`yHAqEx2$T69$OB@jk||@rrLtkr@AqhIhW(^gkc)K zpzFslrMV%^n5T3BHEF0JElt#qke8}}vhoF?WzZQJ7Kcrf)2^{HwH#5ZZR#D&6)jyk zz+X9)U6rSx0=H{V8=s|mE36xOZrncc_w-EG{|orDk{PT$n9fF3&6KGELn*W92@mvN zj47@YLy4^Xz<4vUV8s<18Xb($tfAPh_K%4D2ox_6p-!YU_l(A4(J+k1!imUG9~SF+ z2Z#Hk;la^?ME}UZ6?JwX-mbBIvy7)Wv4S@;nozvCb--{!6=E3oCkaIyp;y^2!*YqO z?M1`GJw5SQLb3IRv1%9TPQ;Zmo&B>*o-WB3FPs95Qe-KnfUE%d0bUaSV{|7>08Xy5 zM)pD~RF{;J&iRJmjngwv$s0P->kmo|hvbGsP|n{~1x?@U6|Wft#d{le%nKDHTBhuB zV4Eatmxb-%z#Fmt1+u_Zxi&K*387?%R1uowr#FJJ`5M63In$Q+r0PK09ffwB4(& zm#f=S38{L$T)iIGRV(DGJ-4^rsY)MxR;qeVu6hoys;YT?_iMYSpSpfv>Ok`F-G;T( zoip2{hRt%r=HwG3mTsGREY&Vmwa!+wrYl-A^=oFl^oN?5iOL%f;`>eq{iQ=kjp>KVzmZqkyeYff#rWnq8Y7$?@~mm zZWH-475-`GwU^R-6XUtv`Y1;uK@?MYY#TzM@!_aDst)EMY^Yk;lm`+{9|USJ&fw)@d^sX;p>W5HB)_mBV-h zphe<*B12sI@CLFBAXKpo#Ho~7^DY+isvm=<(8Bb;B3k@AAf$k)klhU_%d9(?b_eGL z@5FIZ6Kd5GqG_>Uk?s>igXmo@*R&~}c_R(onPp-$-%(n*%$e?K(!IDFp?deqneahM zWgQ8fR+kbWiR)8wZb2xePh$GZ;W6elztdvP)v&08ygfe-+n7f#QO6=B+V%-0YWmQbkKf3t!a+HM>g+Q`JGjRreyrSw^5!A(|up)z&o zvRSz@oSn47BVsJvH#~eXt~g(a42;I&s+KT_lvSl+v7xSy@{mm(<#tL!{5l?rQx}J@ z#Ma1_>M*?ki?6Y8yj$!aNzfu}IDEB^NlwLX{gTpn9AhV+3uI~=ZdAy%d**68W@|g7 z+Cy^fp#{!Sb_}K(L?`WceYI1k)2k27`VOUihbVrzT)XMkzS-IX>DmMLe05XDr#k0+ z+h={-Z$;l9mV8HL-_f-1=*L+$;&knXIi_njq?Ot_xpw_r?bg}aty1lFxfZHSM;Y#d zJlcCAl5dyn+m-h1`gp-jNeEnJHRD5{)7w10q?_?D*xOQfQ3+W?yz9=>3adgtYwrlf zq?S}Q!HmBr>2foXe4TJN1rnhs@gIR0b;9Lbbxo$SYN4W?%v~2FeHQMF&9%YfSa+`obOmPaEW2ow;}{vP$+l9heAo;;DA{7PzGds~DDZ%DwKi zY5Kue5M)#)XBEk0qhNy?dAW*XdvqzqvFl7MesEy8`(pb?RO9d3R9(#k{2vjwH_3;j zBBWRkeg^v4E0hyDiogV-Bb-HLKepZt*$W26ISI?Sn*pjt(>=5D@}_O6HS%S=)w0A} z4fj6dw;Bwf3d(A@_b6}e&6BsGhSsNVOk27?EgH3$O9sPJFpu0$ znBT7DexFk>(h|B{Z-hdhq9tl`4QE{VFdECM+_?u4Rr82)RVDjJ6hM04oL-e_NmY9V z0)2{;**df<4p^u{Y$mBS8@5GppE`5o)YIX!r%r$Ih~ge#W~uRk;gQ(8S%WvqnQCRq z%o=F&j}z;`JRUGAJS%lprRzQaFU9qubAaV9wT))Jx^`;to~M4^M>RPvtebiKcI}aFO-?T2Z=1KYbLYQ zU=74_5U5_=3x_SRYnX7)@pZF&-J9!mYmR%1mA*wSxmOD8l>>VpvDBZ(!;ezR`K+-@ znyv}U|j zla-SPuUAi1Cl9Le&!<}DmMxNZtL)vnbo`8`a`MURPG;BY_NLv9vW>-Ab!Sn$HNtuHJjSX0F8?*#EfXodSU6;|C&}DHzO&xZA64WWuf~sV%Yc~GS zOM*?eP_%y&wJE*=5^}43>{}G@FNtg<5+p)v_)HD2!h(x*srN<55djMLI+1UK#Ho=m zp{v&GB&uBv19S4nZz!Uw2*9si`U4PFYhuq zM&$qCKTZSRDDPTNSRAVtxFzW`)-s28fdiTL-)LZ-naj5w9|-cMBkU@lG}&o4s7{0m z5^a|vZ1jti$!`4XP8*UBh~t<W>e8h936Ma%`@C&2krQ`ODoiW0<97u--fs=w~QV zR-@gr4yo#FoXIgzO0ov0NkAky25Lu&FQ$krEy6gM2qRvnm?Xc5e?o*{j`%&0r6rOr zx+DPp_?Bf6RP}Dd8My~4 zW7JL*Vs_%Ls4Zs8i7l`bH(t?}M8kD%UXH?MJX#VfG2UG)ZjV7z-ct(vgsqJ1NT*W8 zc51k{^Jq?C!ZtA|C1UIa9Dfs2oMJ37Dh`=8D$+^?HP?fmf?$l=<421dHJih^^gias9B$Y)M{R%_NHM1z#Lm z|LmP7fBIb7zftmc%KlE{aK2mR`(AA7VyZp0awc-KFI~M;s@^GA?@U*by|3E!{|%Ms z-xB#Tk(ET)aQ#d25xHGtxO%kVD$q(fNOri=r8Zp2ZyK%?pbl5^EjnE3CM8*nsQIH@ znTD&@Hw#YRGaMv^BZ*$Im`>mG9vO%Y;y@HS@~@+(r++ABI`mARv3#h2&tC8+`6YMw zUhz$fM)T#I=J6BfLc@~mXPmCSsLMoXnA)vEQvjSY!TH})kEtQTR_TcOh%1P&NrH_e z%b;S%92d`8?;`9a7T{xoL;w-6?zN$UCRuG`w4vq~LBjxLfk=k$rn` zlbwghI^9u|Qi#gt#{Yx=1p+iQdX@ z+AY;>kn3QIE7i42o_4?(ZEc1@py|YoD>FN!=Js11w=UfR4%#F8_m~)`w{r5Ssb{XQ zcykmnTW^MDR;U|PB+s^607j1)flP<$qDTxi(QYz$>r|i0{z&6OZ#k6FxEv8XER?HPTr!w@7Yq5^OF% zu$tnDZ4eVEk~5E%7M{`l7ra{NCv77%w<@ze&!yt~w5hP^$(h{+LM-lTP}@FdIMXfK z|3ZJfpAOqOeerkbGil5a1FQq~t4=*b=E<{FGe zhQh-`FmIumLftcHgRX^H@Ub0;5D165am?CqJi+!kiYF-INg|JfU``E?pUtF2-k8g! z_#Y@<5G0O9!W^1)FRL()=HshPMTeq3o&E#yCpuyn{Ar<@^VCiUuZ>M#nym<=)lbV2 zsUjdVKP+c{QNA+ul*F%<`L%QW)>(e5#BZ1R?P-3ywg*rpRCQRcI*f3BzRI5w>N7%3MySgOzWM6<=@U|Q8@766 z>Q=!9#a9Z;8y_^$AQb=C9uB&4I-x8fZ|b=7z%P zU?!52-ir%eljwMh3tbp4)G?XsC5A`CD{ucbGaGS0OTiTvx$+G*vigaVMK{Zp5X*pY z1q5N-<&F5%PYc{wNghs7F#s4^PE?DpG&17csyw!iEw_ZO@zSB{F@D@-N{KK5a|PSo zN+Fz{#^yGsDSnQDmYQ>&LoWq?$!T5yv#05I;na z_%kAZN95;37>^OjM*vT;bw?5m;4%3a&j;w%S|Vn(7L0o~&3~r(LQ(B3B=|b`0h~iDcrg#}92=Rjpiwv+|^> zHo2+|+sz25@~+hxfXg$`(uGoo-`I3xljIN09F_dAIf8)_)8I{SnHHzl;KA%kdRfex zip+Si(agVI_OCaA)k+`EYH54($c+=JL@I`J9_5Dhl6U>g3zB#HB1uB6r)dUMcABng zOABobQ<{vHh((dO@qp$a^U_KR6GoxSpU;h}MXWzj2ZgF7fG-A?vn@`+SJxD8@j_E1 z{5Ayku;UTnf1Eza=bbx=qbKnxEDZ@!tTbtO%%#aOu4OI|sau~YHRg14BHH?-3u{Hp zi9(;!oXB7TWoBQV%QeEmev1G_|29aR8eAKNN%*g>FWx4_myYl| z8(jQf;4!od_P89EeVgWdJ7#@5)XgQd#<`CX37HMQo}SGUfTGNSZ9LB4%nttBUOW zAQiVU(KJF~)L!EIMAi|ZUMEs9i!T;2iBL9Lu@U{BTmc7Q^r?;>qTTR@9S{WMkXwZ5z+w>@i_2|xgv6Hz{O~+Y zjLa2Ed{JBe!Rm;wtrnV(L8lJR9F@=TMn}}40#04BE6jl|CE`ilUrfeJRcdcBe!qi@ z_w)cJ+2vBH(r)A=#pR`9lI&I?6?S+SCKPs-$^-(mvlugUlc^(j!lzX9^y92mQ5Vv3 zXI10K?a{&T`Tn7}QksbL4}{MTkK*L)59xV=#p>P#hKSD4<)0BbM&vk=6GToDc^m}$ zjsxVUcZ&aw$Q~jjC5rPz{((pkBu-Te8|bX-+q*uD zr~PLo|5@3877w%E5IF(aOcmO8-H!alJ_v4G4@r%O@6aJEemVlcPePdsEDq*DpZwH~eP2wFtbm9M^R%%FbPA=(Umz^i-2DPI*9W(g%(+8nG@E|3TtQDXCk+}SgjIv z$ij}K)K9&%P{G$B6_AHSu9ZRD zK7q||SkXQ)DNb*h?9)mH=~Y;V-#>ymN8R-9)C*EQotDy0g6a<0yCaFSP#_}Pu$H4+ zq+kgITUsGX_tDDmLBO)mx?-Kb-B5aNAZ^_4+(`>`=8-Abq%UJuxe?%0FZZ1w( z0k)(TONMj5%>E3XmNZx7bC;`3Yo81!yv+U#9$%U>eKOpE^e25XTx;6=$#8qp#Xfhr zHEHuF!yQZ)`((H?Y4hi9x%XxFf>W@-9dQb=E#L)a?8~2`xO2# zceBs3ks=q8GZ)HC+>4}?;)cLDt*R!fL>5YlH$ab`xO4L$W6vt F{y%Ay-zNY7 literal 0 HcmV?d00001 diff --git a/infer/lib/infer_pack/__pycache__/transforms.cpython-311.pyc b/infer/lib/infer_pack/__pycache__/transforms.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4cb19ca8083405ee3baa7b8e2e87e34989eefb4f GIT binary patch literal 7609 zcmcgxYiJu;mafwKRi%gZwro8dD^ct?ksoGn4oyo3|U2?40mfThH zqdg@?3^SmG-o@Pnq8&1fdgw_jlfdZDLI!rhzn1w~*A%2E2pE>^kRJ;+y~Dz=uzPMv zmMzEb&g92lRh_zZ>fG0N&pr2^qi-q7L_m6e@$$V$3qkxle%VQeTIRbY$UGz%f+6RK zJEZbd-BH0)HK(4}+|iN*qrRhqTMxH^(J&;VP3n^Rq+wde=zdP#F)|cmfTsz*8TWiM z0nZFPGo!_Kt&D-O0N=upKO$oEYut&k7%7{k7t>SOR3@F6yZ1j&wl}U#-gtlLPt`7N zy+5T4qxwcz;HifDyGbAq30|8wWow`CBWFqxIdj^QRqZl!CSJ?y81j)SXI2rsiBaXCsre?$Hwc^As%EIEp|@YJ*lI;k0psbp<7k(%RV6MjrgW){;- z%pmJh>4n8CCpRso(;0R?F_&6RG84-wCOgB)&Iaa8GIehTM7<4MCdsCj64}&Jl9TnR z^iq=Lk}@@)N>5C}tyCn_JFnF$s_(l6pAgVH9}AUW>?S^2OfdLmYJyvsOQnG2vYMM%NKDP%g~6n`X+WIoTKHU!E~JuE$>kK6JZN-vS=I~9HAN3six6!lzqq!qT)hBA8nQP0M%CGV!N>_|JbX{rA6B-kZW& zypuGLQM7snGN9G499L8;D&Nj`f>*5q`cvvD)d29aWc>qizn9-Dro$0i5h=# z3TeXCH>B~uG~2#7w|2WYBD$jG6C18G&&LJx2gv+^X#QY5j?9CCW{|~@$U3at(8ylH zfb~m(&<_K}5b`>64Fa5>RbW~L3g*CM1tW0?h{R)%d8$Z5jbIGkQjX%a3^}WS%>Hdw zfgYD9i2L=4=GBbqnYwW@a_THzbhUK9UaoAk9|I*_YAkgLJ-%Ij)WG2hROkHJzs#nwtm*ocDxHz^~gYR?^mZW?2cf>cb! zhT)+Q*n!jwU^y`ZRsxGBkquyibE_>s$%T{+Ae3N}p>!r2O3g3KCFhfAQ1Gspn7_aGudorURHy7+h#!M2WLY~1#;k|-<8Tx1Or^OLlblGwP}n8dLFN0d=2MN9V5-=T01^7o4V-pXgle-<^J6PwNz z3{poA2tu8PV^U`?&U%m`wi!DqoVp^$h6;MY){YEOsiU)C6dXrEOfVdKMf-*3{uj*` zH<~Z5x1;7^v3VHLBO*N_7)JhVB8(B(A_Vtw(b@qUg>Y(cJ%NA1sjFgO7!R*|=#M1+ z3Ed-N%P4FkLjRa><7VyZzbS^t@y0@X3qJ-T+5Cl{NEY|*M&qN5x@`i;e_^t%S8H{W z!7A8}AVa%kunUeTG929s?b&V=Q&~Gi!PZe<9WWo*o+!5RG}xYphixbOtPHQ6g<>$7 z-Uf?^mxz-1{qnk5ysg1uA&XBBwZxG7tp!isSLCq3-VH2g;f=gy6)Ym7KCmXu8#S4} zSAL(>0y~8dIkj}ZZe#H_`b@XaN>gx1)bI2MTU(yOJzIs+IjRAV`dE4DfO_bLHtW`v zrx^pBo=wlpj{%FPbv(d|>v(0m=4sp>iJztv*1ua`0E!!bz*$$}3-q7YQTJ}TG zvcq9sue4`Z(9$bCx5FXdUPH?Oxhh)r1JT0VgVD0_8rGA=Hn5g>J7_#;|DJ=Db$6%B zInwS1Y5M!6dAhVm8+Y2kD}s0IYsIpsl_s8gr*cl-{!ZmwybUxTdYW_d?suxe!&~2} zoR_yAG-K~nmyfr-Q#n7cStHp#IMcWkE%Wj|tYM6e_bB=et4h&3KkwnaSfZ}=&AcBt zcm@AJSzIa?;5~50sY&8exfVW9+H1k<3c}bL*D7P@o0WE2l(ungt$Zuvs5%2B?83c~ z9s}oToy)04iJy@$CZz^GU~b$W@Htyad~0LN%6PG61oA69w{b>||y=j}$z`}pA7N#eS9=4rPj-n-MYx32|{3D=fuYp`VXb2H!8hzrwb z@pk1RjdIH9@q8WOL;Mjlfx{1=15Fq=z!M|e&bLE<+xH-eHLd;a`wqC*QC7wfeT%tp z@lig4+po>$+t1Bk(X&J5Mj7kdkMI9yZj``K)7a{X5gY*uWiz2fs2*tur8vb$SWabU zQt41C%Y~TCJj7oV|C+^H`)c>KR5}F_qYyaMAtlfQm1IJ55IjkS;7lLN&VZw_I6s+W zLz!tT#jW=2EmMsK{a_(B2(49JoSM6XPiyR1OwIv`sjEJQW#^Km06b6iMcsh!+pkP*>0#IcJy$MrUUn<6O$8|bYZeD zJY@g?aU_kCb+w?J5js-Kns;Mp5U$#Z&zV4AC$GoBjuf*ZTa`H9#BNzzm4Uz4YO8Vf zl-4rIxokoasnb@)jdeDM{p5Yl z-C?gv=tuEG6*37Wxd&h*^TeM65NZ4Q=WA}H_lkONv3Els6!gL0_I-1%d>gg(ifz4> z#f`QBp>5zL?G=2xZq=qWB3h4?9ip`}KfI}TuB{+_u++ApZx{6KitIH+kBKz+ePfc| zRWu-dsMNEej|%#z@_7u=*G2leV7UH@qQCf5@EotG1gZy7JtEbUACf44F;O~EIff$r zsI~w3G-??{)L?!Hyv_%f;_b3SuyrC^r)cZUkG!z4)FFtQDcw&DDU zWOo;xUj|A(WIJBIg=}$Pc?0>8!ZgyjBws7!?m;k6qTFj-@nZ2(@zUC|NCiu=^5sfX zB~ocB4~vmLMD^u|HXWhg#=q$ox`t5ORk7_Va$L=S_=>iEkrKR}m54z1A-Yeb`|?*M zI#41@W0k)_M=qeyg>?@K0&K(JFWcP@0>yjfZo%G#>|LV0D?j?u?s(u@TP-bBM%SkV z`zW%HiuTd`sN`@JbzgRtjw5?#c?sG3faMG3M+-~1zy21;Eh7zfNsCRzL~#;+9+3{0 zM$25~a%H@7xx6ACIg9AC`Kz0HdW%qNFT8ZL7FWxsk)vC5bXU%ajtd2SLH`Q4pO>S^ z5f>ftN=$SN6!e?!;0t$j!yPTLADN*LxMQvO5?_q_-YY=j4% zj{~aV5ivaS$EX;-DQH?Zoy`x&z8WiamzI(9r06_Z(TUD8`D-xvZQ_(RB2i6(JBp~N zWN8sX{m9b){N{S|A109HqiwZ{GHqIX#UBeD1E^&HSuSjARn&V0a!W^0ej#uISx!{k zLjNd=j~-Y&ZrUcG4y6F?D|}QO659JvpbuG27sySI_u=AKi^ZE?t~^*Ng-eOA+somv zW60A{ut+vgl?_|?g*CcijUwwYSW*zrz?b#drp>-~@k?znUX1)|@zH9z69vzR!86Ez z23gM*G?K$D_+rQrD_=*B^Dt?lBW1EY_{4xhalz0cxx*6GDumA?>iqM>ddD9!h`J`x zZQE*+IEWHwKUF-P! zj1a==jmw_iq=Th+Df{)=Z@YfeRk5Q;j~MAebPvX?cU!FqnFK=sf-WxqgAt*%8##MK zXHO;ZukHU5TmKOC-w^w6pwl;y^G3nAX>}KUMU!BS2!_a}-BUb?>@nfQ1!TKW&`P29 zvc3HA6CVmeNT_Ag8I~-bV*9USzl@bG{!{0pPGpG+v3}vgIEsxU%Q#G<%TzGJoOxij ze|Y3gZKY2u&8U3B{#qc6f;B7{!b)H$MzVe|(lJWbg27+-oQ>lxGy+66z?LvI)0IxMI7U?6%w2<5 zhAw{m(XEMVBV!Yn;QnM}@Yb;G-p?MsGIDkJRwIA#%J|6bi?>E@Us3*rQ8xZ+d=SSi zSsNzUrB(KjzZeGKFtRw8e2<+09*)Cv{{du6O_HQUIE0-`A}o33ClN=4#%t3X5C|~7 zEdt@L(k&Z7IV2M;Ssjw&sMHjLYs=M2wr&xP [b, n_h, t, d_k] + b, d, t_s = key.size() + t_t = query.size(2) + query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3) + key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3) + value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3) + + scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1)) + if self.window_size is not None: + assert ( + t_s == t_t + ), "Relative attention is only available for self-attention." + key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s) + rel_logits = self._matmul_with_relative_keys( + query / math.sqrt(self.k_channels), key_relative_embeddings + ) + scores_local = self._relative_position_to_absolute_position(rel_logits) + scores = scores + scores_local + if self.proximal_bias: + assert t_s == t_t, "Proximal bias is only available for self-attention." + scores = scores + self._attention_bias_proximal(t_s).to( + device=scores.device, dtype=scores.dtype + ) + if mask is not None: + scores = scores.masked_fill(mask == 0, -1e4) + if self.block_length is not None: + assert ( + t_s == t_t + ), "Local attention is only available for self-attention." + block_mask = ( + torch.ones_like(scores) + .triu(-self.block_length) + .tril(self.block_length) + ) + scores = scores.masked_fill(block_mask == 0, -1e4) + p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s] + p_attn = self.drop(p_attn) + output = torch.matmul(p_attn, value) + if self.window_size is not None: + relative_weights = self._absolute_position_to_relative_position(p_attn) + value_relative_embeddings = self._get_relative_embeddings( + self.emb_rel_v, t_s + ) + output = output + self._matmul_with_relative_values( + relative_weights, value_relative_embeddings + ) + output = ( + output.transpose(2, 3).contiguous().view(b, d, t_t) + ) # [b, n_h, t_t, d_k] -> [b, d, t_t] + return output, p_attn + + def _matmul_with_relative_values(self, x, y): + """ + x: [b, h, l, m] + y: [h or 1, m, d] + ret: [b, h, l, d] + """ + ret = torch.matmul(x, y.unsqueeze(0)) + return ret + + def _matmul_with_relative_keys(self, x, y): + """ + x: [b, h, l, d] + y: [h or 1, m, d] + ret: [b, h, l, m] + """ + ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1)) + return ret + + def _get_relative_embeddings(self, relative_embeddings, length: int): + max_relative_position = 2 * self.window_size + 1 + # Pad first before slice to avoid using cond ops. + pad_length: int = max(length - (self.window_size + 1), 0) + slice_start_position = max((self.window_size + 1) - length, 0) + slice_end_position = slice_start_position + 2 * length - 1 + if pad_length > 0: + padded_relative_embeddings = F.pad( + relative_embeddings, + # commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]), + [0, 0, pad_length, pad_length, 0, 0], + ) + else: + padded_relative_embeddings = relative_embeddings + used_relative_embeddings = padded_relative_embeddings[ + :, slice_start_position:slice_end_position + ] + return used_relative_embeddings + + def _relative_position_to_absolute_position(self, x): + """ + x: [b, h, l, 2*l-1] + ret: [b, h, l, l] + """ + batch, heads, length, _ = x.size() + # Concat columns of pad to shift from relative to absolute indexing. + x = F.pad( + x, + # commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, 1]]) + [0, 1, 0, 0, 0, 0, 0, 0], + ) + + # Concat extra elements so to add up to shape (len+1, 2*len-1). + x_flat = x.view([batch, heads, length * 2 * length]) + x_flat = F.pad( + x_flat, + # commons.convert_pad_shape([[0, 0], [0, 0], [0, int(length) - 1]]) + [0, int(length) - 1, 0, 0, 0, 0], + ) + + # Reshape and slice out the padded elements. + x_final = x_flat.view([batch, heads, length + 1, 2 * length - 1])[ + :, :, :length, length - 1 : + ] + return x_final + + def _absolute_position_to_relative_position(self, x): + """ + x: [b, h, l, l] + ret: [b, h, l, 2*l-1] + """ + batch, heads, length, _ = x.size() + # padd along column + x = F.pad( + x, + # commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, int(length) - 1]]) + [0, int(length) - 1, 0, 0, 0, 0, 0, 0], + ) + x_flat = x.view([batch, heads, int(length**2) + int(length * (length - 1))]) + # add 0's in the beginning that will skew the elements after reshape + x_flat = F.pad( + x_flat, + # commons.convert_pad_shape([[0, 0], [0, 0], [int(length), 0]]) + [length, 0, 0, 0, 0, 0], + ) + x_final = x_flat.view([batch, heads, length, 2 * length])[:, :, :, 1:] + return x_final + + def _attention_bias_proximal(self, length: int): + """Bias for self-attention to encourage attention to close positions. + Args: + length: an integer scalar. + Returns: + a Tensor with shape [1, 1, length, length] + """ + r = torch.arange(length, dtype=torch.float32) + diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1) + return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0) + + +class FFN(nn.Module): + def __init__( + self, + in_channels, + out_channels, + filter_channels, + kernel_size, + p_dropout=0.0, + activation: str = None, + causal=False, + ): + super(FFN, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.filter_channels = filter_channels + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.activation = activation + self.causal = causal + self.is_activation = True if activation == "gelu" else False + # if causal: + # self.padding = self._causal_padding + # else: + # self.padding = self._same_padding + + self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size) + self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size) + self.drop = nn.Dropout(p_dropout) + + def padding(self, x: torch.Tensor, x_mask: torch.Tensor) -> torch.Tensor: + if self.causal: + padding = self._causal_padding(x * x_mask) + else: + padding = self._same_padding(x * x_mask) + return padding + + def forward(self, x: torch.Tensor, x_mask: torch.Tensor): + x = self.conv_1(self.padding(x, x_mask)) + if self.is_activation: + x = x * torch.sigmoid(1.702 * x) + else: + x = torch.relu(x) + x = self.drop(x) + + x = self.conv_2(self.padding(x, x_mask)) + return x * x_mask + + def _causal_padding(self, x): + if self.kernel_size == 1: + return x + pad_l: int = self.kernel_size - 1 + pad_r: int = 0 + # padding = [[0, 0], [0, 0], [pad_l, pad_r]] + x = F.pad( + x, + # commons.convert_pad_shape(padding) + [pad_l, pad_r, 0, 0, 0, 0], + ) + return x + + def _same_padding(self, x): + if self.kernel_size == 1: + return x + pad_l: int = (self.kernel_size - 1) // 2 + pad_r: int = self.kernel_size // 2 + # padding = [[0, 0], [0, 0], [pad_l, pad_r]] + x = F.pad( + x, + # commons.convert_pad_shape(padding) + [pad_l, pad_r, 0, 0, 0, 0], + ) + return x diff --git a/infer/lib/infer_pack/attentions_onnx.py b/infer/lib/infer_pack/attentions_onnx.py new file mode 100644 index 0000000..a32abc1 --- /dev/null +++ b/infer/lib/infer_pack/attentions_onnx.py @@ -0,0 +1,459 @@ +############################## Warning! ############################## +# # +# Onnx Export Not Support All Of Non-Torch Types # +# Include Python Built-in Types!!!!!!!!!!!!!!!!! # +# If You Want TO Change This File # +# Do Not Use All Of Non-Torch Types! # +# # +############################## Warning! ############################## +import copy +import math +from typing import Optional + +import numpy as np +import torch +from torch import nn +from torch.nn import functional as F + +from infer.lib.infer_pack import commons, modules +from infer.lib.infer_pack.modules import LayerNorm + + +class Encoder(nn.Module): + def __init__( + self, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size=1, + p_dropout=0.0, + window_size=10, + **kwargs + ): + super(Encoder, self).__init__() + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = int(n_layers) + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.window_size = window_size + + self.drop = nn.Dropout(p_dropout) + self.attn_layers = nn.ModuleList() + self.norm_layers_1 = nn.ModuleList() + self.ffn_layers = nn.ModuleList() + self.norm_layers_2 = nn.ModuleList() + for i in range(self.n_layers): + self.attn_layers.append( + MultiHeadAttention( + hidden_channels, + hidden_channels, + n_heads, + p_dropout=p_dropout, + window_size=window_size, + ) + ) + self.norm_layers_1.append(LayerNorm(hidden_channels)) + self.ffn_layers.append( + FFN( + hidden_channels, + hidden_channels, + filter_channels, + kernel_size, + p_dropout=p_dropout, + ) + ) + self.norm_layers_2.append(LayerNorm(hidden_channels)) + + def forward(self, x, x_mask): + attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1) + x = x * x_mask + zippep = zip( + self.attn_layers, self.norm_layers_1, self.ffn_layers, self.norm_layers_2 + ) + for attn_layers, norm_layers_1, ffn_layers, norm_layers_2 in zippep: + y = attn_layers(x, x, attn_mask) + y = self.drop(y) + x = norm_layers_1(x + y) + + y = ffn_layers(x, x_mask) + y = self.drop(y) + x = norm_layers_2(x + y) + x = x * x_mask + return x + + +class Decoder(nn.Module): + def __init__( + self, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size=1, + p_dropout=0.0, + proximal_bias=False, + proximal_init=True, + **kwargs + ): + super(Decoder, self).__init__() + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.proximal_bias = proximal_bias + self.proximal_init = proximal_init + + self.drop = nn.Dropout(p_dropout) + self.self_attn_layers = nn.ModuleList() + self.norm_layers_0 = nn.ModuleList() + self.encdec_attn_layers = nn.ModuleList() + self.norm_layers_1 = nn.ModuleList() + self.ffn_layers = nn.ModuleList() + self.norm_layers_2 = nn.ModuleList() + for i in range(self.n_layers): + self.self_attn_layers.append( + MultiHeadAttention( + hidden_channels, + hidden_channels, + n_heads, + p_dropout=p_dropout, + proximal_bias=proximal_bias, + proximal_init=proximal_init, + ) + ) + self.norm_layers_0.append(LayerNorm(hidden_channels)) + self.encdec_attn_layers.append( + MultiHeadAttention( + hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout + ) + ) + self.norm_layers_1.append(LayerNorm(hidden_channels)) + self.ffn_layers.append( + FFN( + hidden_channels, + hidden_channels, + filter_channels, + kernel_size, + p_dropout=p_dropout, + causal=True, + ) + ) + self.norm_layers_2.append(LayerNorm(hidden_channels)) + + def forward(self, x, x_mask, h, h_mask): + """ + x: decoder input + h: encoder output + """ + self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to( + device=x.device, dtype=x.dtype + ) + encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1) + x = x * x_mask + for i in range(self.n_layers): + y = self.self_attn_layers[i](x, x, self_attn_mask) + y = self.drop(y) + x = self.norm_layers_0[i](x + y) + + y = self.encdec_attn_layers[i](x, h, encdec_attn_mask) + y = self.drop(y) + x = self.norm_layers_1[i](x + y) + + y = self.ffn_layers[i](x, x_mask) + y = self.drop(y) + x = self.norm_layers_2[i](x + y) + x = x * x_mask + return x + + +class MultiHeadAttention(nn.Module): + def __init__( + self, + channels, + out_channels, + n_heads, + p_dropout=0.0, + window_size=None, + heads_share=True, + block_length=None, + proximal_bias=False, + proximal_init=False, + ): + super(MultiHeadAttention, self).__init__() + assert channels % n_heads == 0 + + self.channels = channels + self.out_channels = out_channels + self.n_heads = n_heads + self.p_dropout = p_dropout + self.window_size = window_size + self.heads_share = heads_share + self.block_length = block_length + self.proximal_bias = proximal_bias + self.proximal_init = proximal_init + self.attn = None + + self.k_channels = channels // n_heads + self.conv_q = nn.Conv1d(channels, channels, 1) + self.conv_k = nn.Conv1d(channels, channels, 1) + self.conv_v = nn.Conv1d(channels, channels, 1) + self.conv_o = nn.Conv1d(channels, out_channels, 1) + self.drop = nn.Dropout(p_dropout) + + if window_size is not None: + n_heads_rel = 1 if heads_share else n_heads + rel_stddev = self.k_channels**-0.5 + self.emb_rel_k = nn.Parameter( + torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) + * rel_stddev + ) + self.emb_rel_v = nn.Parameter( + torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) + * rel_stddev + ) + + nn.init.xavier_uniform_(self.conv_q.weight) + nn.init.xavier_uniform_(self.conv_k.weight) + nn.init.xavier_uniform_(self.conv_v.weight) + if proximal_init: + with torch.no_grad(): + self.conv_k.weight.copy_(self.conv_q.weight) + self.conv_k.bias.copy_(self.conv_q.bias) + + def forward( + self, x: torch.Tensor, c: torch.Tensor, attn_mask: Optional[torch.Tensor] = None + ): + q = self.conv_q(x) + k = self.conv_k(c) + v = self.conv_v(c) + + x, _ = self.attention(q, k, v, mask=attn_mask) + + x = self.conv_o(x) + return x + + def attention( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + mask: Optional[torch.Tensor] = None, + ): + # reshape [b, d, t] -> [b, n_h, t, d_k] + b, d, t_s = key.size() + t_t = query.size(2) + query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3) + key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3) + value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3) + + scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1)) + if self.window_size is not None: + key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s) + rel_logits = self._matmul_with_relative_keys( + query / math.sqrt(self.k_channels), key_relative_embeddings + ) + scores_local = self._relative_position_to_absolute_position(rel_logits) + scores = scores + scores_local + if self.proximal_bias: + assert t_s == t_t, "Proximal bias is only available for self-attention." + scores = scores + self._attention_bias_proximal(t_s).to( + device=scores.device, dtype=scores.dtype + ) + if mask is not None: + scores = scores.masked_fill(mask == 0, -1e4) + if self.block_length is not None: + assert ( + t_s == t_t + ), "Local attention is only available for self-attention." + block_mask = ( + torch.ones_like(scores) + .triu(-self.block_length) + .tril(self.block_length) + ) + scores = scores.masked_fill(block_mask == 0, -1e4) + p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s] + p_attn = self.drop(p_attn) + output = torch.matmul(p_attn, value) + if self.window_size is not None: + relative_weights = self._absolute_position_to_relative_position(p_attn) + value_relative_embeddings = self._get_relative_embeddings( + self.emb_rel_v, t_s + ) + output = output + self._matmul_with_relative_values( + relative_weights, value_relative_embeddings + ) + output = ( + output.transpose(2, 3).contiguous().view(b, d, t_t) + ) # [b, n_h, t_t, d_k] -> [b, d, t_t] + return output, p_attn + + def _matmul_with_relative_values(self, x, y): + """ + x: [b, h, l, m] + y: [h or 1, m, d] + ret: [b, h, l, d] + """ + ret = torch.matmul(x, y.unsqueeze(0)) + return ret + + def _matmul_with_relative_keys(self, x, y): + """ + x: [b, h, l, d] + y: [h or 1, m, d] + ret: [b, h, l, m] + """ + ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1)) + return ret + + def _get_relative_embeddings(self, relative_embeddings, length): + max_relative_position = 2 * self.window_size + 1 + # Pad first before slice to avoid using cond ops. + + pad_length = torch.clamp(length - (self.window_size + 1), min=0) + slice_start_position = torch.clamp((self.window_size + 1) - length, min=0) + slice_end_position = slice_start_position + 2 * length - 1 + padded_relative_embeddings = F.pad( + relative_embeddings, + # commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]), + [0, 0, pad_length, pad_length, 0, 0], + ) + used_relative_embeddings = padded_relative_embeddings[ + :, slice_start_position:slice_end_position + ] + return used_relative_embeddings + + def _relative_position_to_absolute_position(self, x): + """ + x: [b, h, l, 2*l-1] + ret: [b, h, l, l] + """ + batch, heads, length, _ = x.size() + # Concat columns of pad to shift from relative to absolute indexing. + x = F.pad( + x, + # commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, 1]]) + [0, 1, 0, 0, 0, 0, 0, 0], + ) + + # Concat extra elements so to add up to shape (len+1, 2*len-1). + x_flat = x.view([batch, heads, length * 2 * length]) + x_flat = F.pad( + x_flat, + [0, length - 1, 0, 0, 0, 0], + ) + + # Reshape and slice out the padded elements. + x_final = x_flat.view([batch, heads, length + 1, 2 * length - 1])[ + :, :, :length, length - 1 : + ] + return x_final + + def _absolute_position_to_relative_position(self, x): + """ + x: [b, h, l, l] + ret: [b, h, l, 2*l-1] + """ + batch, heads, length, _ = x.size() + # padd along column + x = F.pad( + x, + [0, length - 1, 0, 0, 0, 0, 0, 0], + ) + x_flat = x.view([batch, heads, length*length + length * (length - 1)]) + # add 0's in the beginning that will skew the elements after reshape + x_flat = F.pad( + x_flat, + [length, 0, 0, 0, 0, 0], + ) + x_final = x_flat.view([batch, heads, length, 2 * length])[:, :, :, 1:] + return x_final + + def _attention_bias_proximal(self, length): + """Bias for self-attention to encourage attention to close positions. + Args: + length: an integer scalar. + Returns: + a Tensor with shape [1, 1, length, length] + """ + r = torch.arange(length, dtype=torch.float32) + diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1) + return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0) + + +class FFN(nn.Module): + def __init__( + self, + in_channels, + out_channels, + filter_channels, + kernel_size, + p_dropout=0.0, + activation: str = None, + causal=False, + ): + super(FFN, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.filter_channels = filter_channels + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.activation = activation + self.causal = causal + self.is_activation = True if activation == "gelu" else False + # if causal: + # self.padding = self._causal_padding + # else: + # self.padding = self._same_padding + + self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size) + self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size) + self.drop = nn.Dropout(p_dropout) + + def padding(self, x: torch.Tensor, x_mask: torch.Tensor) -> torch.Tensor: + if self.causal: + padding = self._causal_padding(x * x_mask) + else: + padding = self._same_padding(x * x_mask) + return padding + + def forward(self, x: torch.Tensor, x_mask: torch.Tensor): + x = self.conv_1(self.padding(x, x_mask)) + if self.is_activation: + x = x * torch.sigmoid(1.702 * x) + else: + x = torch.relu(x) + x = self.drop(x) + + x = self.conv_2(self.padding(x, x_mask)) + return x * x_mask + + def _causal_padding(self, x): + if self.kernel_size == 1: + return x + pad_l = self.kernel_size - 1 + pad_r = 0 + # padding = [[0, 0], [0, 0], [pad_l, pad_r]] + x = F.pad( + x, + # commons.convert_pad_shape(padding) + [pad_l, pad_r, 0, 0, 0, 0], + ) + return x + + def _same_padding(self, x): + if self.kernel_size == 1: + return x + pad_l = (self.kernel_size - 1) // 2 + pad_r = self.kernel_size // 2 + # padding = [[0, 0], [0, 0], [pad_l, pad_r]] + x = F.pad( + x, + # commons.convert_pad_shape(padding) + [pad_l, pad_r, 0, 0, 0, 0], + ) + return x diff --git a/infer/lib/infer_pack/commons.py b/infer/lib/infer_pack/commons.py new file mode 100644 index 0000000..4ec6c24 --- /dev/null +++ b/infer/lib/infer_pack/commons.py @@ -0,0 +1,172 @@ +from typing import List, Optional +import math + +import numpy as np +import torch +from torch import nn +from torch.nn import functional as F + + +def init_weights(m, mean=0.0, std=0.01): + classname = m.__class__.__name__ + if classname.find("Conv") != -1: + m.weight.data.normal_(mean, std) + + +def get_padding(kernel_size, dilation=1): + return int((kernel_size * dilation - dilation) / 2) + + +# def convert_pad_shape(pad_shape): +# l = pad_shape[::-1] +# pad_shape = [item for sublist in l for item in sublist] +# return pad_shape + + +def kl_divergence(m_p, logs_p, m_q, logs_q): + """KL(P||Q)""" + kl = (logs_q - logs_p) - 0.5 + kl += ( + 0.5 * (torch.exp(2.0 * logs_p) + ((m_p - m_q) ** 2)) * torch.exp(-2.0 * logs_q) + ) + return kl + + +def rand_gumbel(shape): + """Sample from the Gumbel distribution, protect from overflows.""" + uniform_samples = torch.rand(shape) * 0.99998 + 0.00001 + return -torch.log(-torch.log(uniform_samples)) + + +def rand_gumbel_like(x): + g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device) + return g + + +def slice_segments(x, ids_str, segment_size=4): + ret = torch.zeros_like(x[:, :, :segment_size]) + for i in range(x.size(0)): + idx_str = ids_str[i] + idx_end = idx_str + segment_size + ret[i] = x[i, :, idx_str:idx_end] + return ret + + +def slice_segments2(x, ids_str, segment_size=4): + ret = torch.zeros_like(x[:, :segment_size]) + for i in range(x.size(0)): + idx_str = ids_str[i] + idx_end = idx_str + segment_size + ret[i] = x[i, idx_str:idx_end] + return ret + + +def rand_slice_segments(x, x_lengths=None, segment_size=4): + b, d, t = x.size() + if x_lengths is None: + x_lengths = t + ids_str_max = x_lengths - segment_size + 1 + ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long) + ret = slice_segments(x, ids_str, segment_size) + return ret, ids_str + + +def get_timing_signal_1d(length, channels, min_timescale=1.0, max_timescale=1.0e4): + position = torch.arange(length, dtype=torch.float) + num_timescales = channels // 2 + log_timescale_increment = math.log(float(max_timescale) / float(min_timescale)) / ( + num_timescales - 1 + ) + inv_timescales = min_timescale * torch.exp( + torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment + ) + scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1) + signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0) + signal = F.pad(signal, [0, 0, 0, channels % 2]) + signal = signal.view(1, channels, length) + return signal + + +def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4): + b, channels, length = x.size() + signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale) + return x + signal.to(dtype=x.dtype, device=x.device) + + +def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1): + b, channels, length = x.size() + signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale) + return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis) + + +def subsequent_mask(length): + mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0) + return mask + + +@torch.jit.script +def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels): + n_channels_int = n_channels[0] + in_act = input_a + input_b + t_act = torch.tanh(in_act[:, :n_channels_int, :]) + s_act = torch.sigmoid(in_act[:, n_channels_int:, :]) + acts = t_act * s_act + return acts + + +# def convert_pad_shape(pad_shape): +# l = pad_shape[::-1] +# pad_shape = [item for sublist in l for item in sublist] +# return pad_shape + + +def convert_pad_shape(pad_shape: List[List[int]]) -> List[int]: + return torch.tensor(pad_shape).flip(0).reshape(-1).int().tolist() + + +def shift_1d(x): + x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1] + return x + + +def sequence_mask(length: torch.Tensor, max_length: Optional[int] = None): + if max_length is None: + max_length = length.max() + x = torch.arange(max_length, dtype=length.dtype, device=length.device) + return x.unsqueeze(0) < length.unsqueeze(1) + + +def generate_path(duration, mask): + """ + duration: [b, 1, t_x] + mask: [b, 1, t_y, t_x] + """ + device = duration.device + + b, _, t_y, t_x = mask.shape + cum_duration = torch.cumsum(duration, -1) + + cum_duration_flat = cum_duration.view(b * t_x) + path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype) + path = path.view(b, t_x, t_y) + path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1] + path = path.unsqueeze(1).transpose(2, 3) * mask + return path + + +def clip_grad_value_(parameters, clip_value, norm_type=2): + if isinstance(parameters, torch.Tensor): + parameters = [parameters] + parameters = list(filter(lambda p: p.grad is not None, parameters)) + norm_type = float(norm_type) + if clip_value is not None: + clip_value = float(clip_value) + + total_norm = 0 + for p in parameters: + param_norm = p.grad.data.norm(norm_type) + total_norm += param_norm.item() ** norm_type + if clip_value is not None: + p.grad.data.clamp_(min=-clip_value, max=clip_value) + total_norm = total_norm ** (1.0 / norm_type) + return total_norm diff --git a/infer/lib/infer_pack/models.py b/infer/lib/infer_pack/models.py new file mode 100644 index 0000000..a900048 --- /dev/null +++ b/infer/lib/infer_pack/models.py @@ -0,0 +1,1223 @@ +import math +import logging +from typing import Optional + +logger = logging.getLogger(__name__) + +import numpy as np +import torch +from torch import nn +from torch.nn import AvgPool1d, Conv1d, Conv2d, ConvTranspose1d +from torch.nn import functional as F +from torch.nn.utils import remove_weight_norm, spectral_norm, weight_norm +from infer.lib.infer_pack import attentions, commons, modules +from infer.lib.infer_pack.commons import get_padding, init_weights + +has_xpu = bool(hasattr(torch, "xpu") and torch.xpu.is_available()) + + +class TextEncoder(nn.Module): + def __init__( + self, + in_channels, + out_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + f0=True, + ): + super(TextEncoder, self).__init__() + self.out_channels = out_channels + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = float(p_dropout) + self.emb_phone = nn.Linear(in_channels, hidden_channels) + self.lrelu = nn.LeakyReLU(0.1, inplace=True) + if f0 == True: + self.emb_pitch = nn.Embedding(256, hidden_channels) # pitch 256 + self.encoder = attentions.Encoder( + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + float(p_dropout), + ) + self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1) + + def forward( + self, + phone: torch.Tensor, + pitch: torch.Tensor, + lengths: torch.Tensor, + skip_head: Optional[torch.Tensor] = None, + ): + if pitch is None: + x = self.emb_phone(phone) + else: + x = self.emb_phone(phone) + self.emb_pitch(pitch) + x = x * math.sqrt(self.hidden_channels) # [b, t, h] + x = self.lrelu(x) + x = torch.transpose(x, 1, -1) # [b, h, t] + x_mask = torch.unsqueeze(commons.sequence_mask(lengths, x.size(2)), 1).to( + x.dtype + ) + x = self.encoder(x * x_mask, x_mask) + if skip_head is not None: + assert isinstance(skip_head, torch.Tensor) + head = int(skip_head.item()) + x = x[:, :, head:] + x_mask = x_mask[:, :, head:] + stats = self.proj(x) * x_mask + m, logs = torch.split(stats, self.out_channels, dim=1) + return m, logs, x_mask + + +class ResidualCouplingBlock(nn.Module): + def __init__( + self, + channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + n_flows=4, + gin_channels=0, + ): + super(ResidualCouplingBlock, self).__init__() + self.channels = channels + self.hidden_channels = hidden_channels + self.kernel_size = kernel_size + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.n_flows = n_flows + self.gin_channels = gin_channels + + self.flows = nn.ModuleList() + for i in range(n_flows): + self.flows.append( + modules.ResidualCouplingLayer( + channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + gin_channels=gin_channels, + mean_only=True, + ) + ) + self.flows.append(modules.Flip()) + + def forward( + self, + x: torch.Tensor, + x_mask: torch.Tensor, + g: Optional[torch.Tensor] = None, + reverse: bool = False, + ): + if not reverse: + for flow in self.flows: + x, _ = flow(x, x_mask, g=g, reverse=reverse) + else: + for flow in self.flows[::-1]: + x, _ = flow.forward(x, x_mask, g=g, reverse=reverse) + return x + + def remove_weight_norm(self): + for i in range(self.n_flows): + self.flows[i * 2].remove_weight_norm() + + def __prepare_scriptable__(self): + for i in range(self.n_flows): + for hook in self.flows[i * 2]._forward_pre_hooks.values(): + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(self.flows[i * 2]) + + return self + + +class PosteriorEncoder(nn.Module): + def __init__( + self, + in_channels, + out_channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + gin_channels=0, + ): + super(PosteriorEncoder, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.hidden_channels = hidden_channels + self.kernel_size = kernel_size + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.gin_channels = gin_channels + + self.pre = nn.Conv1d(in_channels, hidden_channels, 1) + self.enc = modules.WN( + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + gin_channels=gin_channels, + ) + self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1) + + def forward( + self, x: torch.Tensor, x_lengths: torch.Tensor, g: Optional[torch.Tensor] = None + ): + x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to( + x.dtype + ) + x = self.pre(x) * x_mask + x = self.enc(x, x_mask, g=g) + stats = self.proj(x) * x_mask + m, logs = torch.split(stats, self.out_channels, dim=1) + z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask + return z, m, logs, x_mask + + def remove_weight_norm(self): + self.enc.remove_weight_norm() + + def __prepare_scriptable__(self): + for hook in self.enc._forward_pre_hooks.values(): + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(self.enc) + return self + + +class Generator(torch.nn.Module): + def __init__( + self, + initial_channel, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + gin_channels=0, + ): + super(Generator, self).__init__() + self.num_kernels = len(resblock_kernel_sizes) + self.num_upsamples = len(upsample_rates) + self.conv_pre = Conv1d( + initial_channel, upsample_initial_channel, 7, 1, padding=3 + ) + resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2 + + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): + self.ups.append( + weight_norm( + ConvTranspose1d( + upsample_initial_channel // (2**i), + upsample_initial_channel // (2 ** (i + 1)), + k, + u, + padding=(k - u) // 2, + ) + ) + ) + + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + ch = upsample_initial_channel // (2 ** (i + 1)) + for j, (k, d) in enumerate( + zip(resblock_kernel_sizes, resblock_dilation_sizes) + ): + self.resblocks.append(resblock(ch, k, d)) + + self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False) + self.ups.apply(init_weights) + + if gin_channels != 0: + self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1) + + def forward( + self, + x: torch.Tensor, + g: Optional[torch.Tensor] = None, + n_res: Optional[torch.Tensor] = None, + ): + if n_res is not None: + assert isinstance(n_res, torch.Tensor) + n = int(n_res.item()) + if n != x.shape[-1]: + x = F.interpolate(x, size=n, mode="linear") + x = self.conv_pre(x) + if g is not None: + x = x + self.cond(g) + + for i in range(self.num_upsamples): + x = F.leaky_relu(x, modules.LRELU_SLOPE) + x = self.ups[i](x) + xs = None + for j in range(self.num_kernels): + if xs is None: + xs = self.resblocks[i * self.num_kernels + j](x) + else: + xs += self.resblocks[i * self.num_kernels + j](x) + x = xs / self.num_kernels + x = F.leaky_relu(x) + x = self.conv_post(x) + x = torch.tanh(x) + + return x + + def __prepare_scriptable__(self): + for l in self.ups: + for hook in l._forward_pre_hooks.values(): + # The hook we want to remove is an instance of WeightNorm class, so + # normally we would do `if isinstance(...)` but this class is not accessible + # because of shadowing, so we check the module name directly. + # https://github.com/pytorch/pytorch/blob/be0ca00c5ce260eb5bcec3237357f7a30cc08983/torch/nn/utils/__init__.py#L3 + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(l) + + for l in self.resblocks: + for hook in l._forward_pre_hooks.values(): + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(l) + return self + + def remove_weight_norm(self): + for l in self.ups: + remove_weight_norm(l) + for l in self.resblocks: + l.remove_weight_norm() + + +class SineGen(torch.nn.Module): + """Definition of sine generator + SineGen(samp_rate, harmonic_num = 0, + sine_amp = 0.1, noise_std = 0.003, + voiced_threshold = 0, + flag_for_pulse=False) + samp_rate: sampling rate in Hz + harmonic_num: number of harmonic overtones (default 0) + sine_amp: amplitude of sine-wavefrom (default 0.1) + noise_std: std of Gaussian noise (default 0.003) + voiced_thoreshold: F0 threshold for U/V classification (default 0) + flag_for_pulse: this SinGen is used inside PulseGen (default False) + Note: when flag_for_pulse is True, the first time step of a voiced + segment is always sin(torch.pi) or cos(0) + """ + + def __init__( + self, + samp_rate, + harmonic_num=0, + sine_amp=0.1, + noise_std=0.003, + voiced_threshold=0, + flag_for_pulse=False, + ): + super(SineGen, self).__init__() + self.sine_amp = sine_amp + self.noise_std = noise_std + self.harmonic_num = harmonic_num + self.dim = self.harmonic_num + 1 + self.sampling_rate = samp_rate + self.voiced_threshold = voiced_threshold + + def _f02uv(self, f0): + # generate uv signal + uv = torch.ones_like(f0) + uv = uv * (f0 > self.voiced_threshold) + if uv.device.type == "privateuseone": # for DirectML + uv = uv.float() + return uv + + def _f02sine(self, f0, upp): + """ f0: (batchsize, length, dim) + where dim indicates fundamental tone and overtones + """ + a = torch.arange(1, upp + 1, dtype=f0.dtype, device=f0.device) + rad = f0 / self.sampling_rate * a + rad2 = torch.fmod(rad[:, :-1, -1:].float() + 0.5, 1.0) - 0.5 + rad_acc = rad2.cumsum(dim=1).fmod(1.0).to(f0) + rad += F.pad(rad_acc, (0, 0, 1, 0), mode='constant') + rad = rad.reshape(f0.shape[0], -1, 1) + b = torch.arange(1, self.dim + 1, dtype=f0.dtype, device=f0.device).reshape(1, 1, -1) + rad *= b + rand_ini = torch.rand(1, 1, self.dim, device=f0.device) + rand_ini[..., 0] = 0 + rad += rand_ini + sines = torch.sin(2 * np.pi * rad) + return sines + + def forward(self, f0: torch.Tensor, upp: int): + """sine_tensor, uv = forward(f0) + input F0: tensor(batchsize=1, length, dim=1) + f0 for unvoiced steps should be 0 + output sine_tensor: tensor(batchsize=1, length, dim) + output uv: tensor(batchsize=1, length, 1) + """ + with torch.no_grad(): + f0 = f0.unsqueeze(-1) + sine_waves = self._f02sine(f0, upp) * self.sine_amp + uv = self._f02uv(f0) + uv = F.interpolate( + uv.transpose(2, 1), scale_factor=float(upp), mode="nearest" + ).transpose(2, 1) + noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3 + noise = noise_amp * torch.randn_like(sine_waves) + sine_waves = sine_waves * uv + noise + return sine_waves, uv, noise + + +class SourceModuleHnNSF(torch.nn.Module): + """SourceModule for hn-nsf + SourceModule(sampling_rate, harmonic_num=0, sine_amp=0.1, + add_noise_std=0.003, voiced_threshod=0) + sampling_rate: sampling_rate in Hz + harmonic_num: number of harmonic above F0 (default: 0) + sine_amp: amplitude of sine source signal (default: 0.1) + add_noise_std: std of additive Gaussian noise (default: 0.003) + note that amplitude of noise in unvoiced is decided + by sine_amp + voiced_threshold: threhold to set U/V given F0 (default: 0) + Sine_source, noise_source = SourceModuleHnNSF(F0_sampled) + F0_sampled (batchsize, length, 1) + Sine_source (batchsize, length, 1) + noise_source (batchsize, length 1) + uv (batchsize, length, 1) + """ + + def __init__( + self, + sampling_rate, + harmonic_num=0, + sine_amp=0.1, + add_noise_std=0.003, + voiced_threshod=0, + is_half=True, + ): + super(SourceModuleHnNSF, self).__init__() + + self.sine_amp = sine_amp + self.noise_std = add_noise_std + self.is_half = is_half + # to produce sine waveforms + self.l_sin_gen = SineGen( + sampling_rate, harmonic_num, sine_amp, add_noise_std, voiced_threshod + ) + + # to merge source harmonics into a single excitation + self.l_linear = torch.nn.Linear(harmonic_num + 1, 1) + self.l_tanh = torch.nn.Tanh() + # self.ddtype:int = -1 + + def forward(self, x: torch.Tensor, upp: int = 1): + # if self.ddtype ==-1: + # self.ddtype = self.l_linear.weight.dtype + sine_wavs, uv, _ = self.l_sin_gen(x, upp) + # print(x.dtype,sine_wavs.dtype,self.l_linear.weight.dtype) + # if self.is_half: + # sine_wavs = sine_wavs.half() + # sine_merge = self.l_tanh(self.l_linear(sine_wavs.to(x))) + # print(sine_wavs.dtype,self.ddtype) + # if sine_wavs.dtype != self.l_linear.weight.dtype: + sine_wavs = sine_wavs.to(dtype=self.l_linear.weight.dtype) + sine_merge = self.l_tanh(self.l_linear(sine_wavs)) + return sine_merge, None, None # noise, uv + + +class GeneratorNSF(torch.nn.Module): + def __init__( + self, + initial_channel, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + gin_channels, + sr, + is_half=False, + ): + super(GeneratorNSF, self).__init__() + self.num_kernels = len(resblock_kernel_sizes) + self.num_upsamples = len(upsample_rates) + + self.f0_upsamp = torch.nn.Upsample(scale_factor=math.prod(upsample_rates)) + self.m_source = SourceModuleHnNSF( + sampling_rate=sr, harmonic_num=0, is_half=is_half + ) + self.noise_convs = nn.ModuleList() + self.conv_pre = Conv1d( + initial_channel, upsample_initial_channel, 7, 1, padding=3 + ) + resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2 + + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): + c_cur = upsample_initial_channel // (2 ** (i + 1)) + self.ups.append( + weight_norm( + ConvTranspose1d( + upsample_initial_channel // (2**i), + upsample_initial_channel // (2 ** (i + 1)), + k, + u, + padding=(k - u) // 2, + ) + ) + ) + if i + 1 < len(upsample_rates): + stride_f0 = math.prod(upsample_rates[i + 1 :]) + self.noise_convs.append( + Conv1d( + 1, + c_cur, + kernel_size=stride_f0 * 2, + stride=stride_f0, + padding=stride_f0 // 2, + ) + ) + else: + self.noise_convs.append(Conv1d(1, c_cur, kernel_size=1)) + + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + ch = upsample_initial_channel // (2 ** (i + 1)) + for j, (k, d) in enumerate( + zip(resblock_kernel_sizes, resblock_dilation_sizes) + ): + self.resblocks.append(resblock(ch, k, d)) + + self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False) + self.ups.apply(init_weights) + + if gin_channels != 0: + self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1) + + self.upp = math.prod(upsample_rates) + + self.lrelu_slope = modules.LRELU_SLOPE + + def forward( + self, + x, + f0, + g: Optional[torch.Tensor] = None, + n_res: Optional[torch.Tensor] = None, + ): + har_source, noi_source, uv = self.m_source(f0, self.upp) + har_source = har_source.transpose(1, 2) + if n_res is not None: + assert isinstance(n_res, torch.Tensor) + n = int(n_res.item()) + if n * self.upp != har_source.shape[-1]: + har_source = F.interpolate(har_source, size=n * self.upp, mode="linear") + if n != x.shape[-1]: + x = F.interpolate(x, size=n, mode="linear") + x = self.conv_pre(x) + if g is not None: + x = x + self.cond(g) + # torch.jit.script() does not support direct indexing of torch modules + # That's why I wrote this + for i, (ups, noise_convs) in enumerate(zip(self.ups, self.noise_convs)): + if i < self.num_upsamples: + x = F.leaky_relu(x, self.lrelu_slope) + x = ups(x) + x_source = noise_convs(har_source) + x = x + x_source + xs: Optional[torch.Tensor] = None + l = [i * self.num_kernels + j for j in range(self.num_kernels)] + for j, resblock in enumerate(self.resblocks): + if j in l: + if xs is None: + xs = resblock(x) + else: + xs += resblock(x) + # This assertion cannot be ignored! \ + # If ignored, it will cause torch.jit.script() compilation errors + assert isinstance(xs, torch.Tensor) + x = xs / self.num_kernels + x = F.leaky_relu(x) + x = self.conv_post(x) + x = torch.tanh(x) + + return x + + def remove_weight_norm(self): + for l in self.ups: + remove_weight_norm(l) + for l in self.resblocks: + l.remove_weight_norm() + + def __prepare_scriptable__(self): + for l in self.ups: + for hook in l._forward_pre_hooks.values(): + # The hook we want to remove is an instance of WeightNorm class, so + # normally we would do `if isinstance(...)` but this class is not accessible + # because of shadowing, so we check the module name directly. + # https://github.com/pytorch/pytorch/blob/be0ca00c5ce260eb5bcec3237357f7a30cc08983/torch/nn/utils/__init__.py#L3 + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(l) + for l in self.resblocks: + for hook in self.resblocks._forward_pre_hooks.values(): + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(l) + return self + + +sr2sr = { + "32k": 32000, + "40k": 40000, + "48k": 48000, +} + + +class SynthesizerTrnMs256NSFsid(nn.Module): + def __init__( + self, + spec_channels, + segment_size, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + spk_embed_dim, + gin_channels, + sr, + **kwargs + ): + super(SynthesizerTrnMs256NSFsid, self).__init__() + if isinstance(sr, str): + sr = sr2sr[sr] + self.spec_channels = spec_channels + self.inter_channels = inter_channels + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = float(p_dropout) + self.resblock = resblock + self.resblock_kernel_sizes = resblock_kernel_sizes + self.resblock_dilation_sizes = resblock_dilation_sizes + self.upsample_rates = upsample_rates + self.upsample_initial_channel = upsample_initial_channel + self.upsample_kernel_sizes = upsample_kernel_sizes + self.segment_size = segment_size + self.gin_channels = gin_channels + # self.hop_length = hop_length# + self.spk_embed_dim = spk_embed_dim + self.enc_p = TextEncoder( + 256, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + float(p_dropout), + ) + self.dec = GeneratorNSF( + inter_channels, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + gin_channels=gin_channels, + sr=sr, + is_half=kwargs["is_half"], + ) + self.enc_q = PosteriorEncoder( + spec_channels, + inter_channels, + hidden_channels, + 5, + 1, + 16, + gin_channels=gin_channels, + ) + self.flow = ResidualCouplingBlock( + inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels + ) + self.emb_g = nn.Embedding(self.spk_embed_dim, gin_channels) + logger.debug( + "gin_channels: " + + str(gin_channels) + + ", self.spk_embed_dim: " + + str(self.spk_embed_dim) + ) + + def remove_weight_norm(self): + self.dec.remove_weight_norm() + self.flow.remove_weight_norm() + if hasattr(self, "enc_q"): + self.enc_q.remove_weight_norm() + + def __prepare_scriptable__(self): + for hook in self.dec._forward_pre_hooks.values(): + # The hook we want to remove is an instance of WeightNorm class, so + # normally we would do `if isinstance(...)` but this class is not accessible + # because of shadowing, so we check the module name directly. + # https://github.com/pytorch/pytorch/blob/be0ca00c5ce260eb5bcec3237357f7a30cc08983/torch/nn/utils/__init__.py#L3 + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(self.dec) + for hook in self.flow._forward_pre_hooks.values(): + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(self.flow) + if hasattr(self, "enc_q"): + for hook in self.enc_q._forward_pre_hooks.values(): + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(self.enc_q) + return self + + @torch.jit.ignore + def forward( + self, + phone: torch.Tensor, + phone_lengths: torch.Tensor, + pitch: torch.Tensor, + pitchf: torch.Tensor, + y: torch.Tensor, + y_lengths: torch.Tensor, + ds: Optional[torch.Tensor] = None, + ): # 这里ds是id，[bs,1] + # print(1,pitch.shape)#[bs,t] + g = self.emb_g(ds).unsqueeze(-1) # [b, 256, 1]##1是t，广播的 + m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths) + z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g) + z_p = self.flow(z, y_mask, g=g) + z_slice, ids_slice = commons.rand_slice_segments( + z, y_lengths, self.segment_size + ) + # print(-1,pitchf.shape,ids_slice,self.segment_size,self.hop_length,self.segment_size//self.hop_length) + pitchf = commons.slice_segments2(pitchf, ids_slice, self.segment_size) + # print(-2,pitchf.shape,z_slice.shape) + o = self.dec(z_slice, pitchf, g=g) + return o, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q) + + @torch.jit.export + def infer( + self, + phone: torch.Tensor, + phone_lengths: torch.Tensor, + pitch: torch.Tensor, + nsff0: torch.Tensor, + sid: torch.Tensor, + skip_head: Optional[torch.Tensor] = None, + return_length: Optional[torch.Tensor] = None, + return_length2: Optional[torch.Tensor] = None, + ): + g = self.emb_g(sid).unsqueeze(-1) + if skip_head is not None and return_length is not None: + assert isinstance(skip_head, torch.Tensor) + assert isinstance(return_length, torch.Tensor) + head = int(skip_head.item()) + length = int(return_length.item()) + flow_head = torch.clamp(skip_head - 24, min=0) + dec_head = head - int(flow_head.item()) + m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths, flow_head) + z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask + z = self.flow(z_p, x_mask, g=g, reverse=True) + z = z[:, :, dec_head : dec_head + length] + x_mask = x_mask[:, :, dec_head : dec_head + length] + nsff0 = nsff0[:, head : head + length] + else: + m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths) + z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask + z = self.flow(z_p, x_mask, g=g, reverse=True) + o = self.dec(z * x_mask, nsff0, g=g, n_res=return_length2) + return o, x_mask, (z, z_p, m_p, logs_p) + + +class SynthesizerTrnMs768NSFsid(SynthesizerTrnMs256NSFsid): + def __init__( + self, + spec_channels, + segment_size, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + spk_embed_dim, + gin_channels, + sr, + **kwargs + ): + super(SynthesizerTrnMs768NSFsid, self).__init__( + spec_channels, + segment_size, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + spk_embed_dim, + gin_channels, + sr, + **kwargs + ) + del self.enc_p + self.enc_p = TextEncoder( + 768, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + float(p_dropout), + ) + + +class SynthesizerTrnMs256NSFsid_nono(nn.Module): + def __init__( + self, + spec_channels, + segment_size, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + spk_embed_dim, + gin_channels, + sr=None, + **kwargs + ): + super(SynthesizerTrnMs256NSFsid_nono, self).__init__() + self.spec_channels = spec_channels + self.inter_channels = inter_channels + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = float(p_dropout) + self.resblock = resblock + self.resblock_kernel_sizes = resblock_kernel_sizes + self.resblock_dilation_sizes = resblock_dilation_sizes + self.upsample_rates = upsample_rates + self.upsample_initial_channel = upsample_initial_channel + self.upsample_kernel_sizes = upsample_kernel_sizes + self.segment_size = segment_size + self.gin_channels = gin_channels + # self.hop_length = hop_length# + self.spk_embed_dim = spk_embed_dim + self.enc_p = TextEncoder( + 256, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + float(p_dropout), + f0=False, + ) + self.dec = Generator( + inter_channels, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + gin_channels=gin_channels, + ) + self.enc_q = PosteriorEncoder( + spec_channels, + inter_channels, + hidden_channels, + 5, + 1, + 16, + gin_channels=gin_channels, + ) + self.flow = ResidualCouplingBlock( + inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels + ) + self.emb_g = nn.Embedding(self.spk_embed_dim, gin_channels) + logger.debug( + "gin_channels: " + + str(gin_channels) + + ", self.spk_embed_dim: " + + str(self.spk_embed_dim) + ) + + def remove_weight_norm(self): + self.dec.remove_weight_norm() + self.flow.remove_weight_norm() + if hasattr(self, "enc_q"): + self.enc_q.remove_weight_norm() + + def __prepare_scriptable__(self): + for hook in self.dec._forward_pre_hooks.values(): + # The hook we want to remove is an instance of WeightNorm class, so + # normally we would do `if isinstance(...)` but this class is not accessible + # because of shadowing, so we check the module name directly. + # https://github.com/pytorch/pytorch/blob/be0ca00c5ce260eb5bcec3237357f7a30cc08983/torch/nn/utils/__init__.py#L3 + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(self.dec) + for hook in self.flow._forward_pre_hooks.values(): + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(self.flow) + if hasattr(self, "enc_q"): + for hook in self.enc_q._forward_pre_hooks.values(): + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(self.enc_q) + return self + + @torch.jit.ignore + def forward(self, phone, phone_lengths, y, y_lengths, ds): # 这里ds是id，[bs,1] + g = self.emb_g(ds).unsqueeze(-1) # [b, 256, 1]##1是t，广播的 + m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths) + z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g) + z_p = self.flow(z, y_mask, g=g) + z_slice, ids_slice = commons.rand_slice_segments( + z, y_lengths, self.segment_size + ) + o = self.dec(z_slice, g=g) + return o, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q) + + @torch.jit.export + def infer( + self, + phone: torch.Tensor, + phone_lengths: torch.Tensor, + sid: torch.Tensor, + skip_head: Optional[torch.Tensor] = None, + return_length: Optional[torch.Tensor] = None, + return_length2: Optional[torch.Tensor] = None, + ): + g = self.emb_g(sid).unsqueeze(-1) + if skip_head is not None and return_length is not None: + assert isinstance(skip_head, torch.Tensor) + assert isinstance(return_length, torch.Tensor) + head = int(skip_head.item()) + length = int(return_length.item()) + flow_head = torch.clamp(skip_head - 24, min=0) + dec_head = head - int(flow_head.item()) + m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths, flow_head) + z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask + z = self.flow(z_p, x_mask, g=g, reverse=True) + z = z[:, :, dec_head : dec_head + length] + x_mask = x_mask[:, :, dec_head : dec_head + length] + else: + m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths) + z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask + z = self.flow(z_p, x_mask, g=g, reverse=True) + o = self.dec(z * x_mask, g=g, n_res=return_length2) + return o, x_mask, (z, z_p, m_p, logs_p) + + +class SynthesizerTrnMs768NSFsid_nono(SynthesizerTrnMs256NSFsid_nono): + def __init__( + self, + spec_channels, + segment_size, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + spk_embed_dim, + gin_channels, + sr=None, + **kwargs + ): + super(SynthesizerTrnMs768NSFsid_nono, self).__init__( + spec_channels, + segment_size, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + spk_embed_dim, + gin_channels, + sr, + **kwargs + ) + del self.enc_p + self.enc_p = TextEncoder( + 768, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + float(p_dropout), + f0=False, + ) + + +class MultiPeriodDiscriminator(torch.nn.Module): + def __init__(self, use_spectral_norm=False): + super(MultiPeriodDiscriminator, self).__init__() + periods = [2, 3, 5, 7, 11, 17] + # periods = [3, 5, 7, 11, 17, 23, 37] + + discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)] + discs = discs + [ + DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods + ] + self.discriminators = nn.ModuleList(discs) + + def forward(self, y, y_hat): + y_d_rs = [] # + y_d_gs = [] + fmap_rs = [] + fmap_gs = [] + for i, d in enumerate(self.discriminators): + y_d_r, fmap_r = d(y) + y_d_g, fmap_g = d(y_hat) + # for j in range(len(fmap_r)): + # print(i,j,y.shape,y_hat.shape,fmap_r[j].shape,fmap_g[j].shape) + y_d_rs.append(y_d_r) + y_d_gs.append(y_d_g) + fmap_rs.append(fmap_r) + fmap_gs.append(fmap_g) + + return y_d_rs, y_d_gs, fmap_rs, fmap_gs + + +class MultiPeriodDiscriminatorV2(torch.nn.Module): + def __init__(self, use_spectral_norm=False): + super(MultiPeriodDiscriminatorV2, self).__init__() + # periods = [2, 3, 5, 7, 11, 17] + periods = [2, 3, 5, 7, 11, 17, 23, 37] + + discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)] + discs = discs + [ + DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods + ] + self.discriminators = nn.ModuleList(discs) + + def forward(self, y, y_hat): + y_d_rs = [] # + y_d_gs = [] + fmap_rs = [] + fmap_gs = [] + for i, d in enumerate(self.discriminators): + y_d_r, fmap_r = d(y) + y_d_g, fmap_g = d(y_hat) + # for j in range(len(fmap_r)): + # print(i,j,y.shape,y_hat.shape,fmap_r[j].shape,fmap_g[j].shape) + y_d_rs.append(y_d_r) + y_d_gs.append(y_d_g) + fmap_rs.append(fmap_r) + fmap_gs.append(fmap_g) + + return y_d_rs, y_d_gs, fmap_rs, fmap_gs + + +class DiscriminatorS(torch.nn.Module): + def __init__(self, use_spectral_norm=False): + super(DiscriminatorS, self).__init__() + norm_f = weight_norm if use_spectral_norm == False else spectral_norm + self.convs = nn.ModuleList( + [ + norm_f(Conv1d(1, 16, 15, 1, padding=7)), + norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)), + norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)), + norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)), + norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)), + norm_f(Conv1d(1024, 1024, 5, 1, padding=2)), + ] + ) + self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1)) + + def forward(self, x): + fmap = [] + + for l in self.convs: + x = l(x) + x = F.leaky_relu(x, modules.LRELU_SLOPE) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return x, fmap + + +class DiscriminatorP(torch.nn.Module): + def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False): + super(DiscriminatorP, self).__init__() + self.period = period + self.use_spectral_norm = use_spectral_norm + norm_f = weight_norm if use_spectral_norm == False else spectral_norm + self.convs = nn.ModuleList( + [ + norm_f( + Conv2d( + 1, + 32, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(kernel_size, 1), 0), + ) + ), + norm_f( + Conv2d( + 32, + 128, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(kernel_size, 1), 0), + ) + ), + norm_f( + Conv2d( + 128, + 512, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(kernel_size, 1), 0), + ) + ), + norm_f( + Conv2d( + 512, + 1024, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(kernel_size, 1), 0), + ) + ), + norm_f( + Conv2d( + 1024, + 1024, + (kernel_size, 1), + 1, + padding=(get_padding(kernel_size, 1), 0), + ) + ), + ] + ) + self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0))) + + def forward(self, x): + fmap = [] + + # 1d to 2d + b, c, t = x.shape + if t % self.period != 0: # pad first + n_pad = self.period - (t % self.period) + if has_xpu and x.dtype == torch.bfloat16: + x = F.pad(x.to(dtype=torch.float16), (0, n_pad), "reflect").to( + dtype=torch.bfloat16 + ) + else: + x = F.pad(x, (0, n_pad), "reflect") + t = t + n_pad + x = x.view(b, c, t // self.period, self.period) + + for l in self.convs: + x = l(x) + x = F.leaky_relu(x, modules.LRELU_SLOPE) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return x, fmap diff --git a/infer/lib/infer_pack/models_onnx.py b/infer/lib/infer_pack/models_onnx.py new file mode 100644 index 0000000..e327019 --- /dev/null +++ b/infer/lib/infer_pack/models_onnx.py @@ -0,0 +1,818 @@ +############################## Warning! ############################## +# # +# Onnx Export Not Support All Of Non-Torch Types # +# Include Python Built-in Types!!!!!!!!!!!!!!!!! # +# If You Want TO Change This File # +# Do Not Use All Of Non-Torch Types! # +# # +############################## Warning! ############################## + +import math +import logging + +logger = logging.getLogger(__name__) + +import numpy as np +import torch +from torch import nn +from torch.nn import AvgPool1d, Conv1d, Conv2d, ConvTranspose1d +from torch.nn import functional as F +from torch.nn.utils import remove_weight_norm, spectral_norm, weight_norm + +from infer.lib.infer_pack import commons, modules +import infer.lib.infer_pack.attentions_onnx as attentions +from infer.lib.infer_pack.commons import get_padding, init_weights + + +class TextEncoder256(nn.Module): + def __init__( + self, + out_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + f0=True, + ): + super().__init__() + self.out_channels = out_channels + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.emb_phone = nn.Linear(256, hidden_channels) + self.lrelu = nn.LeakyReLU(0.1, inplace=True) + if f0 == True: + self.emb_pitch = nn.Embedding(256, hidden_channels) # pitch 256 + self.encoder = attentions.Encoder( + hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout + ) + self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1) + + def forward(self, phone, pitch, lengths): + if pitch == None: + x = self.emb_phone(phone) + else: + x = self.emb_phone(phone) + self.emb_pitch(pitch) + x = x * math.sqrt(self.hidden_channels) # [b, t, h] + x = self.lrelu(x) + x = torch.transpose(x, 1, -1) # [b, h, t] + x_mask = torch.unsqueeze(commons.sequence_mask(lengths, x.size(2)), 1).to( + x.dtype + ) + x = self.encoder(x * x_mask, x_mask) + stats = self.proj(x) * x_mask + + m, logs = torch.split(stats, self.out_channels, dim=1) + return m, logs, x_mask + + +class TextEncoder768(nn.Module): + def __init__( + self, + out_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + f0=True, + ): + super().__init__() + self.out_channels = out_channels + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.emb_phone = nn.Linear(768, hidden_channels) + self.lrelu = nn.LeakyReLU(0.1, inplace=True) + if f0 == True: + self.emb_pitch = nn.Embedding(256, hidden_channels) # pitch 256 + self.encoder = attentions.Encoder( + hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout + ) + self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1) + + def forward(self, phone, pitch, lengths): + if pitch == None: + x = self.emb_phone(phone) + else: + x = self.emb_phone(phone) + self.emb_pitch(pitch) + x = x * math.sqrt(self.hidden_channels) # [b, t, h] + x = self.lrelu(x) + x = torch.transpose(x, 1, -1) # [b, h, t] + x_mask = torch.unsqueeze(commons.sequence_mask(lengths, x.size(2)), 1).to( + x.dtype + ) + x = self.encoder(x * x_mask, x_mask) + stats = self.proj(x) * x_mask + + m, logs = torch.split(stats, self.out_channels, dim=1) + return m, logs, x_mask + + +class ResidualCouplingBlock(nn.Module): + def __init__( + self, + channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + n_flows=4, + gin_channels=0, + ): + super().__init__() + self.channels = channels + self.hidden_channels = hidden_channels + self.kernel_size = kernel_size + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.n_flows = n_flows + self.gin_channels = gin_channels + + self.flows = nn.ModuleList() + for i in range(n_flows): + self.flows.append( + modules.ResidualCouplingLayer( + channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + gin_channels=gin_channels, + mean_only=True, + ) + ) + self.flows.append(modules.Flip()) + + def forward(self, x, x_mask, g=None, reverse=False): + if not reverse: + for flow in self.flows: + x, _ = flow(x, x_mask, g=g, reverse=reverse) + else: + for flow in reversed(self.flows): + x, _ = flow(x, x_mask, g=g, reverse=reverse) + return x + + def remove_weight_norm(self): + for i in range(self.n_flows): + self.flows[i * 2].remove_weight_norm() + + +class PosteriorEncoder(nn.Module): + def __init__( + self, + in_channels, + out_channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + gin_channels=0, + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.hidden_channels = hidden_channels + self.kernel_size = kernel_size + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.gin_channels = gin_channels + + self.pre = nn.Conv1d(in_channels, hidden_channels, 1) + self.enc = modules.WN( + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + gin_channels=gin_channels, + ) + self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1) + + def forward(self, x, x_lengths, g=None): + x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to( + x.dtype + ) + x = self.pre(x) * x_mask + x = self.enc(x, x_mask, g=g) + stats = self.proj(x) * x_mask + m, logs = torch.split(stats, self.out_channels, dim=1) + z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask + return z, m, logs, x_mask + + def remove_weight_norm(self): + self.enc.remove_weight_norm() + + +class Generator(torch.nn.Module): + def __init__( + self, + initial_channel, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + gin_channels=0, + ): + super(Generator, self).__init__() + self.num_kernels = len(resblock_kernel_sizes) + self.num_upsamples = len(upsample_rates) + self.conv_pre = Conv1d( + initial_channel, upsample_initial_channel, 7, 1, padding=3 + ) + resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2 + + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): + self.ups.append( + weight_norm( + ConvTranspose1d( + upsample_initial_channel // (2**i), + upsample_initial_channel // (2 ** (i + 1)), + k, + u, + padding=(k - u) // 2, + ) + ) + ) + + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + ch = upsample_initial_channel // (2 ** (i + 1)) + for j, (k, d) in enumerate( + zip(resblock_kernel_sizes, resblock_dilation_sizes) + ): + self.resblocks.append(resblock(ch, k, d)) + + self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False) + self.ups.apply(init_weights) + + if gin_channels != 0: + self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1) + + def forward(self, x, g=None): + x = self.conv_pre(x) + if g is not None: + x = x + self.cond(g) + + for i in range(self.num_upsamples): + x = F.leaky_relu(x, modules.LRELU_SLOPE) + x = self.ups[i](x) + xs = None + for j in range(self.num_kernels): + if xs is None: + xs = self.resblocks[i * self.num_kernels + j](x) + else: + xs += self.resblocks[i * self.num_kernels + j](x) + x = xs / self.num_kernels + x = F.leaky_relu(x) + x = self.conv_post(x) + x = torch.tanh(x) + + return x + + def remove_weight_norm(self): + for l in self.ups: + remove_weight_norm(l) + for l in self.resblocks: + l.remove_weight_norm() + + +class SineGen(torch.nn.Module): + """Definition of sine generator + SineGen(samp_rate, harmonic_num = 0, + sine_amp = 0.1, noise_std = 0.003, + voiced_threshold = 0, + flag_for_pulse=False) + samp_rate: sampling rate in Hz + harmonic_num: number of harmonic overtones (default 0) + sine_amp: amplitude of sine-wavefrom (default 0.1) + noise_std: std of Gaussian noise (default 0.003) + voiced_thoreshold: F0 threshold for U/V classification (default 0) + flag_for_pulse: this SinGen is used inside PulseGen (default False) + Note: when flag_for_pulse is True, the first time step of a voiced + segment is always sin(np.pi) or cos(0) + """ + + def __init__( + self, + samp_rate, + harmonic_num=0, + sine_amp=0.1, + noise_std=0.003, + voiced_threshold=0, + flag_for_pulse=False, + ): + super(SineGen, self).__init__() + self.sine_amp = sine_amp + self.noise_std = noise_std + self.harmonic_num = harmonic_num + self.dim = self.harmonic_num + 1 + self.sampling_rate = samp_rate + self.voiced_threshold = voiced_threshold + + def _f02uv(self, f0): + # generate uv signal + uv = torch.ones_like(f0) + uv = uv * (f0 > self.voiced_threshold) + if uv.device.type == "privateuseone": # for DirectML + uv = uv.float() + return uv + + def _f02sine(self, f0, upp): + """ f0: (batchsize, length, dim) + where dim indicates fundamental tone and overtones + """ + a = torch.arange(1, upp + 1, dtype=f0.dtype, device=f0.device) + rad = f0 / self.sampling_rate * a + rad2 = torch.fmod(rad[:, :-1, -1:].float() + 0.5, 1.0) - 0.5 + rad_acc = rad2.cumsum(dim=1).fmod(1.0).to(f0) + rad += F.pad(rad_acc, (0, 0, 1, 0), mode='constant') + rad = rad.reshape(f0.shape[0], -1, 1) + b = torch.arange(1, self.dim + 1, dtype=f0.dtype, device=f0.device).reshape(1, 1, -1) + rad *= b + rand_ini = torch.rand(1, 1, self.dim, device=f0.device) + rand_ini[..., 0] = 0 + rad += rand_ini + sines = torch.sin(2 * np.pi * rad) + return sines + + def forward(self, f0: torch.Tensor, upp: int): + """sine_tensor, uv = forward(f0) + input F0: tensor(batchsize=1, length, dim=1) + f0 for unvoiced steps should be 0 + output sine_tensor: tensor(batchsize=1, length, dim) + output uv: tensor(batchsize=1, length, 1) + """ + with torch.no_grad(): + f0 = f0.unsqueeze(-1) + sine_waves = self._f02sine(f0, upp) * self.sine_amp + uv = self._f02uv(f0) + uv = F.interpolate( + uv.transpose(2, 1), scale_factor=float(upp), mode="nearest" + ).transpose(2, 1) + noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3 + noise = noise_amp * torch.randn_like(sine_waves) + sine_waves = sine_waves * uv + noise + return sine_waves, uv, noise + + +class SourceModuleHnNSF(torch.nn.Module): + """SourceModule for hn-nsf + SourceModule(sampling_rate, harmonic_num=0, sine_amp=0.1, + add_noise_std=0.003, voiced_threshod=0) + sampling_rate: sampling_rate in Hz + harmonic_num: number of harmonic above F0 (default: 0) + sine_amp: amplitude of sine source signal (default: 0.1) + add_noise_std: std of additive Gaussian noise (default: 0.003) + note that amplitude of noise in unvoiced is decided + by sine_amp + voiced_threshold: threhold to set U/V given F0 (default: 0) + Sine_source, noise_source = SourceModuleHnNSF(F0_sampled) + F0_sampled (batchsize, length, 1) + Sine_source (batchsize, length, 1) + noise_source (batchsize, length 1) + uv (batchsize, length, 1) + """ + + def __init__( + self, + sampling_rate, + harmonic_num=0, + sine_amp=0.1, + add_noise_std=0.003, + voiced_threshod=0, + is_half=True, + ): + super(SourceModuleHnNSF, self).__init__() + + self.sine_amp = sine_amp + self.noise_std = add_noise_std + self.is_half = is_half + # to produce sine waveforms + self.l_sin_gen = SineGen( + sampling_rate, harmonic_num, sine_amp, add_noise_std, voiced_threshod + ) + + # to merge source harmonics into a single excitation + self.l_linear = torch.nn.Linear(harmonic_num + 1, 1) + self.l_tanh = torch.nn.Tanh() + + def forward(self, x, upp=None): + sine_wavs, uv, _ = self.l_sin_gen(x, upp) + if self.is_half: + sine_wavs = sine_wavs.half() + sine_merge = self.l_tanh(self.l_linear(sine_wavs)) + return sine_merge, None, None # noise, uv + + +class GeneratorNSF(torch.nn.Module): + def __init__( + self, + initial_channel, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + gin_channels, + sr, + is_half=False, + ): + super(GeneratorNSF, self).__init__() + self.num_kernels = len(resblock_kernel_sizes) + self.num_upsamples = len(upsample_rates) + + self.f0_upsamp = torch.nn.Upsample(scale_factor=np.prod(upsample_rates)) + self.m_source = SourceModuleHnNSF( + sampling_rate=sr, harmonic_num=0, is_half=is_half + ) + self.noise_convs = nn.ModuleList() + self.conv_pre = Conv1d( + initial_channel, upsample_initial_channel, 7, 1, padding=3 + ) + resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2 + + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): + c_cur = upsample_initial_channel // (2 ** (i + 1)) + self.ups.append( + weight_norm( + ConvTranspose1d( + upsample_initial_channel // (2**i), + upsample_initial_channel // (2 ** (i + 1)), + k, + u, + padding=(k - u) // 2, + ) + ) + ) + if i + 1 < len(upsample_rates): + stride_f0 = np.prod(upsample_rates[i + 1 :]) + self.noise_convs.append( + Conv1d( + 1, + c_cur, + kernel_size=stride_f0 * 2, + stride=stride_f0, + padding=stride_f0 // 2, + ) + ) + else: + self.noise_convs.append(Conv1d(1, c_cur, kernel_size=1)) + + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + ch = upsample_initial_channel // (2 ** (i + 1)) + for j, (k, d) in enumerate( + zip(resblock_kernel_sizes, resblock_dilation_sizes) + ): + self.resblocks.append(resblock(ch, k, d)) + + self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False) + self.ups.apply(init_weights) + + if gin_channels != 0: + self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1) + + self.upp = np.prod(upsample_rates) + + def forward(self, x, f0, g=None): + har_source, noi_source, uv = self.m_source(f0, self.upp) + har_source = har_source.transpose(1, 2) + x = self.conv_pre(x) + if g is not None: + x = x + self.cond(g) + + for i in range(self.num_upsamples): + x = F.leaky_relu(x, modules.LRELU_SLOPE) + x = self.ups[i](x) + x_source = self.noise_convs[i](har_source) + x = x + x_source + xs = None + for j in range(self.num_kernels): + if xs is None: + xs = self.resblocks[i * self.num_kernels + j](x) + else: + xs += self.resblocks[i * self.num_kernels + j](x) + x = xs / self.num_kernels + x = F.leaky_relu(x) + x = self.conv_post(x) + x = torch.tanh(x) + return x + + def remove_weight_norm(self): + for l in self.ups: + remove_weight_norm(l) + for l in self.resblocks: + l.remove_weight_norm() + + +sr2sr = { + "32k": 32000, + "40k": 40000, + "48k": 48000, +} + + +class SynthesizerTrnMsNSFsidM(nn.Module): + def __init__( + self, + spec_channels, + segment_size, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + spk_embed_dim, + gin_channels, + sr, + version, + **kwargs, + ): + super().__init__() + if type(sr) == type("strr"): + sr = sr2sr[sr] + self.spec_channels = spec_channels + self.inter_channels = inter_channels + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.resblock = resblock + self.resblock_kernel_sizes = resblock_kernel_sizes + self.resblock_dilation_sizes = resblock_dilation_sizes + self.upsample_rates = upsample_rates + self.upsample_initial_channel = upsample_initial_channel + self.upsample_kernel_sizes = upsample_kernel_sizes + self.segment_size = segment_size + self.gin_channels = gin_channels + # self.hop_length = hop_length# + self.spk_embed_dim = spk_embed_dim + if version == "v1": + self.enc_p = TextEncoder256( + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + ) + else: + self.enc_p = TextEncoder768( + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + ) + self.dec = GeneratorNSF( + inter_channels, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + gin_channels=gin_channels, + sr=sr, + is_half=kwargs["is_half"], + ) + self.enc_q = PosteriorEncoder( + spec_channels, + inter_channels, + hidden_channels, + 5, + 1, + 16, + gin_channels=gin_channels, + ) + self.flow = ResidualCouplingBlock( + inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels + ) + self.emb_g = nn.Embedding(self.spk_embed_dim, gin_channels) + self.speaker_map = None + logger.debug( + f"gin_channels: {gin_channels}, self.spk_embed_dim: {self.spk_embed_dim}" + ) + + def remove_weight_norm(self): + self.dec.remove_weight_norm() + self.flow.remove_weight_norm() + self.enc_q.remove_weight_norm() + + def construct_spkmixmap(self, n_speaker): + self.speaker_map = torch.zeros((n_speaker, 1, 1, self.gin_channels)) + for i in range(n_speaker): + self.speaker_map[i] = self.emb_g(torch.LongTensor([[i]])) + self.speaker_map = self.speaker_map.unsqueeze(0) + + def forward(self, phone, phone_lengths, pitch, nsff0, g, rnd, max_len=None): + if self.speaker_map is not None: # [N, S] * [S, B, 1, H] + g = g.reshape((g.shape[0], g.shape[1], 1, 1, 1)) # [N, S, B, 1, 1] + g = g * self.speaker_map # [N, S, B, 1, H] + g = torch.sum(g, dim=1) # [N, 1, B, 1, H] + g = g.transpose(0, -1).transpose(0, -2).squeeze(0) # [B, H, N] + else: + g = g.unsqueeze(0) + g = self.emb_g(g).transpose(1, 2) + + m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths) + z_p = (m_p + torch.exp(logs_p) * rnd) * x_mask + z = self.flow(z_p, x_mask, g=g, reverse=True) + o = self.dec((z * x_mask)[:, :, :max_len], nsff0, g=g) + return o + + +class MultiPeriodDiscriminator(torch.nn.Module): + def __init__(self, use_spectral_norm=False): + super(MultiPeriodDiscriminator, self).__init__() + periods = [2, 3, 5, 7, 11, 17] + # periods = [3, 5, 7, 11, 17, 23, 37] + + discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)] + discs = discs + [ + DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods + ] + self.discriminators = nn.ModuleList(discs) + + def forward(self, y, y_hat): + y_d_rs = [] # + y_d_gs = [] + fmap_rs = [] + fmap_gs = [] + for i, d in enumerate(self.discriminators): + y_d_r, fmap_r = d(y) + y_d_g, fmap_g = d(y_hat) + # for j in range(len(fmap_r)): + # print(i,j,y.shape,y_hat.shape,fmap_r[j].shape,fmap_g[j].shape) + y_d_rs.append(y_d_r) + y_d_gs.append(y_d_g) + fmap_rs.append(fmap_r) + fmap_gs.append(fmap_g) + + return y_d_rs, y_d_gs, fmap_rs, fmap_gs + + +class MultiPeriodDiscriminatorV2(torch.nn.Module): + def __init__(self, use_spectral_norm=False): + super(MultiPeriodDiscriminatorV2, self).__init__() + # periods = [2, 3, 5, 7, 11, 17] + periods = [2, 3, 5, 7, 11, 17, 23, 37] + + discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)] + discs = discs + [ + DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods + ] + self.discriminators = nn.ModuleList(discs) + + def forward(self, y, y_hat): + y_d_rs = [] # + y_d_gs = [] + fmap_rs = [] + fmap_gs = [] + for i, d in enumerate(self.discriminators): + y_d_r, fmap_r = d(y) + y_d_g, fmap_g = d(y_hat) + # for j in range(len(fmap_r)): + # print(i,j,y.shape,y_hat.shape,fmap_r[j].shape,fmap_g[j].shape) + y_d_rs.append(y_d_r) + y_d_gs.append(y_d_g) + fmap_rs.append(fmap_r) + fmap_gs.append(fmap_g) + + return y_d_rs, y_d_gs, fmap_rs, fmap_gs + + +class DiscriminatorS(torch.nn.Module): + def __init__(self, use_spectral_norm=False): + super(DiscriminatorS, self).__init__() + norm_f = weight_norm if use_spectral_norm == False else spectral_norm + self.convs = nn.ModuleList( + [ + norm_f(Conv1d(1, 16, 15, 1, padding=7)), + norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)), + norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)), + norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)), + norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)), + norm_f(Conv1d(1024, 1024, 5, 1, padding=2)), + ] + ) + self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1)) + + def forward(self, x): + fmap = [] + + for l in self.convs: + x = l(x) + x = F.leaky_relu(x, modules.LRELU_SLOPE) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return x, fmap + + +class DiscriminatorP(torch.nn.Module): + def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False): + super(DiscriminatorP, self).__init__() + self.period = period + self.use_spectral_norm = use_spectral_norm + norm_f = weight_norm if use_spectral_norm == False else spectral_norm + self.convs = nn.ModuleList( + [ + norm_f( + Conv2d( + 1, + 32, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(kernel_size, 1), 0), + ) + ), + norm_f( + Conv2d( + 32, + 128, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(kernel_size, 1), 0), + ) + ), + norm_f( + Conv2d( + 128, + 512, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(kernel_size, 1), 0), + ) + ), + norm_f( + Conv2d( + 512, + 1024, + (kernel_size, 1), + (stride, 1), + padding=(get_padding(kernel_size, 1), 0), + ) + ), + norm_f( + Conv2d( + 1024, + 1024, + (kernel_size, 1), + 1, + padding=(get_padding(kernel_size, 1), 0), + ) + ), + ] + ) + self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0))) + + def forward(self, x): + fmap = [] + + # 1d to 2d + b, c, t = x.shape + if t % self.period != 0: # pad first + n_pad = self.period - (t % self.period) + x = F.pad(x, (0, n_pad), "reflect") + t = t + n_pad + x = x.view(b, c, t // self.period, self.period) + + for l in self.convs: + x = l(x) + x = F.leaky_relu(x, modules.LRELU_SLOPE) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return x, fmap diff --git a/infer/lib/infer_pack/modules.py b/infer/lib/infer_pack/modules.py new file mode 100644 index 0000000..51aeaf0 --- /dev/null +++ b/infer/lib/infer_pack/modules.py @@ -0,0 +1,615 @@ +import copy +import math +from typing import Optional, Tuple + +import numpy as np +import scipy +import torch +from torch import nn +from torch.nn import AvgPool1d, Conv1d, Conv2d, ConvTranspose1d +from torch.nn import functional as F +from torch.nn.utils import remove_weight_norm, weight_norm + +from infer.lib.infer_pack import commons +from infer.lib.infer_pack.commons import get_padding, init_weights +from infer.lib.infer_pack.transforms import piecewise_rational_quadratic_transform + +LRELU_SLOPE = 0.1 + + +class LayerNorm(nn.Module): + def __init__(self, channels, eps=1e-5): + super(LayerNorm, self).__init__() + self.channels = channels + self.eps = eps + + self.gamma = nn.Parameter(torch.ones(channels)) + self.beta = nn.Parameter(torch.zeros(channels)) + + def forward(self, x): + x = x.transpose(1, -1) + x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps) + return x.transpose(1, -1) + + +class ConvReluNorm(nn.Module): + def __init__( + self, + in_channels, + hidden_channels, + out_channels, + kernel_size, + n_layers, + p_dropout, + ): + super(ConvReluNorm, self).__init__() + self.in_channels = in_channels + self.hidden_channels = hidden_channels + self.out_channels = out_channels + self.kernel_size = kernel_size + self.n_layers = n_layers + self.p_dropout = float(p_dropout) + assert n_layers > 1, "Number of layers should be larger than 0." + + self.conv_layers = nn.ModuleList() + self.norm_layers = nn.ModuleList() + self.conv_layers.append( + nn.Conv1d( + in_channels, hidden_channels, kernel_size, padding=kernel_size // 2 + ) + ) + self.norm_layers.append(LayerNorm(hidden_channels)) + self.relu_drop = nn.Sequential(nn.ReLU(), nn.Dropout(float(p_dropout))) + for _ in range(n_layers - 1): + self.conv_layers.append( + nn.Conv1d( + hidden_channels, + hidden_channels, + kernel_size, + padding=kernel_size // 2, + ) + ) + self.norm_layers.append(LayerNorm(hidden_channels)) + self.proj = nn.Conv1d(hidden_channels, out_channels, 1) + self.proj.weight.data.zero_() + self.proj.bias.data.zero_() + + def forward(self, x, x_mask): + x_org = x + for i in range(self.n_layers): + x = self.conv_layers[i](x * x_mask) + x = self.norm_layers[i](x) + x = self.relu_drop(x) + x = x_org + self.proj(x) + return x * x_mask + + +class DDSConv(nn.Module): + """ + Dialted and Depth-Separable Convolution + """ + + def __init__(self, channels, kernel_size, n_layers, p_dropout=0.0): + super(DDSConv, self).__init__() + self.channels = channels + self.kernel_size = kernel_size + self.n_layers = n_layers + self.p_dropout = float(p_dropout) + + self.drop = nn.Dropout(float(p_dropout)) + self.convs_sep = nn.ModuleList() + self.convs_1x1 = nn.ModuleList() + self.norms_1 = nn.ModuleList() + self.norms_2 = nn.ModuleList() + for i in range(n_layers): + dilation = kernel_size**i + padding = (kernel_size * dilation - dilation) // 2 + self.convs_sep.append( + nn.Conv1d( + channels, + channels, + kernel_size, + groups=channels, + dilation=dilation, + padding=padding, + ) + ) + self.convs_1x1.append(nn.Conv1d(channels, channels, 1)) + self.norms_1.append(LayerNorm(channels)) + self.norms_2.append(LayerNorm(channels)) + + def forward(self, x, x_mask, g: Optional[torch.Tensor] = None): + if g is not None: + x = x + g + for i in range(self.n_layers): + y = self.convs_sep[i](x * x_mask) + y = self.norms_1[i](y) + y = F.gelu(y) + y = self.convs_1x1[i](y) + y = self.norms_2[i](y) + y = F.gelu(y) + y = self.drop(y) + x = x + y + return x * x_mask + + +class WN(torch.nn.Module): + def __init__( + self, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + gin_channels=0, + p_dropout=0, + ): + super(WN, self).__init__() + assert kernel_size % 2 == 1 + self.hidden_channels = hidden_channels + self.kernel_size = (kernel_size,) + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.gin_channels = gin_channels + self.p_dropout = float(p_dropout) + + self.in_layers = torch.nn.ModuleList() + self.res_skip_layers = torch.nn.ModuleList() + self.drop = nn.Dropout(float(p_dropout)) + + if gin_channels != 0: + cond_layer = torch.nn.Conv1d( + gin_channels, 2 * hidden_channels * n_layers, 1 + ) + self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name="weight") + + for i in range(n_layers): + dilation = dilation_rate**i + padding = int((kernel_size * dilation - dilation) / 2) + in_layer = torch.nn.Conv1d( + hidden_channels, + 2 * hidden_channels, + kernel_size, + dilation=dilation, + padding=padding, + ) + in_layer = torch.nn.utils.weight_norm(in_layer, name="weight") + self.in_layers.append(in_layer) + + # last one is not necessary + if i < n_layers - 1: + res_skip_channels = 2 * hidden_channels + else: + res_skip_channels = hidden_channels + + res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1) + res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name="weight") + self.res_skip_layers.append(res_skip_layer) + + def forward( + self, x: torch.Tensor, x_mask: torch.Tensor, g: Optional[torch.Tensor] = None + ): + output = torch.zeros_like(x) + n_channels_tensor = torch.IntTensor([self.hidden_channels]) + + if g is not None: + g = self.cond_layer(g) + + for i, (in_layer, res_skip_layer) in enumerate( + zip(self.in_layers, self.res_skip_layers) + ): + x_in = in_layer(x) + if g is not None: + cond_offset = i * 2 * self.hidden_channels + g_l = g[:, cond_offset : cond_offset + 2 * self.hidden_channels, :] + else: + g_l = torch.zeros_like(x_in) + + acts = commons.fused_add_tanh_sigmoid_multiply(x_in, g_l, n_channels_tensor) + acts = self.drop(acts) + + res_skip_acts = res_skip_layer(acts) + if i < self.n_layers - 1: + res_acts = res_skip_acts[:, : self.hidden_channels, :] + x = (x + res_acts) * x_mask + output = output + res_skip_acts[:, self.hidden_channels :, :] + else: + output = output + res_skip_acts + return output * x_mask + + def remove_weight_norm(self): + if self.gin_channels != 0: + torch.nn.utils.remove_weight_norm(self.cond_layer) + for l in self.in_layers: + torch.nn.utils.remove_weight_norm(l) + for l in self.res_skip_layers: + torch.nn.utils.remove_weight_norm(l) + + def __prepare_scriptable__(self): + if self.gin_channels != 0: + for hook in self.cond_layer._forward_pre_hooks.values(): + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(self.cond_layer) + for l in self.in_layers: + for hook in l._forward_pre_hooks.values(): + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(l) + for l in self.res_skip_layers: + for hook in l._forward_pre_hooks.values(): + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(l) + return self + + +class ResBlock1(torch.nn.Module): + def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)): + super(ResBlock1, self).__init__() + self.convs1 = nn.ModuleList( + [ + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[0], + padding=get_padding(kernel_size, dilation[0]), + ) + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[1], + padding=get_padding(kernel_size, dilation[1]), + ) + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[2], + padding=get_padding(kernel_size, dilation[2]), + ) + ), + ] + ) + self.convs1.apply(init_weights) + + self.convs2 = nn.ModuleList( + [ + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=1, + padding=get_padding(kernel_size, 1), + ) + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=1, + padding=get_padding(kernel_size, 1), + ) + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=1, + padding=get_padding(kernel_size, 1), + ) + ), + ] + ) + self.convs2.apply(init_weights) + self.lrelu_slope = LRELU_SLOPE + + def forward(self, x: torch.Tensor, x_mask: Optional[torch.Tensor] = None): + for c1, c2 in zip(self.convs1, self.convs2): + xt = F.leaky_relu(x, self.lrelu_slope) + if x_mask is not None: + xt = xt * x_mask + xt = c1(xt) + xt = F.leaky_relu(xt, self.lrelu_slope) + if x_mask is not None: + xt = xt * x_mask + xt = c2(xt) + x = xt + x + if x_mask is not None: + x = x * x_mask + return x + + def remove_weight_norm(self): + for l in self.convs1: + remove_weight_norm(l) + for l in self.convs2: + remove_weight_norm(l) + + def __prepare_scriptable__(self): + for l in self.convs1: + for hook in l._forward_pre_hooks.values(): + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(l) + for l in self.convs2: + for hook in l._forward_pre_hooks.values(): + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(l) + return self + + +class ResBlock2(torch.nn.Module): + def __init__(self, channels, kernel_size=3, dilation=(1, 3)): + super(ResBlock2, self).__init__() + self.convs = nn.ModuleList( + [ + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[0], + padding=get_padding(kernel_size, dilation[0]), + ) + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[1], + padding=get_padding(kernel_size, dilation[1]), + ) + ), + ] + ) + self.convs.apply(init_weights) + self.lrelu_slope = LRELU_SLOPE + + def forward(self, x, x_mask: Optional[torch.Tensor] = None): + for c in self.convs: + xt = F.leaky_relu(x, self.lrelu_slope) + if x_mask is not None: + xt = xt * x_mask + xt = c(xt) + x = xt + x + if x_mask is not None: + x = x * x_mask + return x + + def remove_weight_norm(self): + for l in self.convs: + remove_weight_norm(l) + + def __prepare_scriptable__(self): + for l in self.convs: + for hook in l._forward_pre_hooks.values(): + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(l) + return self + + +class Log(nn.Module): + def forward( + self, + x: torch.Tensor, + x_mask: torch.Tensor, + g: Optional[torch.Tensor] = None, + reverse: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + if not reverse: + y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask + logdet = torch.sum(-y, [1, 2]) + return y, logdet + else: + x = torch.exp(x) * x_mask + return x + + +class Flip(nn.Module): + # torch.jit.script() Compiled functions \ + # can't take variable number of arguments or \ + # use keyword-only arguments with defaults + def forward( + self, + x: torch.Tensor, + x_mask: torch.Tensor, + g: Optional[torch.Tensor] = None, + reverse: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + x = torch.flip(x, [1]) + if not reverse: + logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device) + return x, logdet + else: + return x, torch.zeros([1], device=x.device) + + +class ElementwiseAffine(nn.Module): + def __init__(self, channels): + super(ElementwiseAffine, self).__init__() + self.channels = channels + self.m = nn.Parameter(torch.zeros(channels, 1)) + self.logs = nn.Parameter(torch.zeros(channels, 1)) + + def forward(self, x, x_mask, reverse=False, **kwargs): + if not reverse: + y = self.m + torch.exp(self.logs) * x + y = y * x_mask + logdet = torch.sum(self.logs * x_mask, [1, 2]) + return y, logdet + else: + x = (x - self.m) * torch.exp(-self.logs) * x_mask + return x + + +class ResidualCouplingLayer(nn.Module): + def __init__( + self, + channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + p_dropout=0, + gin_channels=0, + mean_only=False, + ): + assert channels % 2 == 0, "channels should be divisible by 2" + super(ResidualCouplingLayer, self).__init__() + self.channels = channels + self.hidden_channels = hidden_channels + self.kernel_size = kernel_size + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.half_channels = channels // 2 + self.mean_only = mean_only + + self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1) + self.enc = WN( + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + p_dropout=float(p_dropout), + gin_channels=gin_channels, + ) + self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1) + self.post.weight.data.zero_() + self.post.bias.data.zero_() + + def forward( + self, + x: torch.Tensor, + x_mask: torch.Tensor, + g: Optional[torch.Tensor] = None, + reverse: bool = False, + ): + x0, x1 = torch.split(x, [self.half_channels] * 2, 1) + h = self.pre(x0) * x_mask + h = self.enc(h, x_mask, g=g) + stats = self.post(h) * x_mask + if not self.mean_only: + m, logs = torch.split(stats, [self.half_channels] * 2, 1) + else: + m = stats + logs = torch.zeros_like(m) + + if not reverse: + x1 = m + x1 * torch.exp(logs) * x_mask + x = torch.cat([x0, x1], 1) + logdet = torch.sum(logs, [1, 2]) + return x, logdet + else: + x1 = (x1 - m) * torch.exp(-logs) * x_mask + x = torch.cat([x0, x1], 1) + return x, torch.zeros([1]) + + def remove_weight_norm(self): + self.enc.remove_weight_norm() + + def __prepare_scriptable__(self): + for hook in self.enc._forward_pre_hooks.values(): + if ( + hook.__module__ == "torch.nn.utils.weight_norm" + and hook.__class__.__name__ == "WeightNorm" + ): + torch.nn.utils.remove_weight_norm(self.enc) + return self + + +class ConvFlow(nn.Module): + def __init__( + self, + in_channels, + filter_channels, + kernel_size, + n_layers, + num_bins=10, + tail_bound=5.0, + ): + super(ConvFlow, self).__init__() + self.in_channels = in_channels + self.filter_channels = filter_channels + self.kernel_size = kernel_size + self.n_layers = n_layers + self.num_bins = num_bins + self.tail_bound = tail_bound + self.half_channels = in_channels // 2 + + self.pre = nn.Conv1d(self.half_channels, filter_channels, 1) + self.convs = DDSConv(filter_channels, kernel_size, n_layers, p_dropout=0.0) + self.proj = nn.Conv1d( + filter_channels, self.half_channels * (num_bins * 3 - 1), 1 + ) + self.proj.weight.data.zero_() + self.proj.bias.data.zero_() + + def forward( + self, + x: torch.Tensor, + x_mask: torch.Tensor, + g: Optional[torch.Tensor] = None, + reverse=False, + ): + x0, x1 = torch.split(x, [self.half_channels] * 2, 1) + h = self.pre(x0) + h = self.convs(h, x_mask, g=g) + h = self.proj(h) * x_mask + + b, c, t = x0.shape + h = h.reshape(b, c, -1, t).permute(0, 1, 3, 2) # [b, cx?, t] -> [b, c, t, ?] + + unnormalized_widths = h[..., : self.num_bins] / math.sqrt(self.filter_channels) + unnormalized_heights = h[..., self.num_bins : 2 * self.num_bins] / math.sqrt( + self.filter_channels + ) + unnormalized_derivatives = h[..., 2 * self.num_bins :] + + x1, logabsdet = piecewise_rational_quadratic_transform( + x1, + unnormalized_widths, + unnormalized_heights, + unnormalized_derivatives, + inverse=reverse, + tails="linear", + tail_bound=self.tail_bound, + ) + + x = torch.cat([x0, x1], 1) * x_mask + logdet = torch.sum(logabsdet * x_mask, [1, 2]) + if not reverse: + return x, logdet + else: + return x diff --git a/infer/lib/infer_pack/onnx_inference.py b/infer/lib/infer_pack/onnx_inference.py new file mode 100644 index 0000000..3d8328b --- /dev/null +++ b/infer/lib/infer_pack/onnx_inference.py @@ -0,0 +1,149 @@ +import librosa +import numpy as np +import onnxruntime +import soundfile + +import logging + +logger = logging.getLogger(__name__) + + +class ContentVec: + def __init__(self, vec_path="pretrained/vec-768-layer-12.onnx", device=None): + logger.info("Load model(s) from {}".format(vec_path)) + if device == "cpu" or device is None: + providers = ["CPUExecutionProvider"] + elif device == "cuda": + providers = ["CUDAExecutionProvider", "CPUExecutionProvider"] + elif device == "dml": + providers = ["DmlExecutionProvider"] + else: + raise RuntimeError("Unsportted Device") + self.model = onnxruntime.InferenceSession(vec_path, providers=providers) + + def __call__(self, wav): + return self.forward(wav) + + def forward(self, wav): + feats = wav + if feats.ndim == 2: # double channels + feats = feats.mean(-1) + assert feats.ndim == 1, feats.ndim + feats = np.expand_dims(np.expand_dims(feats, 0), 0) + onnx_input = {self.model.get_inputs()[0].name: feats} + logits = self.model.run(None, onnx_input)[0] + return logits.transpose(0, 2, 1) + + +def get_f0_predictor(f0_predictor, hop_length, sampling_rate, **kargs): + if f0_predictor == "pm": + from lib.infer_pack.modules.F0Predictor.PMF0Predictor import PMF0Predictor + + f0_predictor_object = PMF0Predictor( + hop_length=hop_length, sampling_rate=sampling_rate + ) + elif f0_predictor == "harvest": + from lib.infer_pack.modules.F0Predictor.HarvestF0Predictor import ( + HarvestF0Predictor, + ) + + f0_predictor_object = HarvestF0Predictor( + hop_length=hop_length, sampling_rate=sampling_rate + ) + elif f0_predictor == "dio": + from lib.infer_pack.modules.F0Predictor.DioF0Predictor import DioF0Predictor + + f0_predictor_object = DioF0Predictor( + hop_length=hop_length, sampling_rate=sampling_rate + ) + else: + raise Exception("Unknown f0 predictor") + return f0_predictor_object + + +class OnnxRVC: + def __init__( + self, + model_path, + sr=40000, + hop_size=512, + vec_path="vec-768-layer-12", + device="cpu", + ): + vec_path = f"pretrained/{vec_path}.onnx" + self.vec_model = ContentVec(vec_path, device) + if device == "cpu" or device is None: + providers = ["CPUExecutionProvider"] + elif device == "cuda": + providers = ["CUDAExecutionProvider", "CPUExecutionProvider"] + elif device == "dml": + providers = ["DmlExecutionProvider"] + else: + raise RuntimeError("Unsportted Device") + self.model = onnxruntime.InferenceSession(model_path, providers=providers) + self.sampling_rate = sr + self.hop_size = hop_size + + def forward(self, hubert, hubert_length, pitch, pitchf, ds, rnd): + onnx_input = { + self.model.get_inputs()[0].name: hubert, + self.model.get_inputs()[1].name: hubert_length, + self.model.get_inputs()[2].name: pitch, + self.model.get_inputs()[3].name: pitchf, + self.model.get_inputs()[4].name: ds, + self.model.get_inputs()[5].name: rnd, + } + return (self.model.run(None, onnx_input)[0] * 32767).astype(np.int16) + + def inference( + self, + raw_path, + sid, + f0_method="dio", + f0_up_key=0, + pad_time=0.5, + cr_threshold=0.02, + ): + f0_min = 50 + f0_max = 1100 + f0_mel_min = 1127 * np.log(1 + f0_min / 700) + f0_mel_max = 1127 * np.log(1 + f0_max / 700) + f0_predictor = get_f0_predictor( + f0_method, + hop_length=self.hop_size, + sampling_rate=self.sampling_rate, + threshold=cr_threshold, + ) + wav, sr = librosa.load(raw_path, sr=self.sampling_rate) + org_length = len(wav) + if org_length / sr > 50.0: + raise RuntimeError("Reached Max Length") + + wav16k = librosa.resample(wav, orig_sr=self.sampling_rate, target_sr=16000) + wav16k = wav16k + + hubert = self.vec_model(wav16k) + hubert = np.repeat(hubert, 2, axis=2).transpose(0, 2, 1).astype(np.float32) + hubert_length = hubert.shape[1] + + pitchf = f0_predictor.compute_f0(wav, hubert_length) + pitchf = pitchf * 2 ** (f0_up_key / 12) + pitch = pitchf.copy() + f0_mel = 1127 * np.log(1 + pitch / 700) + f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / ( + f0_mel_max - f0_mel_min + ) + 1 + f0_mel[f0_mel <= 1] = 1 + f0_mel[f0_mel > 255] = 255 + pitch = np.rint(f0_mel).astype(np.int64) + + pitchf = pitchf.reshape(1, len(pitchf)).astype(np.float32) + pitch = pitch.reshape(1, len(pitch)) + ds = np.array([sid]).astype(np.int64) + + rnd = np.random.randn(1, 192, hubert_length).astype(np.float32) + hubert_length = np.array([hubert_length]).astype(np.int64) + + out_wav = self.forward(hubert, hubert_length, pitch, pitchf, ds, rnd).squeeze() + out_wav = np.pad(out_wav, (0, 2 * self.hop_size), "constant") + return out_wav[0:org_length] diff --git a/infer/lib/infer_pack/transforms.py b/infer/lib/infer_pack/transforms.py new file mode 100644 index 0000000..6d07b3b --- /dev/null +++ b/infer/lib/infer_pack/transforms.py @@ -0,0 +1,207 @@ +import numpy as np +import torch +from torch.nn import functional as F + +DEFAULT_MIN_BIN_WIDTH = 1e-3 +DEFAULT_MIN_BIN_HEIGHT = 1e-3 +DEFAULT_MIN_DERIVATIVE = 1e-3 + + +def piecewise_rational_quadratic_transform( + inputs, + unnormalized_widths, + unnormalized_heights, + unnormalized_derivatives, + inverse=False, + tails=None, + tail_bound=1.0, + min_bin_width=DEFAULT_MIN_BIN_WIDTH, + min_bin_height=DEFAULT_MIN_BIN_HEIGHT, + min_derivative=DEFAULT_MIN_DERIVATIVE, +): + if tails is None: + spline_fn = rational_quadratic_spline + spline_kwargs = {} + else: + spline_fn = unconstrained_rational_quadratic_spline + spline_kwargs = {"tails": tails, "tail_bound": tail_bound} + + outputs, logabsdet = spline_fn( + inputs=inputs, + unnormalized_widths=unnormalized_widths, + unnormalized_heights=unnormalized_heights, + unnormalized_derivatives=unnormalized_derivatives, + inverse=inverse, + min_bin_width=min_bin_width, + min_bin_height=min_bin_height, + min_derivative=min_derivative, + **spline_kwargs + ) + return outputs, logabsdet + + +def searchsorted(bin_locations, inputs, eps=1e-6): + bin_locations[..., -1] += eps + return torch.sum(inputs[..., None] >= bin_locations, dim=-1) - 1 + + +def unconstrained_rational_quadratic_spline( + inputs, + unnormalized_widths, + unnormalized_heights, + unnormalized_derivatives, + inverse=False, + tails="linear", + tail_bound=1.0, + min_bin_width=DEFAULT_MIN_BIN_WIDTH, + min_bin_height=DEFAULT_MIN_BIN_HEIGHT, + min_derivative=DEFAULT_MIN_DERIVATIVE, +): + inside_interval_mask = (inputs >= -tail_bound) & (inputs <= tail_bound) + outside_interval_mask = ~inside_interval_mask + + outputs = torch.zeros_like(inputs) + logabsdet = torch.zeros_like(inputs) + + if tails == "linear": + unnormalized_derivatives = F.pad(unnormalized_derivatives, pad=(1, 1)) + constant = np.log(np.exp(1 - min_derivative) - 1) + unnormalized_derivatives[..., 0] = constant + unnormalized_derivatives[..., -1] = constant + + outputs[outside_interval_mask] = inputs[outside_interval_mask] + logabsdet[outside_interval_mask] = 0 + else: + raise RuntimeError("{} tails are not implemented.".format(tails)) + + ( + outputs[inside_interval_mask], + logabsdet[inside_interval_mask], + ) = rational_quadratic_spline( + inputs=inputs[inside_interval_mask], + unnormalized_widths=unnormalized_widths[inside_interval_mask, :], + unnormalized_heights=unnormalized_heights[inside_interval_mask, :], + unnormalized_derivatives=unnormalized_derivatives[inside_interval_mask, :], + inverse=inverse, + left=-tail_bound, + right=tail_bound, + bottom=-tail_bound, + top=tail_bound, + min_bin_width=min_bin_width, + min_bin_height=min_bin_height, + min_derivative=min_derivative, + ) + + return outputs, logabsdet + + +def rational_quadratic_spline( + inputs, + unnormalized_widths, + unnormalized_heights, + unnormalized_derivatives, + inverse=False, + left=0.0, + right=1.0, + bottom=0.0, + top=1.0, + min_bin_width=DEFAULT_MIN_BIN_WIDTH, + min_bin_height=DEFAULT_MIN_BIN_HEIGHT, + min_derivative=DEFAULT_MIN_DERIVATIVE, +): + if torch.min(inputs) < left or torch.max(inputs) > right: + raise ValueError("Input to a transform is not within its domain") + + num_bins = unnormalized_widths.shape[-1] + + if min_bin_width * num_bins > 1.0: + raise ValueError("Minimal bin width too large for the number of bins") + if min_bin_height * num_bins > 1.0: + raise ValueError("Minimal bin height too large for the number of bins") + + widths = F.softmax(unnormalized_widths, dim=-1) + widths = min_bin_width + (1 - min_bin_width * num_bins) * widths + cumwidths = torch.cumsum(widths, dim=-1) + cumwidths = F.pad(cumwidths, pad=(1, 0), mode="constant", value=0.0) + cumwidths = (right - left) * cumwidths + left + cumwidths[..., 0] = left + cumwidths[..., -1] = right + widths = cumwidths[..., 1:] - cumwidths[..., :-1] + + derivatives = min_derivative + F.softplus(unnormalized_derivatives) + + heights = F.softmax(unnormalized_heights, dim=-1) + heights = min_bin_height + (1 - min_bin_height * num_bins) * heights + cumheights = torch.cumsum(heights, dim=-1) + cumheights = F.pad(cumheights, pad=(1, 0), mode="constant", value=0.0) + cumheights = (top - bottom) * cumheights + bottom + cumheights[..., 0] = bottom + cumheights[..., -1] = top + heights = cumheights[..., 1:] - cumheights[..., :-1] + + if inverse: + bin_idx = searchsorted(cumheights, inputs)[..., None] + else: + bin_idx = searchsorted(cumwidths, inputs)[..., None] + + input_cumwidths = cumwidths.gather(-1, bin_idx)[..., 0] + input_bin_widths = widths.gather(-1, bin_idx)[..., 0] + + input_cumheights = cumheights.gather(-1, bin_idx)[..., 0] + delta = heights / widths + input_delta = delta.gather(-1, bin_idx)[..., 0] + + input_derivatives = derivatives.gather(-1, bin_idx)[..., 0] + input_derivatives_plus_one = derivatives[..., 1:].gather(-1, bin_idx)[..., 0] + + input_heights = heights.gather(-1, bin_idx)[..., 0] + + if inverse: + a = (inputs - input_cumheights) * ( + input_derivatives + input_derivatives_plus_one - 2 * input_delta + ) + input_heights * (input_delta - input_derivatives) + b = input_heights * input_derivatives - (inputs - input_cumheights) * ( + input_derivatives + input_derivatives_plus_one - 2 * input_delta + ) + c = -input_delta * (inputs - input_cumheights) + + discriminant = b.pow(2) - 4 * a * c + assert (discriminant >= 0).all() + + root = (2 * c) / (-b - torch.sqrt(discriminant)) + outputs = root * input_bin_widths + input_cumwidths + + theta_one_minus_theta = root * (1 - root) + denominator = input_delta + ( + (input_derivatives + input_derivatives_plus_one - 2 * input_delta) + * theta_one_minus_theta + ) + derivative_numerator = input_delta.pow(2) * ( + input_derivatives_plus_one * root.pow(2) + + 2 * input_delta * theta_one_minus_theta + + input_derivatives * (1 - root).pow(2) + ) + logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator) + + return outputs, -logabsdet + else: + theta = (inputs - input_cumwidths) / input_bin_widths + theta_one_minus_theta = theta * (1 - theta) + + numerator = input_heights * ( + input_delta * theta.pow(2) + input_derivatives * theta_one_minus_theta + ) + denominator = input_delta + ( + (input_derivatives + input_derivatives_plus_one - 2 * input_delta) + * theta_one_minus_theta + ) + outputs = input_cumheights + numerator / denominator + + derivative_numerator = input_delta.pow(2) * ( + input_derivatives_plus_one * theta.pow(2) + + 2 * input_delta * theta_one_minus_theta + + input_derivatives * (1 - theta).pow(2) + ) + logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator) + + return outputs, logabsdet