Merge pull request #544 from idiap/xtts-hf

Download models via `huggingface_hub`

Author: eginhard

TTS/.models.json

@@ -23,24 +23,6 @@
 if not is_torch_available() or not is_torchaudio_available:
     raise ImportError(PYTORCH_IMPORT_ERROR)
 
-if is_torch_greater_or_equal("2.4"):
-    import _codecs
-    from collections import defaultdict
-
-    import numpy as np
-    import torch
-    from packaging import version
-
-    from TTS.config.shared_configs import BaseDatasetConfig
-    from TTS.tts.configs.xtts_config import XttsConfig
-    from TTS.tts.models.xtts import XttsArgs, XttsAudioConfig
-    from TTS.utils.radam import RAdam
-
-    torch.serialization.add_safe_globals([dict, defaultdict, RAdam])
-
-    # XTTS
-    torch.serialization.add_safe_globals([BaseDatasetConfig, XttsConfig, XttsAudioConfig, XttsArgs])
-
 if is_torch_greater_or_equal("2.9"):
     if not is_torchcodec_available():
         raise ImportError(TORCHCODEC_IMPORT_ERROR)

TTS/__init__.py

@@ -8,6 +8,7 @@
 from typing import Any
 
 from torch import nn
+from trainer.io import get_user_data_dir
 
 from TTS.config import load_config
 from TTS.utils.manage import ModelManager
@@ -81,6 +82,7 @@ def __init__(
         self.synthesizer: Synthesizer | None = None
         self.voice_converter: Synthesizer | None = None
         self.model_name = ""
+        self.voice_dir = None
 
         self.vocoder_path = vocoder_path
         self.vocoder_config_path = vocoder_config_path
@@ -93,6 +95,7 @@ def __init__(
             warnings.warn("`gpu` will be deprecated. Please use `tts.to(device)` instead.")
 
         if model_name is not None and len(model_name) > 0:
+            self.voice_dir = get_user_data_dir("tts") / model_name / "voices"
             if "tts_models" in model_name:
                 self.load_tts_model_by_name(model_name, vocoder_name, gpu=gpu)
             elif "voice_conversion_models" in model_name:
@@ -158,22 +161,10 @@ def list_models() -> list[str]:
 
     def download_model_by_name(
         self, model_name: str, vocoder_name: str | None = None
-    ) -> tuple[Path | None, Path | None, Path | None, Path | None, Path | None]:
+    ) -> tuple[Path | None, Path | None, Path | None, Path | None]:
         model_path, config_path, model_item = self.manager.download_model(model_name)
-        if (
-            "fairseq" in model_name
-            or "openvoice" in model_name
-            or (
-                model_item is not None
-                and isinstance(model_item["model_url"], list)
-                and len(model_item["model_url"]) > 2
-            )
-        ):
-            # return model directory if there are multiple files
-            # we assume that the model knows how to load itself
-            return None, None, None, None, model_path
         if model_item.get("default_vocoder") is None:
-            return model_path, config_path, None, None, None
+            return model_path, config_path, None, None
         if vocoder_name is None:
             vocoder_name = model_item["default_vocoder"]
         vocoder_path, vocoder_config_path = None, None
@@ -183,7 +174,7 @@ def download_model_by_name(
             vocoder_config_path = self.vocoder_config_path
         if vocoder_path is None or vocoder_config_path is None:
             vocoder_path, vocoder_config_path, _ = self.manager.download_model(vocoder_name)
-        return model_path, config_path, vocoder_path, vocoder_config_path, None
+        return model_path, config_path, vocoder_path, vocoder_config_path
 
     def load_model_by_name(self, model_name: str, vocoder_name: str | None = None, *, gpu: bool = False) -> None:
         """Load one of the 🐸TTS models by name.
@@ -202,15 +193,15 @@ def load_vc_model_by_name(self, model_name: str, vocoder_name: str | None = None
             gpu (bool, optional): Enable/disable GPU. Some models might be too slow on CPU. Defaults to False.
         """
         self.model_name = model_name
-        model_path, config_path, vocoder_path, vocoder_config_path, model_dir = self.download_model_by_name(
+        model_path, config_path, vocoder_path, vocoder_config_path = self.download_model_by_name(
             model_name, vocoder_name
         )
         self.voice_converter = Synthesizer(
             vc_checkpoint=model_path,
             vc_config=config_path,
             vocoder_checkpoint=vocoder_path,
             vocoder_config=vocoder_config_path,
-            model_dir=model_dir,
+            voice_dir=self.voice_dir,
             use_cuda=gpu,
         )
 
@@ -225,7 +216,7 @@ def load_tts_model_by_name(self, model_name: str, vocoder_name: str | None = Non
         """
         self.model_name = model_name
 
-        model_path, config_path, vocoder_path, vocoder_config_path, model_dir = self.download_model_by_name(
+        model_path, config_path, vocoder_path, vocoder_config_path = self.download_model_by_name(
             model_name, vocoder_name
         )
 
@@ -240,7 +231,7 @@ def load_tts_model_by_name(self, model_name: str, vocoder_name: str | None = Non
             vocoder_config=vocoder_config_path,
             encoder_checkpoint=self.encoder_path,
             encoder_config=self.encoder_config_path,
-            model_dir=model_dir,
+            voice_dir=self.voice_dir,
             use_cuda=gpu,
         )
 
@@ -266,6 +257,7 @@ def load_tts_model_by_path(self, model_path: str, config_path: str, *, gpu: bool
             encoder_config=self.encoder_config_path,
             use_cuda=gpu,
         )
+        self.voice_dir = self.synthesizer.voice_dir
 
     def _check_arguments(
         self,

TTS/api.py

@@ -343,7 +343,6 @@ def main(arg_list: list[str] | None = None) -> None:
         encoder_config_path = None
         vc_path = None
         vc_config_path = None
-        model_dir = None
 
         # 0) Print version number
         if args.version:

TTS/bin/synthesize.py

@@ -4,9 +4,9 @@
 from trainer import Trainer, TrainerArgs
 
 from TTS.config.shared_configs import BaseDatasetConfig
+from TTS.tts.configs.xtts_config import XttsAudioConfig
 from TTS.tts.datasets import load_tts_samples
 from TTS.tts.layers.xtts.trainer.gpt_trainer import GPTArgs, GPTTrainer, GPTTrainerConfig
-from TTS.tts.models.xtts import XttsAudioConfig
 from TTS.utils.manage import ModelManager
 
 

TTS/demos/xtts_ft_demo/utils/gpt_train.py

@@ -7,13 +7,17 @@
 from trainer import TrainerModel
 from trainer.io import load_fsspec
 
+from TTS.config.shared_configs import BaseTrainingConfig
+
 
 class BaseTrainerModel(TrainerModel):
     """BaseTrainerModel model expanding TrainerModel with required functions by 🐸TTS.
 
     Every new Coqui model must inherit it.
     """
 
+    config: BaseTrainingConfig
+
     @staticmethod
     @abstractmethod
     def init_from_config(config: Coqpit) -> "BaseTrainerModel":

TTS/model.py

@@ -1,7 +1,56 @@
 from dataclasses import dataclass, field
 
+from TTS.config.shared_configs import ModelArgs
 from TTS.tts.configs.shared_configs import BaseTTSConfig
-from TTS.tts.models.align_tts import AlignTTSArgs
+
+
+@dataclass
+class AlignTTSArgs(ModelArgs):
+    """
+    Args:
+        num_chars (int):
+            number of unique input to characters
+        out_channels (int):
+            number of output tensor channels. It is equal to the expected spectrogram size.
+        hidden_channels (int):
+            number of channels in all the model layers.
+        hidden_channels_ffn (int):
+            number of channels in transformer's conv layers.
+        hidden_channels_dp (int):
+            number of channels in duration predictor network.
+        num_heads (int):
+            number of attention heads in transformer networks.
+        num_transformer_layers (int):
+            number of layers in encoder and decoder transformer blocks.
+        dropout_p (int):
+            dropout rate in transformer layers.
+        length_scale (int, optional):
+            coefficient to set the speech speed. <1 slower, >1 faster. Defaults to 1.
+        num_speakers (int, optional):
+            number of speakers for multi-speaker training. Defaults to 0.
+        external_c (bool, optional):
+            enable external speaker embeddings. Defaults to False.
+        c_in_channels (int, optional):
+            number of channels in speaker embedding vectors. Defaults to 0.
+    """
+
+    num_chars: int = None
+    out_channels: int = 80
+    hidden_channels: int = 256
+    hidden_channels_dp: int = 256
+    encoder_type: str = "fftransformer"
+    encoder_params: dict = field(
+        default_factory=lambda: {"hidden_channels_ffn": 1024, "num_heads": 2, "num_layers": 6, "dropout_p": 0.1}
+    )
+    decoder_type: str = "fftransformer"
+    decoder_params: dict = field(
+        default_factory=lambda: {"hidden_channels_ffn": 1024, "num_heads": 2, "num_layers": 6, "dropout_p": 0.1}
+    )
+    length_scale: float = 1.0
+    num_speakers: int = 0
+    use_speaker_embedding: bool = False
+    use_d_vector_file: bool = False
+    d_vector_dim: int = 0
 
 
 @dataclass

TTS/tts/configs/align_tts_config.py

@@ -1,12 +1,18 @@
 import os
 from dataclasses import dataclass, field
 
+from coqpit import Coqpit
 from trainer.io import get_user_data_dir
 
 from TTS.tts.configs.shared_configs import BaseTTSConfig
 from TTS.tts.layers.bark.model import GPTConfig
 from TTS.tts.layers.bark.model_fine import FineGPTConfig
-from TTS.tts.models.bark import BarkAudioConfig
+
+
+@dataclass
+class BarkAudioConfig(Coqpit):
+    sample_rate: int = 24000
+    output_sample_rate: int = 24000
 
 
 @dataclass

TTS/tts/configs/bark_config.py

@@ -1,7 +1,78 @@
 from dataclasses import dataclass, field
 
+from coqpit import Coqpit
+
+from TTS.config.shared_configs import BaseAudioConfig, ModelArgs
 from TTS.tts.configs.shared_configs import BaseTTSConfig
-from TTS.tts.models.delightful_tts import DelightfulTtsArgs, DelightfulTtsAudioConfig, VocoderConfig
+
+
+@dataclass
+class VocoderConfig(Coqpit):
+    resblock_type_decoder: str = "1"
+    resblock_kernel_sizes_decoder: list[int] = field(default_factory=lambda: [3, 7, 11])
+    resblock_dilation_sizes_decoder: list[list[int]] = field(default_factory=lambda: [[1, 3, 5], [1, 3, 5], [1, 3, 5]])
+    upsample_rates_decoder: list[int] = field(default_factory=lambda: [8, 8, 2, 2])
+    upsample_initial_channel_decoder: int = 512
+    upsample_kernel_sizes_decoder: list[int] = field(default_factory=lambda: [16, 16, 4, 4])
+    use_spectral_norm_discriminator: bool = False
+    upsampling_rates_discriminator: list[int] = field(default_factory=lambda: [4, 4, 4, 4])
+    periods_discriminator: list[int] = field(default_factory=lambda: [2, 3, 5, 7, 11])
+    pretrained_model_path: str | None = None
+
+
+@dataclass
+class DelightfulTtsAudioConfig(BaseAudioConfig):
+    mel_fmax: float = 8000
+    num_mels: int = 100
+
+
+@dataclass
+class DelightfulTtsArgs(ModelArgs):
+    num_chars: int = 100
+    spec_segment_size: int = 32
+    n_hidden_conformer_encoder: int = 512
+    n_layers_conformer_encoder: int = 6
+    n_heads_conformer_encoder: int = 8
+    dropout_conformer_encoder: float = 0.1
+    kernel_size_conv_mod_conformer_encoder: int = 7
+    kernel_size_depthwise_conformer_encoder: int = 7
+    lrelu_slope: float = 0.3
+    n_hidden_conformer_decoder: int = 512
+    n_layers_conformer_decoder: int = 6
+    n_heads_conformer_decoder: int = 8
+    dropout_conformer_decoder: float = 0.1
+    kernel_size_conv_mod_conformer_decoder: int = 11
+    kernel_size_depthwise_conformer_decoder: int = 11
+    bottleneck_size_p_reference_encoder: int = 4
+    bottleneck_size_u_reference_encoder: int = 512
+    ref_enc_filters_reference_encoder = [32, 32, 64, 64, 128, 128]
+    ref_enc_size_reference_encoder: int = 3
+    ref_enc_strides_reference_encoder = [1, 2, 1, 2, 1]
+    ref_enc_pad_reference_encoder = [1, 1]
+    ref_enc_gru_size_reference_encoder: int = 32
+    ref_attention_dropout_reference_encoder: float = 0.2
+    token_num_reference_encoder: int = 32
+    predictor_kernel_size_reference_encoder: int = 5
+    n_hidden_variance_adaptor: int = 512
+    kernel_size_variance_adaptor: int = 5
+    dropout_variance_adaptor: float = 0.5
+    n_bins_variance_adaptor: int = 256
+    emb_kernel_size_variance_adaptor: int = 3
+    use_speaker_embedding: bool = False
+    num_speakers: int = 0
+    speakers_file: str = None
+    d_vector_file: str = None
+    speaker_embedding_channels: int = 384
+    use_d_vector_file: bool = False
+    d_vector_dim: int = 0
+    freeze_vocoder: bool = False
+    freeze_text_encoder: bool = False
+    freeze_duration_predictor: bool = False
+    freeze_pitch_predictor: bool = False
+    freeze_energy_predictor: bool = False
+    freeze_basis_vectors_predictor: bool = False
+    freeze_decoder: bool = False
+    length_scale: float = 1.0
 
 
 @dataclass

TTS/tts/configs/delightful_tts_config.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass, field
 
+from TTS.tts.configs.forward_tts_config import ForwardTTSArgs
 from TTS.tts.configs.shared_configs import BaseTTSConfig
-from TTS.tts.models.forward_tts import ForwardTTSArgs
 
 
 @dataclass