From 0cc5631027307aede7effe2e283010243dd543e2 Mon Sep 17 00:00:00 2001 From: yunusemretom Date: Fri, 14 Aug 2026 18:52:36 +0300 Subject: [PATCH 1/4] Fix two CUDA-only crashes in multi-speaker paths Both only trigger when a GPU is present, which is why they survived CI. `_forward_encoder` cast speaker ids with `.type(torch.LongTensor)`, which is the CPU long tensor type, so the ids were moved away from the CUDA `emb_g` embedding and training multi-speaker FastPitch/FastSpeech2/DelightfulTTS died on a device mismatch. Use `.long()`, which keeps the device. DelightfulTTS's local `id_to_torch`/`embedding_to_torch` called `.cuda()` on the `None` they had just decided not to convert. `numpy_to_torch` next to them and the copies in `TTS/tts/utils/synthesis.py` already return early instead; do the same here. Co-Authored-By: Claude Opus 5 --- TTS/tts/models/delightful_tts.py | 16 +++++++++------- TTS/tts/models/forward_tts.py | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/TTS/tts/models/delightful_tts.py b/TTS/tts/models/delightful_tts.py index b1cf886bea..8dad85304a 100644 --- a/TTS/tts/models/delightful_tts.py +++ b/TTS/tts/models/delightful_tts.py @@ -38,19 +38,21 @@ def id_to_torch(aux_id, cuda=False): - if aux_id is not None: - aux_id = np.asarray(aux_id) - aux_id = torch.from_numpy(aux_id) + if aux_id is None: + return None + aux_id = np.asarray(aux_id) + aux_id = torch.from_numpy(aux_id) if cuda: return aux_id.cuda() return aux_id def embedding_to_torch(d_vector, cuda=False): - if d_vector is not None: - d_vector = np.asarray(d_vector) - d_vector = torch.from_numpy(d_vector).float() - d_vector = d_vector.squeeze().unsqueeze(0) + if d_vector is None: + return None + d_vector = np.asarray(d_vector) + d_vector = torch.from_numpy(d_vector).float() + d_vector = d_vector.squeeze().unsqueeze(0) if cuda: return d_vector.cuda() return d_vector diff --git a/TTS/tts/models/forward_tts.py b/TTS/tts/models/forward_tts.py index b6e9ac8a14..ad8c1d09f6 100644 --- a/TTS/tts/models/forward_tts.py +++ b/TTS/tts/models/forward_tts.py @@ -397,7 +397,7 @@ def _forward_encoder( - g: :math:`(B, C)` """ if hasattr(self, "emb_g"): - g = g.type(torch.LongTensor) + g = g.long() # `.type(torch.LongTensor)` would move the ids to CPU, away from `emb_g` g = self.emb_g(g) # [B, C, 1] if g is not None: g = g.unsqueeze(-1) From 94a3a74f951292b170c681a26b848aee008432da Mon Sep 17 00:00:00 2001 From: yunusemretom Date: Fri, 14 Aug 2026 18:52:49 +0300 Subject: [PATCH 2/4] Speed up XTTS inference with float16 and batched sentence decoding Profiling XTTS v2 on an RTX 4050 Laptop puts 93% of inference in the autoregressive GPT loop. That loop is ~100% GPU-busy with no launch-overhead gap, and its step time barely moves with batch size (7.0 ms at B=1, 9.5 ms at B=4), so it is bound by reading the GPT's 379M weights rather than by arithmetic. Two changes follow from that; swapping GPT2's eager attention for SDPA was also measured and gave exactly 1.00x, so it is not included. Half precision halves the traffic that sets the pace. Three things blocked it: - `null_position_embeddings` returned hardcoded float32 zeros, and GPT2 adds those to the input embeddings, promoting every activation back to float32. It is now a module carrying a non-persistent buffer, so its dtype follows `.half()` / `.to()` on the parent. - `GroupNorm32` cast its input to float32 but not its own weights, so half precision hit it as a mixed-dtype error. It now casts both, which is what normalizing in float32 was meant to mean. - Conditioning latents and vocoder inputs did not follow the model's dtype. `use_half_precision()` leaves the conditioning encoder in float32, where it overflows to NaN in half precision and costs nothing to keep, and the vocoder likewise. `load_checkpoint(half=True)` applies it at load time. Batching decodes several sentences in one pass, so the weights are read once for the whole batch. `inference(batch_size=N)` groups sentences by length to waste as few decoding steps as possible; grouping does not affect results, so the original order is restored afterwards. Verified by decoding greedily both ways: the token sequences are identical. Paragraph-length text, RTX 4050 Laptop: RTF 0.259 -> 0.178 with half precision, -> 0.092 with batch_size=4, and peak VRAM drops from 2.21 to 1.49 GB. Co-Authored-By: Claude Opus 5 --- TTS/tts/layers/xtts/gpt.py | 109 +++++++++++++++- TTS/tts/layers/xtts/latent_encoder.py | 15 ++- TTS/tts/models/xtts.py | 130 +++++++++++++++--- docs/source/models/xtts.md | 33 +++++ scripts/bench_xtts.py | 181 ++++++++++++++++++++++++++ scripts/bench_xtts_paragraph.py | 109 ++++++++++++++++ 6 files changed, 552 insertions(+), 25 deletions(-) create mode 100644 scripts/bench_xtts.py create mode 100644 scripts/bench_xtts_paragraph.py diff --git a/TTS/tts/layers/xtts/gpt.py b/TTS/tts/layers/xtts/gpt.py index e7b186b858..8bf54d05f1 100644 --- a/TTS/tts/layers/xtts/gpt.py +++ b/TTS/tts/layers/xtts/gpt.py @@ -1,6 +1,5 @@ # ported from: https://github.com/neonbjb/tortoise-tts -import functools import math import random @@ -14,8 +13,27 @@ from TTS.tts.layers.xtts.perceiver_encoder import PerceiverResampler -def null_position_embeddings(range, dim): - return torch.zeros((range.shape[0], range.shape[1], dim), device=range.device) +class NullPositionEmbeddings(nn.Module): + """Stand-in for an unused position embedding table. + + Kept as a module rather than a plain function so that its output dtype follows + ``.half()`` / ``.to()`` on the parent model. Returning hardcoded float32 zeros + would promote the activations back to float32 when added to the input + embeddings, which silently defeats half precision inference. + """ + + def __init__(self, dim): + super().__init__() + self.dim = dim + # non-persistent so it stays out of the checkpoint's state_dict + self.register_buffer("_dtype_probe", torch.zeros(1), persistent=False) + + def forward(self, range): # pylint: disable=redefined-builtin + return torch.zeros( + (range.shape[0], range.shape[1], self.dim), + device=range.device, + dtype=self._dtype_probe.dtype, + ) class LearnedPositionEmbeddings(nn.Module): @@ -67,19 +85,19 @@ def build_hf_gpt_transformer( gpt = GPT2Model(gpt_config) # Override the built in positional embeddings del gpt.wpe - gpt.wpe = functools.partial(null_position_embeddings, dim=model_dim) + gpt.wpe = NullPositionEmbeddings(model_dim) # Built-in token embeddings are unused. del gpt.wte mel_pos_emb = ( LearnedPositionEmbeddings(max_mel_seq_len, model_dim) if max_mel_seq_len != -1 - else functools.partial(null_position_embeddings, dim=model_dim) + else NullPositionEmbeddings(model_dim) ) text_pos_emb = ( LearnedPositionEmbeddings(max_text_seq_len, model_dim) if max_mel_seq_len != -1 - else functools.partial(null_position_embeddings, dim=model_dim) + else NullPositionEmbeddings(model_dim) ) # gpt = torch.compile(gpt, mode="reduce-overhead", fullgraph=True) return gpt, mel_pos_emb, text_pos_emb, None, None @@ -359,6 +377,9 @@ def get_style_emb(self, cond_input, return_latent=False): if not return_latent: if cond_input.ndim == 4: cond_input = cond_input.squeeze(1) + # the reference mel is always computed in float32; follow the encoder's dtype so + # that the model can be run in half precision + cond_input = cond_input.to(next(self.conditioning_encoder.parameters()).dtype) conds = self.conditioning_encoder(cond_input) # (b, d, s) if self.use_perceiver_resampler: conds = self.conditioning_perceiver(conds.permute(0, 2, 1)).transpose(1, 2) # (b, d, 32) @@ -502,6 +523,7 @@ def forward( # Compute speech conditioning input if cond_latents is None: cond_latents = self.get_style_emb(cond_mels).transpose(1, 2) + cond_latents = cond_latents.to(text_emb.dtype) # Get logits sub = -5 # don't ask me why 😄 @@ -566,7 +588,9 @@ def compute_embeddings( text_inputs = F.pad(text_inputs, (0, 1), value=self.stop_text_token) text_inputs = F.pad(text_inputs, (1, 0), value=self.start_text_token) emb = self.text_embedding(text_inputs) + self.text_pos_embedding(text_inputs) - emb = torch.cat([cond_latents, emb], dim=1) + # follow the model's dtype: float32 latents would promote the whole prefix back to + # float32 and silently undo half precision inference + emb = torch.cat([cond_latents.to(emb.dtype), emb], dim=1) self.gpt_inference.store_prefix_emb(emb) gpt_inputs = torch.full( ( @@ -599,6 +623,77 @@ def generate( return gen.sequences[:, gpt_inputs.shape[1] :], gen return gen[:, gpt_inputs.shape[1] :] + def compute_embeddings_batch(self, cond_latents, text_inputs, text_lengths): + """Batched counterpart of :meth:`compute_embeddings`. + + Unlike the single-sequence path, the start/stop text tokens are expected to be + part of ``text_inputs`` already, so that every sentence keeps the same positional + embeddings it would get on its own; the padding sits after the stop token and is + hidden with the returned attention mask. + + Args: + cond_latents: conditioning latents, ``(B, cond_len, dim)``. + text_inputs: right-padded ``[start, tokens..., stop]`` sequences, ``(B, T)``. + text_lengths: real length of each row of ``text_inputs``, ``(B,)``. + + Returns: + Tuple of the placeholder gpt inputs ``(B, cond_len + T + 1)`` and the matching + attention mask. + """ + emb = self.text_embedding(text_inputs) + self.text_pos_embedding(text_inputs) + emb = torch.cat([cond_latents.to(emb.dtype), emb], dim=1) + self.gpt_inference.store_prefix_emb(emb) + + gpt_inputs = torch.full( + (emb.shape[0], emb.shape[1] + 1), # +1 for the start_audio_token + fill_value=1, + dtype=torch.long, + device=text_inputs.device, + ) + gpt_inputs[:, -1] = self.start_audio_token + + cond_len = cond_latents.shape[1] + positions = torch.arange(text_inputs.shape[1], device=text_inputs.device) + attention_mask = torch.ones_like(gpt_inputs) + attention_mask[:, cond_len : cond_len + text_inputs.shape[1]] = ( + positions[None, :] < text_lengths[:, None] + ).long() + return gpt_inputs, attention_mask + + def generate_batch( + self, + cond_latents, + text_inputs, + text_lengths, + **hf_generate_kwargs, + ): + """Generate mel codes for several sentences at once. + + The autoregressive loop is bound by reading the model weights, so decoding a batch + costs barely more than decoding a single sequence. Returns one code sequence per + row, each already trimmed at its own stop token. + """ + gpt_inputs, attention_mask = self.compute_embeddings_batch(cond_latents, text_inputs, text_lengths) + gen = self.gpt_inference.generate( + gpt_inputs, + attention_mask=attention_mask, + bos_token_id=self.start_audio_token, + pad_token_id=self.stop_audio_token, + eos_token_id=self.stop_audio_token, + max_length=self.max_gen_mel_tokens + gpt_inputs.shape[-1], + **hf_generate_kwargs, + ) + codes = gen[:, gpt_inputs.shape[1] :] + + # Sequences that stopped early are padded with stop tokens up to the longest one; + # cut each back to its own stop token, which is what single-sentence decoding returns. + trimmed = [] + for row in codes: + stops = (row == self.stop_audio_token).nonzero() + end = stops[0].item() + 1 if len(stops) > 0 else row.shape[0] + trimmed.append(row[:end].unsqueeze(0)) + return trimmed + def get_generator(self, fake_inputs, **hf_generate_kwargs): return self.gpt_inference.generate_stream( fake_inputs, diff --git a/TTS/tts/layers/xtts/latent_encoder.py b/TTS/tts/layers/xtts/latent_encoder.py index f9d62a36f1..f0cee8bda4 100644 --- a/TTS/tts/layers/xtts/latent_encoder.py +++ b/TTS/tts/layers/xtts/latent_encoder.py @@ -8,8 +8,21 @@ class GroupNorm32(nn.GroupNorm): + """GroupNorm that always normalizes in float32, whatever dtype the model runs in. + + The weights have to be cast along with the input: `super().forward(x.float())` would + otherwise feed float32 activations to half precision parameters. + """ + def forward(self, x): - return super().forward(x.float()).type(x.dtype) + out = F.group_norm( + x.float(), + self.num_groups, + self.weight.float() if self.weight is not None else None, + self.bias.float() if self.bias is not None else None, + self.eps, + ) + return out.type(x.dtype) def conv_nd(dims, *args, **kwargs): diff --git a/TTS/tts/models/xtts.py b/TTS/tts/models/xtts.py index 8e9d6bd382..583909703d 100644 --- a/TTS/tts/models/xtts.py +++ b/TTS/tts/models/xtts.py @@ -254,6 +254,27 @@ def init_models(self): def device(self): return next(self.parameters()).device + def use_half_precision(self): + """Run the autoregressive GPT in float16. + + Decoding is paced by reading the GPT weights, so halving them is the cheapest real + speedup available. The speaker conditioning encoder is left in float32: it overflows + in half precision and produces NaN latents, and it runs once per call, so it costs + nothing to keep. The vocoder stays in float32 for the same reason. + + Inference only -- the model must already be in eval mode. + """ + self.gpt.half() + self.gpt.conditioning_encoder.float() + if self.args.gpt_use_perceiver_resampler: + self.gpt.conditioning_perceiver.float() + return self + + @property + def hifigan_decoder_dtype(self): + """Dtype of the vocoder, which may differ from the GPT's when running it in half precision.""" + return next(self.hifigan_decoder.parameters()).dtype + @torch.inference_mode() def get_gpt_cond_latents(self, audio, sr, length: int = 30, chunk_length: int = 6): """Compute the conditioning latents for the GPT model from the given audio. @@ -499,6 +520,65 @@ def full_inference( **hf_generate_kwargs, ) + def _generate_gpt_codes(self, text_tokens_list, gpt_cond_latent, batch_size, **gpt_generate_kwargs): + """Run the autoregressive GPT over every sentence, batching `batch_size` of them at a time. + + The decoding loop is bound by reading the GPT weights rather than by arithmetic, so + sentences decoded together cost little more than a single one. Batching is skipped + when `gpt_batch_size` asks for several samples per sentence, since the two ways of + using the batch dimension cannot be combined. + """ + if batch_size < 1: + raise ValueError(f" ❗ batch_size must be >= 1, got {batch_size}") + if batch_size == 1 or self.gpt_batch_size != 1: + return [ + self.gpt.generate( + cond_latents=gpt_cond_latent, + text_inputs=text_tokens, + input_tokens=None, + num_return_sequences=self.gpt_batch_size, + **gpt_generate_kwargs, + ) + for text_tokens in text_tokens_list + ] + + start_token, stop_token = self.gpt.start_text_token, self.gpt.stop_text_token + # A batch runs until its longest sentence is done, so group sentences of similar + # length together to waste as few decoding steps as possible. Results do not depend + # on how sentences are grouped, so the original order is simply restored afterwards. + order = sorted(range(len(text_tokens_list)), key=lambda i: text_tokens_list[i].shape[-1]) + gpt_codes_by_index = {} + for i in range(0, len(order), batch_size): + indices = order[i : i + batch_size] + chunk = [text_tokens_list[j] for j in indices] + if len(chunk) == 1: + gpt_codes_by_index[indices[0]] = self.gpt.generate( + cond_latents=gpt_cond_latent, + text_inputs=chunk[0], + input_tokens=None, + num_return_sequences=self.gpt_batch_size, + **gpt_generate_kwargs, + ) + continue + + # add the start/stop tokens here so that padding lands after the stop token and + # every sentence keeps the positional embeddings it would get on its own + rows = [F.pad(F.pad(tokens, (0, 1), value=stop_token), (1, 0), value=start_token) for tokens in chunk] + text_lengths = torch.tensor([row.shape[-1] for row in rows], device=self.device) + max_len = int(text_lengths.max()) + padded = torch.cat([F.pad(row, (0, max_len - row.shape[-1]), value=stop_token) for row in rows], dim=0) + + codes = self.gpt.generate_batch( + cond_latents=gpt_cond_latent.expand(len(chunk), -1, -1), + text_inputs=padded, + text_lengths=text_lengths, + input_tokens=None, + num_return_sequences=1, + **gpt_generate_kwargs, + ) + gpt_codes_by_index.update(zip(indices, codes)) + return [gpt_codes_by_index[i] for i in range(len(text_tokens_list))] + @torch.inference_mode() def inference( self, @@ -516,6 +596,7 @@ def inference( num_beams=1, speed=1.0, enable_text_splitting=False, + batch_size=1, **hf_generate_kwargs, ): language = language.split("-")[0] # remove the country code @@ -527,8 +608,7 @@ def inference( else: text = [text] - wavs = [] - gpt_latents_list = [] + text_tokens_list = [] for sent in text: sent = sent.strip().lower() text_tokens = torch.IntTensor(self.tokenizer.encode(sent, lang=language)).unsqueeze(0).to(self.device) @@ -536,23 +616,28 @@ def inference( assert ( text_tokens.shape[-1] < self.args.gpt_max_text_tokens ), " ❗ XTTS can only generate text with a maximum of 400 tokens." + text_tokens_list.append(text_tokens) + with torch.no_grad(): + gpt_codes_list = self._generate_gpt_codes( + text_tokens_list, + gpt_cond_latent, + batch_size, + do_sample=do_sample, + top_p=top_p, + top_k=top_k, + temperature=temperature, + num_beams=num_beams, + length_penalty=length_penalty, + repetition_penalty=repetition_penalty, + output_attentions=False, + **hf_generate_kwargs, + ) + + wavs = [] + gpt_latents_list = [] + for text_tokens, gpt_codes in zip(text_tokens_list, gpt_codes_list): with torch.no_grad(): - gpt_codes = self.gpt.generate( - cond_latents=gpt_cond_latent, - text_inputs=text_tokens, - input_tokens=None, - do_sample=do_sample, - top_p=top_p, - top_k=top_k, - temperature=temperature, - num_return_sequences=self.gpt_batch_size, - num_beams=num_beams, - length_penalty=length_penalty, - repetition_penalty=repetition_penalty, - output_attentions=False, - **hf_generate_kwargs, - ) expected_output_len = torch.tensor( [gpt_codes.shape[-1] * self.gpt.code_stride_len], device=text_tokens.device ) @@ -574,6 +659,7 @@ def inference( ).transpose(1, 2) gpt_latents_list.append(gpt_latents.cpu()) + gpt_latents = gpt_latents.to(self.hifigan_decoder_dtype) wavs.append(self.hifigan_decoder(gpt_latents, g=speaker_embedding).cpu().squeeze()) return { @@ -684,6 +770,7 @@ def inference_stream( gpt_latents = F.interpolate( gpt_latents.transpose(1, 2), scale_factor=length_scale, mode="linear" ).transpose(1, 2) + gpt_latents = gpt_latents.to(self.hifigan_decoder_dtype) wav_gen = self.hifigan_decoder(gpt_latents, g=speaker_embedding.to(self.device)) wav_chunk, wav_gen_prev, wav_overlap = self.handle_chunks( wav_gen.squeeze(), wav_gen_prev, wav_overlap, overlap_wav_len @@ -738,6 +825,7 @@ def load_checkpoint( strict=True, use_deepspeed=False, speaker_file_path=None, + half=False, ): """ Loads a checkpoint from disk and initializes the model's state and tokenizer. @@ -749,10 +837,16 @@ def load_checkpoint( vocab_path (str, optional): The path to the vocabulary file. Defaults to None. eval (bool, optional): Whether to set the model to evaluation mode. Defaults to True. strict (bool, optional): Whether to strictly enforce that the keys in the checkpoint match the keys in the model. Defaults to True. + half (bool, optional): Run the autoregressive GPT in float16. The decoding loop is bound by + reading the GPT weights, so this roughly halves the traffic that sets its pace. The + vocoder is left in float32, where it is cheap and better conditioned. Inference only; + requires `eval=True`. Defaults to False. Returns: None """ + if half and not eval: + raise ValueError(" ❗ half precision is only supported for inference, pass eval=True") model_path = checkpoint_path or os.path.join(checkpoint_dir, "model.pth") vocab_path = vocab_path or os.path.join(checkpoint_dir, "vocab.json") @@ -784,6 +878,8 @@ def load_checkpoint( self.hifigan_decoder.eval() self.gpt.init_gpt_for_inference(kv_cache=self.args.kv_cache, use_deepspeed=use_deepspeed) self.gpt.eval() + if half: + self.use_half_precision() def train_step(self): raise NotImplementedError( diff --git a/docs/source/models/xtts.md b/docs/source/models/xtts.md index b979d04f6e..0511951571 100644 --- a/docs/source/models/xtts.md +++ b/docs/source/models/xtts.md @@ -186,6 +186,39 @@ pip install deepspeed==0.10.3 - `top_p`: Lower values mean the decoder produces more "likely" (aka boring) outputs. Defaults to 0.8. - `speed`: The speed rate of the generated audio. Defaults to 1.0. (can produce artifacts if far from 1.0) - `enable_text_splitting`: Whether to split the text into sentences and generate audio for each sentence. It allows you to have infinite input length but might loose important context between sentences. Defaults to True. +- `batch_size`: How many of those sentences to decode at once. Defaults to 1 (one after another). See "Faster inference" below. + +##### Faster inference + +The autoregressive GPT produces one token at a time and reads all of its weights for each of +them, so inference is paced by memory bandwidth rather than by arithmetic. Two settings follow +from that, and they combine: + +- `load_checkpoint(..., half=True)` runs the GPT in float16, halving the traffic. The speaker + conditioning encoder and the vocoder stay in float32, where half precision is unstable and + buys nothing. On an existing model, `model.use_half_precision()` does the same thing. +- `batch_size > 1` (with `enable_text_splitting=True`) decodes several sentences in the same + pass. Since the weights are read once for the whole batch, four sentences cost little more + than one. Sentences are grouped by length and the result is identical to decoding them one + by one. + +On an RTX 4050 Laptop, reading a paragraph aloud goes from a real-time factor of 0.26 to 0.09 +with `half=True` and `batch_size=4`. + +Note that the 🐸TTS API splits text into sentences before it reaches the model, so batching +only kicks in when you let the model do the splitting instead: + +```python +tts.tts_to_file( + text=long_text, + speaker_wav="reference.wav", + language="en", + file_path="output.wav", + split_sentences=False, # hand the whole text to XTTS + enable_text_splitting=True, + batch_size=4, +) +``` ##### Inference diff --git a/scripts/bench_xtts.py b/scripts/bench_xtts.py new file mode 100644 index 0000000000..ca48309ea8 --- /dev/null +++ b/scripts/bench_xtts.py @@ -0,0 +1,181 @@ +"""Benchmark XTTS v2 inference speed under different runtime settings. + +Reports a per-phase breakdown (GPT autoregressive generation, the latent +forward pass, HiFiGAN decoding) so it is clear which part dominates. + +Usage: + python scripts/bench_xtts.py --configs fp32 fp16 --runs 3 +""" + +import argparse +import os +import time + +import torch +import torch.nn.functional as F + +from TTS.tts.configs.xtts_config import XttsConfig +from TTS.tts.models.xtts import Xtts +from TTS.utils.manage import ModelManager + +MODEL_NAME = "tts_models/multilingual/multi-dataset/xtts_v2" + +TEXT = ( + "The quick brown fox jumps over the lazy dog while the engineer measures " + "how many seconds of audio this model can synthesize in one second of compute." +) + + +def download_model(): + return ModelManager().download_model(MODEL_NAME)[0] + + +def load_model(model_path, device, use_deepspeed=False): + config = XttsConfig() + config.load_json(os.path.join(model_path, "config.json")) + model = Xtts.init_from_config(config) + model.load_checkpoint(config, checkpoint_dir=model_path, use_deepspeed=use_deepspeed, eval=True) + model.to(device) + return model + + +def sync(): + if torch.cuda.is_available(): + torch.cuda.synchronize() + + +def timeit(fn, *args, **kwargs): + sync() + t0 = time.perf_counter() + out = fn(*args, **kwargs) + sync() + return out, time.perf_counter() - t0 + + +def inference_profiled(model, text, language, gpt_cond_latent, speaker_embedding, half=False): + """Mirror of Xtts.inference, instrumented per phase.""" + dtype = torch.float16 if half else torch.float32 + gpt_cond_latent = gpt_cond_latent.to(model.device).to(dtype) + speaker_embedding = speaker_embedding.to(model.device) + + sent = text.strip().lower() + text_tokens = torch.IntTensor(model.tokenizer.encode(sent, lang=language)).unsqueeze(0).to(model.device) + + with torch.no_grad(): + gpt_codes, t_gen = timeit( + model.gpt.generate, + cond_latents=gpt_cond_latent, + text_inputs=text_tokens, + input_tokens=None, + do_sample=True, + top_p=0.85, + top_k=50, + temperature=0.75, + num_return_sequences=model.gpt_batch_size, + num_beams=1, + length_penalty=1.0, + repetition_penalty=10.0, + output_attentions=False, + ) + + expected_output_len = torch.tensor( + [gpt_codes.shape[-1] * model.gpt.code_stride_len], device=text_tokens.device + ) + text_len = torch.tensor([text_tokens.shape[-1]], device=model.device) + + gpt_latents, t_latent = timeit( + model.gpt, + text_tokens, + text_len, + gpt_codes, + expected_output_len, + cond_latents=gpt_cond_latent, + return_attentions=False, + return_latent=True, + ) + + wav, t_vocoder = timeit(model.hifigan_decoder, gpt_latents.float(), g=speaker_embedding) + + return { + "wav": wav.cpu().squeeze().numpy(), + "tokens": gpt_codes.shape[-1], + "gen": t_gen, + "latent": t_latent, + "vocoder": t_vocoder, + } + + +def run_config(name, model_path, device, speaker_wav, runs, **load_kwargs): + print(f"\n=== config: {name} ===") + torch.manual_seed(0) + half = load_kwargs.pop("half", False) + model, load_s = timeit(load_model, model_path, device, **load_kwargs) + print(f"load: {load_s:.2f}s") + + # conditioning runs in fp32: the reference mel is computed in fp32 either way + (gpt_cond_latent, speaker_embedding), cond_s = timeit(model.get_conditioning_latents, audio_path=[speaker_wav]) + print(f"conditioning: {cond_s:.2f}s") + + if half: + model.use_half_precision() + + sr = model.config.audio.output_sample_rate + rows = [] + for i in range(runs): + torch.manual_seed(1234 + i) + out, total = timeit( + inference_profiled, model, TEXT, "en", gpt_cond_latent, speaker_embedding, half=half + ) + audio_s = len(out["wav"]) / sr + rows.append((total, audio_s, out)) + print( + f"run {i + 1}: total {total:.2f}s | gen {out['gen']:.2f}s ({out['tokens']} tok) " + f"| latent {out['latent']:.2f}s | vocoder {out['vocoder']:.2f}s " + f"| audio {audio_s:.2f}s | RTF {total / audio_s:.3f}" + ) + + steady = rows[1:] or rows # drop warmup + mean_rtf = sum(t / a for t, a, _ in steady) / len(steady) + share = lambda k: sum(o[k] for _, _, o in steady) / sum(t for t, _, _ in steady) * 100 + print(f"mean RTF: {mean_rtf:.3f} | time split: gen {share('gen'):.0f}% " + f"latent {share('latent'):.0f}% vocoder {share('vocoder'):.0f}%") + + if torch.cuda.is_available(): + print(f"peak VRAM: {torch.cuda.max_memory_allocated() / 1e9:.2f} GB") + torch.cuda.reset_peak_memory_stats() + + del model + if torch.cuda.is_available(): + torch.cuda.empty_cache() + return mean_rtf + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--configs", nargs="+", default=["fp32", "fp16"], choices=["fp32", "fp16"]) + parser.add_argument("--runs", type=int, default=3) + parser.add_argument("--speaker-wav", default="tests/data/ljspeech/wavs/LJ001-0001.wav") + parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu") + args = parser.parse_args() + + model_path = download_model() + print(f"model: {model_path}") + print(f"device: {args.device}") + if torch.cuda.is_available(): + print(f"gpu: {torch.cuda.get_device_name(0)}") + + load_kwargs = {"fp32": {}, "fp16": {"half": True}} + + results = {} + for cfg in args.configs: + results[cfg] = run_config(cfg, model_path, args.device, args.speaker_wav, args.runs, **load_kwargs[cfg]) + + print("\n=== summary (mean RTF, lower is better) ===") + base = results.get("fp32") + for cfg, rtf in results.items(): + speedup = f" ({base / rtf:.2f}x vs fp32)" if base else "" + print(f"{cfg:10s} {rtf:.3f}{speedup}") + + +if __name__ == "__main__": + main() diff --git a/scripts/bench_xtts_paragraph.py b/scripts/bench_xtts_paragraph.py new file mode 100644 index 0000000000..7acd074548 --- /dev/null +++ b/scripts/bench_xtts_paragraph.py @@ -0,0 +1,109 @@ +"""End-to-end XTTS v2 benchmark on paragraph-length text. + +Compares the stock settings against half precision and batched sentence decoding, +which is the case that matters for reading a document out loud. + +Usage: + python scripts/bench_xtts_paragraph.py --runs 2 +""" + +import argparse +import os +import time + +import torch + +from TTS.tts.configs.xtts_config import XttsConfig +from TTS.tts.models.xtts import Xtts +from TTS.utils.manage import ModelManager + +MODEL_NAME = "tts_models/multilingual/multi-dataset/xtts_v2" + +PARAGRAPH = ( + "Speech synthesis has changed a great deal over the last few years. " + "Models that once needed a studio recording session for every new voice can now copy " + "a speaker from a few seconds of audio. " + "That shift moved the hard problem from data collection to inference cost. " + "A model is only useful in a product if it can keep up with the person listening to it. " + "On a laptop graphics card, the autoregressive decoder is the part that decides whether " + "the system feels responsive or sluggish. " + "Measuring where the time actually goes is the first step to making it faster." +) + + +def load_model(model_path, device, half): + config = XttsConfig() + config.load_json(os.path.join(model_path, "config.json")) + model = Xtts.init_from_config(config) + model.load_checkpoint(config, checkpoint_dir=model_path, eval=True) + model.to(device) + return model + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--runs", type=int, default=2) + parser.add_argument("--speaker-wav", default="tests/data/ljspeech/wavs/LJ001-0001.wav") + args = parser.parse_args() + + device = "cuda" if torch.cuda.is_available() else "cpu" + model_path = ModelManager().download_model(MODEL_NAME)[0] + print(f"device: {device}") + if torch.cuda.is_available(): + print(f"gpu: {torch.cuda.get_device_name(0)}") + + model = load_model(model_path, device, half=False) + gpt_cond_latent, speaker_embedding = model.get_conditioning_latents(audio_path=[args.speaker_wav]) + sr = model.config.audio.output_sample_rate + + configs = [ + ("fp32, sequential", False, 1), + ("fp16, sequential", True, 1), + ("fp16, batch=4", True, 4), + ("fp16, batch=8", True, 8), + ] + + results = {} + halved = False + for name, half, batch_size in configs: + if half and not halved: + model.use_half_precision() + halved = True + + times = [] + for i in range(args.runs): + torch.manual_seed(1234 + i) + if torch.cuda.is_available(): + torch.cuda.synchronize() + t0 = time.perf_counter() + out = model.inference( + PARAGRAPH, + "en", + gpt_cond_latent, + speaker_embedding, + enable_text_splitting=True, + batch_size=batch_size, + ) + if torch.cuda.is_available(): + torch.cuda.synchronize() + elapsed = time.perf_counter() - t0 + times.append((elapsed, len(out["wav"]) / sr)) + + steady = times[1:] or times + rtf = sum(t / a for t, a in steady) / len(steady) + wall = sum(t for t, _ in steady) / len(steady) + audio = sum(a for _, a in steady) / len(steady) + results[name] = rtf + print(f"{name:20s} {wall:5.2f}s for {audio:5.2f}s audio | RTF {rtf:.3f}") + if torch.cuda.is_available(): + print(f"{'':20s} peak VRAM {torch.cuda.max_memory_allocated() / 1e9:.2f} GB") + torch.cuda.reset_peak_memory_stats() + + print("\n=== summary ===") + base = results["fp32, sequential"] + for name, rtf in results.items(): + print(f"{name:20s} RTF {rtf:.3f} ({base / rtf:.2f}x vs stock)") + + +if __name__ == "__main__": + main() From 8e5c32348e2b070d9b58204688ad31a28e750b35 Mon Sep 17 00:00:00 2001 From: yunusemretom Date: Fri, 14 Aug 2026 18:58:21 +0300 Subject: [PATCH 3/4] Add a setup guide for this fork Resolving requirements.txt against current package versions produces an environment that imports but does not work: cu130 torch wheels against a CUDA 12.4 driver, librosa reaching for the removed pkg_resources, numpy 2 against a cython extension built for 1.x, and transformers 5 missing the generation symbols stream_generator.py imports. KURULUM.md records the version set that passes the tests, the espeak-ng symlink the phonemizer already expects, how to run commands when ROS leaks into PYTHONPATH, which test failures are environmental rather than bugs, and how to turn on the half precision and batched decoding paths. Co-Authored-By: Claude Opus 5 --- KURULUM.md | 196 +++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 5 ++ 2 files changed, 201 insertions(+) create mode 100644 KURULUM.md diff --git a/KURULUM.md b/KURULUM.md new file mode 100644 index 0000000000..b186364345 --- /dev/null +++ b/KURULUM.md @@ -0,0 +1,196 @@ +# Kurulum ve Hızlı Başlangıç + +Bu depo (Coqui TTS 0.22.0) artık bakımda değil ve `requirements.txt` bugünkü paket +sürümleriyle çözüldüğünde **import edilebilen ama çalışmayan** bir ortam üretiyor. Aşağıdaki +adımlar, denenmiş ve testleri geçen sürüm kümesini kurar. + +Referans donanım: NVIDIA RTX 4050 Laptop (6 GB), sürücü 550.163 (CUDA 12.4), Ubuntu, Python 3.10. + +--- + +## 1. Kurulum + +`setup.py` Python **3.9–3.11** arası ister; 3.12 ve üzeri reddedilir. + +```bash +cd /path/to/TTS +uv venv --python 3.10 +uv pip install -e '.[all,dev,notebooks]' +``` + +Bu ilk komut tek başına yeterli değil. Ardından üç düzeltme gerekiyor: + +### 1a. Sürücünüze uygun torch + +Varsayılan çözümleme `cu130` tekerleklerini kurar; bunlar 580+ sürücü ister. CUDA 12.4 +sürücüsüyle `torch.cuda.is_available()` sessizce `False` döner ve her şey CPU'da çalışır. + +```bash +uv pip install "torch==2.5.1+cu124" "torchaudio==2.5.1+cu124" \ + --extra-index-url https://download.pytorch.org/whl/cu124 \ + --index-strategy unsafe-best-match +``` + +> `--index-url` (extra olmayan) kullanmayın: `nvidia-*` bağımlılıkları çözülemez ve kurulum +> "unsatisfiable" hatasıyla düşer. + +Farklı bir sürücünüz varsa `nvidia-smi` çıktısındaki **CUDA Version** alanına bakıp uygun +`cuXXX` derlemesini seçin. + +### 1b. Çalışan paket sürümleri + +```bash +uv pip install "librosa==0.10.2.post1" "transformers==4.40.2" \ + "numpy==1.26.4" "scipy==1.11.4" +``` + +Neden bu sürümler: + +| Paket | Sorun | +|---|---| +| `librosa` 0.10.0 | `pkg_resources` import ediyor; setuptools 81+ bunu kaldırdı, `import TTS` patlıyor | +| `transformers` 5.x | `SampleOutput` ve `LogitsWarper` kaldırıldı; `stream_generator.py` ve `tortoise/arch_utils.py` bunları kullanıyor | +| `numpy` 2.x | Derlenmiş `monotonic_align` cython uzantısı numpy 1.x ABI'sine göre | + +### 1c. espeak ve nltk verisi + +Fonemleştirme `espeak` adında bir çalıştırılabilir arıyor. Sistemde çoğunlukla yalnızca +`espeak-ng` bulunur; kodun kendisi bunun symlink olmasını zaten bekliyor, dolayısıyla sudo'ya +gerek yok: + +```bash +ln -sf /usr/bin/espeak-ng .venv/bin/espeak +``` + +`espeak-ng` yoksa: `sudo apt install espeak-ng`. + +Korece dışındaki diller için gerekmez, ama testler için nltk verisi: + +```bash +.venv/bin/python -c "import nltk; [nltk.download(p, quiet=True) for p in ['averaged_perceptron_tagger','averaged_perceptron_tagger_eng','punkt','cmudict']]" +``` + +--- + +## 2. Komutları çalıştırma + +Bu makinede kabuk profili ROS Humble'ı source ediyor ve `PYTHONPATH` venv'in içine sızıyor. +Ayrıca depo testleri çıplak `python` çağırıyor; sistemde yalnızca `python3` var. İkisini de +çözen kalıp: + +```bash +env PYTHONPATH= PATH="$PWD/.venv/bin:$PATH" python ... +``` + +ROS kullanmıyorsanız `PYTHONPATH=` kısmı zararsızdır, bırakabilirsiniz. + +### Kurulumu doğrulama + +```bash +env PYTHONPATH= .venv/bin/python -c " +import torch, TTS +from TTS.api import TTS as T +print('torch', torch.__version__, '| cuda', torch.cuda.is_available()) +print('gpu ', torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'YOK - CPU modunda') +" +``` + +`cuda True` ve GPU adı görmelisiniz. `False` görüyorsanız adım 1a atlanmış demektir. + +--- + +## 3. Hızlı test + +```bash +env PYTHONPATH= PATH="$PWD/.venv/bin:$PATH" COQUI_TOS_AGREED=1 python deneme.py +``` + +`COQUI_TOS_AGREED=1`, XTTS'in lisans onayı sorusunu atlar (etkileşimsiz çalıştırmada gerekir). + +`deneme.py` temel bir duman testidir: model indirir, ses klonlar, `output.wav` üretir. Tek +eksiği, GPU'ya düşülüp düşülmediğini söylememesi — CUDA çalışmıyorsa sessizce CPU'ya +geçer, sadece çok yavaşlar. Bunu görmek için başına şunu ekleyin: + +```python +print("device:", device) +``` + +--- + +## 4. Hızlı çıkarım (XTTS) + +Bu çatalda XTTS için iki hızlandırma var. Ayrıntılar: `docs/source/models/xtts.md`. + +Doğrudan model API'siyle: + +```python +model.load_checkpoint(config, checkpoint_dir="/path/to/xtts/", eval=True, half=True) +out = model.inference(text, "tr", gpt_cond_latent, speaker_embedding, + enable_text_splitting=True, batch_size=4) +``` + +`TTS.api` üzerinden — dikkat: API metni modele ulaşmadan cümlelere böldüğü için batch ancak +bölmeyi modele bırakınca devreye girer: + +```python +from TTS.api import TTS +tts = TTS("tts_models/multilingual/multi-dataset/xtts_v2").to("cuda") +tts.synthesizer.tts_model.use_half_precision() + +tts.tts_to_file( + text=uzun_metin, + speaker_wav="Recording.wav", + language="tr", + file_path="output.wav", + split_sentences=False, # metnin tamamını XTTS'e ver + enable_text_splitting=True, + batch_size=4, +) +``` + +RTX 4050 Laptop'ta paragraf uzunluğunda metin: RTF 0.259 → 0.178 (fp16) → **0.092** +(fp16 + batch=4). VRAM 2.21 → 1.49 GB. + +Ölçmek için: + +```bash +env PYTHONPATH= PATH="$PWD/.venv/bin:$PATH" COQUI_TOS_AGREED=1 python scripts/bench_xtts_paragraph.py +``` + +--- + +## 5. Testler + +```bash +# hızlı olanlar (~1 dk) +env PYTHONPATH= PATH="$PWD/.venv/bin:$PATH" python -m pytest tests/text_tests tests/data_tests -q + +# ses işleme ve çıkarım (~3 dk) +env PYTHONPATH= PATH="$PWD/.venv/bin:$PATH" python -m pytest tests/aux_tests tests/inference_tests -q + +# model ve eğitim testleri (~20 dk) +env PYTHONPATH= PATH="$PWD/.venv/bin:$PATH" python -m pytest tests/tts_tests tests/tts_tests2 tests/vocoder_tests \ + --continue-on-collection-errors -q +``` + +`pytest` ayrıca kurulmalıdır: `uv pip install pytest`. + +### Bilinen ve kod hatası olmayan düşen testler + +| Test | Neden | +|---|---| +| `test_phonemizer.py` espeak testleri (4) | espeak-ng 1.50 ile 1.51+ arasındaki fonem farkı (`ᵻ` ↔ `ɪ`) | +| `test_tokenizer.py::...eos_bos_and_blank` | aynı espeak-ng sürüm farkı (`c` ↔ `k`) | +| `test_korean_phonemizer.py` | `mecab` kurulu değil; yalnızca Korece'yi etkiler | +| `test_losses.py::BCELossTest` | `1.4e-45 != 0.0` — torch sürümünden gelen denormal sayı | +| `tests/vc_tests` (GPU'da) | Test CPU tensörü üretip CUDA modeline veriyor; `CUDA_VISIBLE_DEVICES="" ` ile 11/11 geçer | +| `test_xtts_v2-0_gpt_train.py` | XTTS v2 GPT eğitimi 6 GB VRAM'e sığmıyor; `CUDA_VISIBLE_DEVICES=""` ile geçer | + +--- + +## 6. Bu makinede çalışmayanlar + +- **DeepSpeed** — `nvcc` yok (inference kernel'lerini JIT derleyemez) ve depo, güncel + DeepSpeed'in kaldırdığı `replace_method` argümanını geçiyor. Hızlandırma için yukarıdaki + fp16 + batch yolunu kullanın. +- **XTTS v2 GPT eğitimi** — 6 GB VRAM yetmiyor. diff --git a/README.md b/README.md index e3205c1bd3..2d29217fc4 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,11 @@ Underlined "TTS*" and "Judy*" are **internal** 🐸TTS models that are not relea You can also help us implement more models. ## Installation + +> **Bu çatal için:** aşağıdaki adımlar bugünkü paket sürümleriyle çalışmayan bir ortam +> üretiyor (torch/CUDA uyumsuzluğu, `pkg_resources`, numpy 2, transformers 5). Denenmiş +> kurulum, hızlı test ve hızlı çıkarım ayarları için **[KURULUM.md](KURULUM.md)**. + 🐸TTS is tested on Ubuntu 18.04 with **python >= 3.9, < 3.12.**. If you are only interested in [synthesizing speech](https://tts.readthedocs.io/en/latest/inference.html) with the released 🐸TTS models, installing from PyPI is the easiest option. From a31435c2b8858ff15317fbf6bf7ada24dfb7ba38 Mon Sep 17 00:00:00 2001 From: yunusemretom Date: Mon, 17 Aug 2026 14:25:03 +0300 Subject: [PATCH 4/4] feat: add voice synthesis scripts with caching and update .gitignore --- deneme.py | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 deneme.py diff --git a/deneme.py b/deneme.py new file mode 100644 index 0000000000..775b0daba6 --- /dev/null +++ b/deneme.py @@ -0,0 +1,97 @@ +import shutil +import subprocess +import time + +import soundfile as sf +import torch + +from TTS.api import TTS + +# --- ayarlar --------------------------------------------------------------- +HALF = True # fp16: GPT'yi yarı hassasiyette çalıştırır (yalnızca GPU'da) +BATCH_SIZE = 4 # birden çok cümleyi tek geçişte çöz; 1 = kapalı (uzun metinde etkili) +WARMUP = True # ilk çağrı CUDA kernel/bellek kurulumunu da içerir, ölçüme katma +PLAY = True # üretim biter bitmez sesi çal + +LANGUAGE = "tr" +SPEAKER_WAV = "ses2.mp3" +OUTPUT = "output.wav" # her turda üzerine yazılır +# --------------------------------------------------------------------------- + + +def play(path): + """Sesi çalar; bir oynatıcı başarısız olursa sıradakini dener.""" + players = { + "paplay": [path], + "pw-play": [path], + "aplay": ["-q", path], + "ffplay": ["-nodisp", "-autoexit", "-loglevel", "quiet", path], + } + for name, args in players.items(): + exe = shutil.which(name) + if exe is None: + continue + if subprocess.run([exe, *args], check=False).returncode == 0: + return + print(f" {name} çalamadı, sıradaki deneniyor") + print(f" ses çalınamadı, dosya kaydedildi: {path}") + + +device = "cuda" if torch.cuda.is_available() else "cpu" +print(f"device: {device}" + (f" ({torch.cuda.get_device_name(0)})" if device == "cuda" else "")) + +# Kullanılabilir modelleri görmek için: +# print(TTS().list_models()) + +tts = TTS("tts_models/multilingual/multi-dataset/xtts_v2").to(device) +model = tts.synthesizer.tts_model +sample_rate = model.config.audio.output_sample_rate + +if HALF: + if device == "cuda": + model.use_half_precision() + print("fp16: açık") + else: + print("fp16: atlandı (yarı hassasiyet yalnızca GPU'da anlamlı)") +else: + print("fp16: kapalı") +print(f"batch: {BATCH_SIZE}" if BATCH_SIZE > 1 else "batch: kapalı") + +# Konuşmacı latent'leri bir kez çıkarılıp her turda yeniden kullanılır. tts.tts_to_file() +# bunları her çağrıda baştan hesaplar; bir döngüde bu tur başına boşa giden zamandır. +print(f"\nkonuşmacı analiz ediliyor: {SPEAKER_WAV}") +gpt_cond_latent, speaker_embedding = model.get_conditioning_latents(audio_path=[SPEAKER_WAV]) + +if WARMUP: + model.inference("Isınma turu.", LANGUAGE, gpt_cond_latent, speaker_embedding) + +print("\nMetin girin. Boş satır, Ctrl+D veya Ctrl+C çıkar.") +while True: + try: + text = input("\nmetin> ").strip() + except (EOFError, KeyboardInterrupt): + print() + break + + if not text: + break + + start = time.perf_counter() + out = model.inference( + text, + LANGUAGE, + gpt_cond_latent, + speaker_embedding, + enable_text_splitting=True, + batch_size=BATCH_SIZE, + ) + elapsed = time.perf_counter() - start + + sf.write(OUTPUT, out["wav"], sample_rate) + audio_seconds = len(out["wav"]) / sample_rate + print(f"süre: {elapsed:.2f} s | ses: {audio_seconds:.2f} s | RTF: {elapsed / audio_seconds:.3f}") + + if PLAY: + play(OUTPUT) + +print("çıkıldı.")