Upload folder using huggingface_hub

.gitattributes

@@ -82,3 +82,4 @@ trained_190k_steps/rtf_test_results/test_018.wav filter=lfs diff=lfs merge=lfs -
 trained_190k_steps/rtf_test_results/test_019.wav filter=lfs diff=lfs merge=lfs -text
 trained_190k_steps/rtf_test_results/test_020.wav filter=lfs diff=lfs merge=lfs -text
 trained_190k_steps/test.wav filter=lfs diff=lfs merge=lfs -text
+trained_190k_steps/output_streaming.wav filter=lfs diff=lfs merge=lfs -text

trained_190k_steps/output.wav

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b56b3b68f11fdb8539634bb27312f1346b3876ede818d311f6c89dd8b8e94dd
-size 499244
+oid sha256:252c4c7ef3a1623c636d83cae90aa59ca30c1089d7bbaaddd97d8cd0afa4b8dc
+size 599724

trained_190k_steps/output_streaming.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4578311c27d08e522b4bab81bc38cc85e8528e439e7b57b6794cbf48deeed36
+size 499244

trained_190k_steps/tts_streaming.py

@@ -0,0 +1,142 @@
+import os
+import sys
+current_dir = os.path.dirname(os.path.abspath(__file__))
+print('add current dir to sys.path', current_dir)
+sys.path.append(current_dir)
+from sparktts.models.audio_tokenizer import BiCodecTokenizer
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import soundfile as sf
+import numpy as np
+import torch
+from utilities import generate_embeddings
+def generate_speech(model, tokenizer, text, bicodec, prompt_text=None, prompt_audio=None, 
+                   max_new_tokens=3000, do_sample=True, top_k=50, top_p=0.95, 
+                   temperature=1.0, device="cuda:0"):
+    """
+    生成语音的函数
+    
+    Args:
+        model: 语言模型
+        tokenizer: 文本分词器
+        text: 要生成语音的文本
+        bicodec: BiCodecTokenizer 实例
+        prompt_text: 提示文本（可选）
+        prompt_audio: 提示音频数组（可选）
+        max_new_tokens: 最大生成token数
+        do_sample: 是否使用采样
+        top_k: top-k采样参数
+        top_p: top-p采样参数
+        temperature: 温度参数
+        device: 设备
+    
+    Returns:
+        wav: 生成的音频波形
+    """
+    # 设置eos_token_id - 根据训练代码，eos_token_id = model.config.vocab_size - 1
+    eos_token_id = model.config.vocab_size - 1
+    print(f"EOS token ID: {eos_token_id}")
+    
+    # 生成输入嵌入
+    embeddings = generate_embeddings(
+        model=model,
+        tokenizer=tokenizer,
+        text=text,
+        bicodec=bicodec,
+        prompt_text=prompt_text,
+        prompt_audio=prompt_audio
+    )
+    
+    print("开始生成语音...")
+    print(f"输入嵌入形状: {embeddings['input_embs'].shape}")
+    global_tokens = embeddings['global_tokens'].unsqueeze(0)
+    # 设置模型为评估模式
+    print(f'embeddings dtype: {embeddings["input_embs"].dtype}')
+    model.eval()
+    
+    with torch.no_grad():
+        # 使用模型的generate方法
+        generated_outputs = model.generate(
+            inputs_embeds=embeddings['input_embs'],
+            attention_mask=torch.ones((1, embeddings['input_embs'].shape[1]),dtype=torch.long,device=device),
+            max_new_tokens=max_new_tokens,
+            do_sample=do_sample,
+            top_k=top_k,
+            top_p=top_p,
+            temperature=temperature,
+            eos_token_id=eos_token_id,
+            pad_token_id=tokenizer.pad_token_id if hasattr(tokenizer, 'pad_token_id') else tokenizer.eos_token_id,
+            use_cache=True
+        )
+    print(f"generated_outputs: {generated_outputs}")
+    
+    print(f"生成的token数量: {generated_outputs.shape}")
+    print(f"生成的token IDs: {generated_outputs.tolist()}")
+    
+    # 直接使用生成的token ID作为semantic tokens
+    # 注意：这里生成的token ID是模型词表中的ID，不是原始tokenizer的词表
+    semantic_tokens_tensor = generated_outputs[:,:-1]
+    
+    print(f"Semantic tokens shape: {semantic_tokens_tensor.shape}")
+    
+    #simulate streaming
+    target_sample_rate = bicodec.config['sample_rate']
+    print(f"Global tokens shape: {global_tokens.shape}")
+    BUF_SIZE = 25 # since 50 tokens per second, 25 tokens is 0.5 second
+    chunk_size = 125 # start to generate audio after 125 tokens
+    buffered_semantic_tokens = torch.zeros((1, 0), dtype=torch.long, device=device)
+    whole_wav = np.array([], dtype=np.float32)
+    for i in range(0, semantic_tokens_tensor.shape[1], chunk_size):
+        buffered_size = buffered_semantic_tokens.shape[1]
+        current_semantic_tokens = semantic_tokens_tensor[:, i:i+chunk_size]
+        print(f"generate segmant [{i}:{i+chunk_size}]: shape {current_semantic_tokens.shape}")
+        current_semantic_tokens = torch.cat([buffered_semantic_tokens, current_semantic_tokens], dim=1)
+        print(f"After concat: shape {current_semantic_tokens.shape} with buffered shape {buffered_semantic_tokens.shape}")
+        buffered_semantic_tokens = current_semantic_tokens[:, -BUF_SIZE:]
+        with torch.no_grad():
+            wav = bicodec.detokenize(global_tokens, current_semantic_tokens)
+        print(f"Generated audio shape: {wav.shape}")
+        wav = wav[int(target_sample_rate * buffered_size/50):]
+        print(f"After cut: shape {wav.shape}")
+        whole_wav = np.concatenate([whole_wav, wav])
+    print(f"Whole wav shape: {whole_wav.shape}")
+    return whole_wav
+
+device = 'cuda:2'
+
+audio_tokenizer = BiCodecTokenizer(model_dir=current_dir, device=device)
+
+print(audio_tokenizer)
+
+tokenizer = AutoTokenizer.from_pretrained(current_dir, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(current_dir, trust_remote_code=True)
+print(tokenizer)
+print(model)
+
+model = model.bfloat16().to(device)
+model.eval()
+
+prompt_text = "我们并不是通过物理移动手段找到星河的。"
+prompt_audio_file = os.path.join(current_dir, 'kafka.wav')
+prompt_audio, sampling_rate = sf.read(prompt_audio_file)
+
+print(f"Loaded prompt audio from {prompt_audio_file}")
+print(f"Original sampling rate: {sampling_rate}Hz")
+print(f"Audio shape: {prompt_audio.shape}")
+
+target_sample_rate = audio_tokenizer.config['sample_rate']
+if sampling_rate != target_sample_rate:
+    print(f"Resampling from {sampling_rate}Hz to {target_sample_rate}Hz...")
+    from librosa import resample
+    prompt_audio = resample(prompt_audio, orig_sr=sampling_rate, target_sr=target_sample_rate)
+    prompt_audio = np.array(prompt_audio, dtype=np.float32)
+    print(f"Resampled audio shape: {prompt_audio.shape}")
+else:
+    print(f"Audio sampling rate already matches target ({target_sample_rate}Hz)")
+
+text = "二房他们已经接受了老爷子安排的：大房拿企业、二房拿钱的设定。富贵闲人他们也做了。在嫡长女和国资抢股权期间不出来搅局，就连老爷子的葬礼都没有露面，安安静静坐实老爷子一辈子的完美人设。"
+wav = generate_speech(model, tokenizer, text, audio_tokenizer, prompt_audio=prompt_audio, device=device)
+sf.write('output_streaming.wav', wav, target_sample_rate)
+
+
+
+