ZK-AI
/
livetalking
mirror of https://gitee.com/dumengxue/livetalking.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add support gpt-sovits

This commit is contained in:
lipku 2024-04-21 17:09:08 +08:00
parent 6d4952c1bf
commit 2e64be4b5d
3 changed files with 78 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
app.py
View File

@ -75,7 +75,7 @@ def xtts(text, speaker, language, server_url, stream_chunk_size) -> Iterator[byt
return
first = True
for chunk in res.iter_content(chunk_size=960):
for chunk in res.iter_content(chunk_size=960): #24K*20ms*2
if first:
end = time.perf_counter()
print(f"xtts Time to first chunk: {end-start}s")
@ -85,12 +85,20 @@ def xtts(text, speaker, language, server_url, stream_chunk_size) -> Iterator[byt
print("xtts response.elapsed:", res.elapsed)
def gpt_sovits(text, speaker, language, server_url, stream_chunk_size) -> Iterator[bytes]:
def gpt_sovits(text, character, language, server_url, stream_chunk_size) -> Iterator[bytes]:
start = time.perf_counter()
speaker["text"] = text
speaker["language"] = language
speaker["stream_chunk_size"] = stream_chunk_size # you can reduce it to get faster response, but degrade quality
res = requests.get(f"{server_url}&text="+text,stream=True)
req={}
req["text"] = text
req["text_language"] = language
req["character"] = character
#req["emotion"] = emotion
#req["stream_chunk_size"] = stream_chunk_size # you can reduce it to get faster response, but degrade quality
req["stream"] = True
res = requests.post(
f"{server_url}/tts",
json=req,
stream=True,
)
end = time.perf_counter()
print(f"gpt_sovits Time to make POST: {end-start}s")
@ -99,7 +107,7 @@ def gpt_sovits(text, speaker, language, server_url, stream_chunk_size) -> Iterat
return
first = True
for chunk in res.iter_content(chunk_size=960):
for chunk in res.iter_content(chunk_size=1280): #32K*20ms*2
if first:
end = time.perf_counter()
print(f"gpt_sovits Time to first chunk: {end-start}s")
@ -109,7 +117,7 @@ def gpt_sovits(text, speaker, language, server_url, stream_chunk_size) -> Iterat
print("gpt_sovits response.elapsed:", res.elapsed)
def stream_xtts(audio_stream,render):
def stream_tts(audio_stream,render):
for chunk in audio_stream:
if chunk is not None:
render.push_audio(chunk)
@ -121,19 +129,19 @@ def txt_to_audio(text_):
t = time.time()
asyncio.get_event_loop().run_until_complete(main(voicename,text,nerfreal))
print(f'-------edge tts time:{time.time()-t:.4f}s')
elif tts_type == "gpt": #gpt_sovits
stream_xtts(
elif tts_type == "gpt-sovits": #gpt_sovits
stream_tts(
gpt_sovits(
text_,
gspeaker,
"zh-cn", #en args.language,
"http://127.0.0.1:9880/tts_ava?ava=maimai&streaming_mode=true", #args.server_url,
"test", #character
"zh", #en args.language,
"http://127.0.0.1:5000", #args.server_url,
"20" #args.stream_chunk_size
),
nerfreal
)
else: #xtts
stream_xtts(
stream_tts(
xtts(
text_,
gspeaker,
@ -354,18 +362,18 @@ if __name__ == '__main__':
parser.add_argument('--fullbody_offset_x', type=int, default=0)
parser.add_argument('--fullbody_offset_y', type=int, default=0)
parser.add_argument('--tts', type=str, default='edgetts') #xtts
parser.add_argument('--tts', type=str, default='edgetts') #xtts gpt-sovits
parser.add_argument('--ref_file', type=str, default=None)
parser.add_argument('--xtts_server', type=str, default='http://localhost:9000')
parser.add_argument('--tts_server', type=str, default='http://localhost:9000')
opt = parser.parse_args()
app.config.from_object(opt)
#print(app.config['xtts_server'])
#print(app.config['tts_server'])
tts_type = opt.tts
if tts_type == "xtts":
print("Computing the latents for a new reference...")
gspeaker = get_speaker(opt.ref_file, opt.xtts_server)
gspeaker = get_speaker(opt.ref_file, opt.tts_server)
# assert test mode
opt.test = True

7
asrreal.py
View File

@ -314,10 +314,13 @@ class ASR:
def push_audio(self,buffer): #push audio pcm from tts
print(f'[INFO] push_audio {len(buffer)}')
if self.opt.tts == "xtts":
if self.opt.tts == "xtts" or self.opt.tts == "gpt-sovits":
if len(buffer)>0:
stream = np.frombuffer(buffer, dtype=np.int16).astype(np.float32) / 32767
stream = resampy.resample(x=stream, sr_orig=24000, sr_new=self.sample_rate)
if self.opt.tts == "xtts":
stream = resampy.resample(x=stream, sr_orig=24000, sr_new=self.sample_rate)
else:
stream = resampy.resample(x=stream, sr_orig=32000, sr_new=self.sample_rate)
#byte_stream=BytesIO(buffer)
#stream = self.__create_bytes_stream(byte_stream)
streamlen = stream.shape[0]

70
tts/README.md
View File

@ -1,27 +1,51 @@
一、采用gpt-sovits方案，bert-sovits适合长音频训练，gpt-sovits运行短音频快速推理
下载tts服务端代码
https://github.com/yanyuxiyangzk/GPT-SoVITS/tree/fast_inference_
api_v2.py即启动的服务端代码，也可以打开声音克隆界面进行训练，可以训练带感情语气等
# 采用gpt-sovits方案，bert-sovits适合长音频训练，gpt-sovits运行短音频快速推理
## 部署tts推理
git clone https://github.com/X-T-E-R/GPT-SoVITS-Inference.git
1、启动
python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/tts_infer.yaml
1. 安装依赖库
```
conda create -n GPTSoVits python=3.9
conda activate GPTSoVits
bash install.sh
```
从 [GPT-SoVITS Models](https://huggingface.co/lj1995/GPT-SoVITS) 下载预训练模型，并将它们放置在 `GPT_SoVITS\pretrained_models` 中
2. Model Folder Format
模型文件下载地址 https://www.yuque.com/xter/zibxlp/gsximn7ditzgispg
下载的模型文件放到trained目录下, 如 `trained/Character1/`
Put the pth / ckpt / wav files in it, the wav should be named as the prompt text
Like :
```
trained
--hutao
----hutao-e75.ckpt
----hutao_e60_s3360.pth
----hutao said something.wav
```
3. 启动
后端接口: python Inference/src/tts_backend.py
如果有错误提示找不到cmudict，从这下载https://github.com/nltk/nltk_data，将packages改名为nltk_data放到home目录下
管理页面: python Inference/src/TTS_Webui.py, 浏览器打开可以管理character和emotion
http://127.0.0.1:9880/set_sovits_weights?weights_path=SoVITS_weights/maimai_e55_s1210.pth
http://127.0.0.1:9880/set_gpt_weights?weights_path=GPT_weights/maimai-e21.ckpt
4. 接口测试
Character and Emotion List
To obtain the supported characters and their corresponding emotions, please visit the following URL:
- URL: `http://127.0.0.1:5000/character_list`
- Returns: A JSON format list of characters and corresponding emotions
- Method: `GET`
2、接口测试
http://127.0.0.1:9880/set_ava?ava=maimai
http://127.0.0.1:9880/tts_ava?ava=maimai&text=我是一个粉刷匠，粉刷本领强。我要把那新房子，刷得更漂亮。刷了房顶又刷墙，刷子像飞一样。哎呀我的小鼻子，变呀变了样
http://127.0.0.1:9880/tts_ava?ava=maimai&text=我是一个粉刷匠，粉刷本领强。我要把那新房子，刷得更漂亮。刷了房顶又刷墙，刷子像飞一样。哎呀我的小鼻子，变呀变了样&streaming_mode=true
http://127.0.0.1:9880/tts_ava?ava=maimai&text=我是一个粉刷匠，粉刷本领强。我要把那新房子，刷得更漂亮。刷了房顶又刷墙，刷子像飞一样。哎呀我的小鼻子，变呀变了样。&text_lang=zh&ref_audio_path=mengpai.wav&prompt_lang=zh&prompt_text=呜哇好生气啊！不要把我跟一斗相提并论！&text_split_method=cut5&batch_size=1&media_type=wav&streaming_mode=true
3、使用
设置角色
http://127.0.0.1:9880/set_ava?ava=maimai
tts接口
http://127.0.0.1:9880/tts_ava?ava=maimai&text=我是一个粉刷匠，粉刷本领强。我要把那新房子，刷得更漂亮。刷了房顶又刷墙，刷子像飞一样。哎呀我的小鼻子，变呀变了样&streaming_mode=true
```
{
"Hanabi": [
"default",
"Normal",
"Yandere",
],
"Hutao": [
"default"
]
}
```