ZK-AI
/
livetalking
mirror of https://gitee.com/dumengxue/livetalking.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update readme

This commit is contained in:
lipku 2024-07-01 07:38:26 +08:00
parent 9fe4c7fccf
commit c812e45f35
4 changed files with 92 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -6,11 +6,10 @@ Real time interactive streaming digital human， realize audio video synchronous
## Features ## Features
1. 支持多种数字人模型: ernerf、musetalk、wav2lip 1. 支持多种数字人模型: ernerf、musetalk、wav2lip
2. 支持声音克隆 2. 支持声音克隆
3. 支持多种音频特征驱动：wav2vec、hubert 3. 支持数字人说话被打断
4. 支持全身视频拼接 4. 支持全身视频拼接
5. 支持rtmp和webrtc 5. 支持rtmp和webrtc
6. 支持视频编排：不说话时播放自定义视频 6. 支持视频编排：不说话时播放自定义视频
7. 支持大模型对话
## 1. Installation ## 1. Installation
@ -178,7 +177,7 @@ python -m scripts.realtime_inference --inference_config configs/inference/realti
- 下载模型 - 下载模型
下载wav2lip运行需要的模型，网盘地址 https://drive.uc.cn/s/551be97d7cfa4 下载wav2lip运行需要的模型，网盘地址 https://drive.uc.cn/s/551be97d7cfa4
将s3fd.pth拷到本项目wav2lip/face_detection/detection/sfd/s3fd.pth, 将wav2lip.pth拷到本项目的models下 将s3fd.pth拷到本项目wav2lip/face_detection/detection/sfd/s3fd.pth, 将wav2lip.pth拷到本项目的models下
数字人模型文件 wav2lip_avatar1.tar.gz, 解压后将整个文件夹拷到本项目的data/avatars下 数字人模型文件 wav2lip_avatar1.tar.gz，网盘地址 https://drive.uc.cn/s/5bd0cde0b0774, 解压后将整个文件夹拷到本项目的data/avatars下
- 运行 - 运行
python app.py --transport webrtc --model wav2lip --avatar_id wav2lip_avatar1 python app.py --transport webrtc --model wav2lip --avatar_id wav2lip_avatar1
用浏览器打开http://serverip:8010/webrtcapi.html 用浏览器打开http://serverip:8010/webrtcapi.html

44
web/chat.html
View File

@ -29,22 +29,22 @@
$(document).ready(function() { $(document).ready(function() {
var host = window.location.hostname var host = window.location.hostname
var ws = new WebSocket("ws://"+host+":8000/humanchat"); // var ws = new WebSocket("ws://"+host+":8000/humanecho");
//document.getElementsByTagName("video")[0].setAttribute("src", aa["video"]); // //document.getElementsByTagName("video")[0].setAttribute("src", aa["video"]);
ws.onopen = function() { // ws.onopen = function() {
console.log('Connected'); // console.log('Connected');
}; // };
ws.onmessage = function(e) { // ws.onmessage = function(e) {
console.log('Received: ' + e.data); // console.log('Received: ' + e.data);
data = e // data = e
var vid = JSON.parse(data.data); // var vid = JSON.parse(data.data);
console.log(typeof(vid),vid) // console.log(typeof(vid),vid)
//document.getElementsByTagName("video")[0].setAttribute("src", vid["video"]); // //document.getElementsByTagName("video")[0].setAttribute("src", vid["video"]);
}; // };
ws.onclose = function(e) { // ws.onclose = function(e) {
console.log('Closed'); // console.log('Closed');
}; // };
flvPlayer = mpegts.createPlayer({type: 'flv', url: "http://"+host+":8080/live/livestream.flv", isLive: true, enableStashBuffer: false}); flvPlayer = mpegts.createPlayer({type: 'flv', url: "http://"+host+":8080/live/livestream.flv", isLive: true, enableStashBuffer: false});
flvPlayer.attachMediaElement(document.getElementById('video_player')); flvPlayer.attachMediaElement(document.getElementById('video_player'));
@ -55,9 +55,19 @@
e.preventDefault(); e.preventDefault();
var message = $('#message').val(); var message = $('#message').val();
console.log('Sending: ' + message); console.log('Sending: ' + message);
ws.send(message); fetch('/human', {
body: JSON.stringify({
text: message,
type: 'chat',
}),
headers: {
'Content-Type': 'application/json'
},
method: 'POST'
});
//ws.send(message);
$('#message').val(''); $('#message').val('');
}); });
}); });
</script> </script>
</html> </html>

52
web/rtcpushchat.html
View File

@ -51,30 +51,40 @@
<script type="text/javascript" charset="utf-8"> <script type="text/javascript" charset="utf-8">
$(document).ready(function() { $(document).ready(function() {
var host = window.location.hostname // var host = window.location.hostname
var ws = new WebSocket("ws://"+host+":8000/humanchat"); // var ws = new WebSocket("ws://"+host+":8000/humanecho");
//document.getElementsByTagName("video")[0].setAttribute("src", aa["video"]); // //document.getElementsByTagName("video")[0].setAttribute("src", aa["video"]);
ws.onopen = function() { // ws.onopen = function() {
console.log('Connected'); // console.log('Connected');
}; // };
ws.onmessage = function(e) { // ws.onmessage = function(e) {
console.log('Received: ' + e.data); // console.log('Received: ' + e.data);
data = e // data = e
var vid = JSON.parse(data.data); // var vid = JSON.parse(data.data);
console.log(typeof(vid),vid) // console.log(typeof(vid),vid)
//document.getElementsByTagName("video")[0].setAttribute("src", vid["video"]); // //document.getElementsByTagName("video")[0].setAttribute("src", vid["video"]);
}; // };
ws.onclose = function(e) { // ws.onclose = function(e) {
console.log('Closed'); // console.log('Closed');
}; // };
$('#echo-form').on('submit', function(e) { $('#echo-form').on('submit', function(e) {
e.preventDefault(); e.preventDefault();
var message = $('#message').val(); var message = $('#message').val();
console.log('Sending: ' + message); console.log('Sending: ' + message);
ws.send(message); fetch('/human', {
$('#message').val(''); body: JSON.stringify({
text: message,
type: 'chat',
}),
headers: {
'Content-Type': 'application/json'
},
method: 'POST'
});
//ws.send(message);
$('#message').val('');
}); });
}); });

53
web/webrtcchat.html
View File

@ -53,30 +53,41 @@
<script type="text/javascript" charset="utf-8"> <script type="text/javascript" charset="utf-8">
$(document).ready(function() { $(document).ready(function() {
var host = window.location.hostname // var host = window.location.hostname
var ws = new WebSocket("ws://"+host+":8000/humanchat"); // var ws = new WebSocket("ws://"+host+":8000/humanecho");
//document.getElementsByTagName("video")[0].setAttribute("src", aa["video"]); // //document.getElementsByTagName("video")[0].setAttribute("src", aa["video"]);
ws.onopen = function() { // ws.onopen = function() {
console.log('Connected'); // console.log('Connected');
}; // };
ws.onmessage = function(e) { // ws.onmessage = function(e) {
console.log('Received: ' + e.data); // console.log('Received: ' + e.data);
data = e // data = e
var vid = JSON.parse(data.data); // var vid = JSON.parse(data.data);
console.log(typeof(vid),vid) // console.log(typeof(vid),vid)
//document.getElementsByTagName("video")[0].setAttribute("src", vid["video"]); // //document.getElementsByTagName("video")[0].setAttribute("src", vid["video"]);
}; // };
ws.onclose = function(e) { // ws.onclose = function(e) {
console.log('Closed'); // console.log('Closed');
}; // };
$('#echo-form').on('submit', function(e) { $('#echo-form').on('submit', function(e) {
e.preventDefault(); e.preventDefault();
var message = $('#message').val(); var message = $('#message').val();
console.log('Sending: ' + message); console.log('Sending: ' + message);
ws.send(message); fetch('/human', {
$('#message').val(''); body: JSON.stringify({
text: message,
type: 'chat',
interrupt: true,
}),
headers: {
'Content-Type': 'application/json'
},
method: 'POST'
});
//ws.send(message);
$('#message').val('');
}); });
}); });
</script> </script>