Spaces:
Running
on
Zero
Running
on
Zero
fだ
Browse files- staticfiles/livetest copy.html +632 -0
- staticfiles/livetest.html +2 -2
staticfiles/livetest copy.html
ADDED
@@ -0,0 +1,632 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
<!DOCTYPE html>
|
2 |
+
<html>
|
3 |
+
|
4 |
+
<head>
|
5 |
+
<meta charset="utf-8">
|
6 |
+
<meta name="viewport" content="width=device-width">
|
7 |
+
<title>Sample Chat AI VTuber</title>
|
8 |
+
<link rel="stylesheet" href="aivtuber.css">
|
9 |
+
<script src="./aivtuber.js"></script>
|
10 |
+
<title>Live2Dサンプル</title>
|
11 |
+
<!-- Live2D -->
|
12 |
+
<script src="./live2dcubismcore.js"></script>
|
13 |
+
<script src="//cdn.jsdelivr.net/gh/dylanNew/live2d/webgl/Live2D/lib/live2d.min.js"></script>
|
14 |
+
<!-- PixiJS -->
|
15 |
+
<script src="//cdnjs.cloudflare.com/ajax/libs/pixi.js/5.1.3/pixi.min.js"></script>
|
16 |
+
<script src="//cdn.jsdelivr.net/npm/pixi-live2d-display/dist/index.min.js"></script>
|
17 |
+
<!-- Kalidokit -->
|
18 |
+
<script src="//cdn.jsdelivr.net/npm/[email protected]/dist/kalidokit.umd.js"></script>
|
19 |
+
</head>
|
20 |
+
|
21 |
+
<body>
|
22 |
+
|
23 |
+
<video id="my-video" autoplay></video>
|
24 |
+
|
25 |
+
<script>
|
26 |
+
let _debug = 0;
|
27 |
+
'use strict';
|
28 |
+
|
29 |
+
function speaks(input) {
|
30 |
+
var text = input;//document.getElementById("text").value;
|
31 |
+
var speech = new SpeechSynthesisUtterance(text);
|
32 |
+
speech.lang = "ja-JP"; // 日本語に設定
|
33 |
+
window.speechSynthesis.speak(speech);
|
34 |
+
}
|
35 |
+
|
36 |
+
function speak(input) {
|
37 |
+
var text = input;//document.getElementById("text").value;
|
38 |
+
var speech = new SpeechSynthesisUtterance(text);
|
39 |
+
|
40 |
+
// 声のリストから男性の声を選択
|
41 |
+
var voices = window.speechSynthesis.getVoices();
|
42 |
+
for (var i = 0; i < voices.length; i++) {
|
43 |
+
if (voices[i].name.includes("Google 日本語") || voices[i].name.includes("Male")) {
|
44 |
+
speech.voice = voices[i];
|
45 |
+
break;
|
46 |
+
}
|
47 |
+
}
|
48 |
+
|
49 |
+
// 音声の設定
|
50 |
+
speech.lang = "ja-JP"; // 日本語の設定
|
51 |
+
speech.pitch = 1; // ピッチ(声の高さ)
|
52 |
+
speech.rate = 1; // 読み上げ速度
|
53 |
+
speech.volume = 1; // 音量
|
54 |
+
|
55 |
+
// 音声の再生
|
56 |
+
window.speechSynthesis.speak(speech);
|
57 |
+
}
|
58 |
+
|
59 |
+
class MiiboAvatar {
|
60 |
+
constructor(config) {
|
61 |
+
this.container = config.container;
|
62 |
+
this.speechToTextOptions = config.option.speech_to_text;
|
63 |
+
this.miiboOptions = config.option.miibo;
|
64 |
+
this.didOptions = config.option.d_id;
|
65 |
+
this.initialize();
|
66 |
+
}
|
67 |
+
|
68 |
+
initialize() {
|
69 |
+
//alert("38");
|
70 |
+
const RTCPeerConnection = window.RTCPeerConnection || window.webkitRTCPeerConnection || window.mozRTCPeerConnection;
|
71 |
+
|
72 |
+
this.videoElement = document.getElementById(this.container);
|
73 |
+
this.videoElement.setAttribute('playsinline', '');
|
74 |
+
this.playIdleVideo();
|
75 |
+
|
76 |
+
this.createNewStream();
|
77 |
+
this.rec = new webkitSpeechRecognition()
|
78 |
+
|
79 |
+
this.speaching = false;
|
80 |
+
this.processing = false;
|
81 |
+
this.streams = [];
|
82 |
+
}
|
83 |
+
|
84 |
+
async createNewStream() {
|
85 |
+
if(_debug==1)
|
86 |
+
alert("53")
|
87 |
+
try {
|
88 |
+
this.stopAllStreams();
|
89 |
+
this.closePC();
|
90 |
+
|
91 |
+
let presenter = {"source_url": this.didOptions.presenter.image_url}
|
92 |
+
const sessionResponse = await this.fetchWithRetries(`https://api.d-id.com/${this.didOptions.service}/streams`, {
|
93 |
+
method: 'POST',
|
94 |
+
headers: {
|
95 |
+
Authorization: `Basic ${this.didOptions.key}`,
|
96 |
+
'Content-Type': 'application/json',
|
97 |
+
},
|
98 |
+
body: JSON.stringify(presenter),
|
99 |
+
});
|
100 |
+
|
101 |
+
const { id: newStreamId, offer, ice_servers: iceServers, session_id: newSessionId } = await sessionResponse.json();
|
102 |
+
this.streamId = newStreamId;
|
103 |
+
this.sessionId = newSessionId;
|
104 |
+
|
105 |
+
try {
|
106 |
+
this.sessionClientAnswer = await this.createPeerConnection(offer, iceServers);
|
107 |
+
} catch (e) {
|
108 |
+
console.log('Error creating peer connection:', e);
|
109 |
+
this.stopAllStreams();
|
110 |
+
this.closePC();
|
111 |
+
return;
|
112 |
+
}
|
113 |
+
|
114 |
+
const sdpResponse = await fetch(`https://api.d-id.com/${this.didOptions.service}/streams/${this.streamId}/sdp`, {
|
115 |
+
method: 'POST',
|
116 |
+
headers: {
|
117 |
+
Authorization: `Basic ${this.didOptions.key}`,
|
118 |
+
'Content-Type': 'application/json',
|
119 |
+
},
|
120 |
+
body: JSON.stringify({
|
121 |
+
answer: this.sessionClientAnswer,
|
122 |
+
session_id: this.sessionId,
|
123 |
+
}),
|
124 |
+
});
|
125 |
+
|
126 |
+
// Handle sdpResponse if needed
|
127 |
+
} catch (error) {
|
128 |
+
console.log('Error creating new stream:', error);
|
129 |
+
// Handle error
|
130 |
+
}
|
131 |
+
}
|
132 |
+
|
133 |
+
speechRecogInit() {
|
134 |
+
this.audioContext = new (window.AudioContext || window.webkitAudioContext)()
|
135 |
+
|
136 |
+
if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
|
137 |
+
navigator.mediaDevices.getUserMedia({audio: true}).then((stream) => {
|
138 |
+
var input = this.audioContext.createMediaStreamSource(stream)
|
139 |
+
this.audioContext.resume()
|
140 |
+
this.recorder = new Recorder(input)
|
141 |
+
})
|
142 |
+
}
|
143 |
+
}
|
144 |
+
|
145 |
+
|
146 |
+
startRecording() {
|
147 |
+
this.recorder && this.recorder.record();
|
148 |
+
}
|
149 |
+
|
150 |
+
stopRecording() {
|
151 |
+
this.playLoadingVideo();
|
152 |
+
this.recorder && this.recorder.stop();
|
153 |
+
this.audioRecognize();
|
154 |
+
this.recorder.clear();
|
155 |
+
}
|
156 |
+
|
157 |
+
audioRecognize() {
|
158 |
+
this.recorder && this.recorder.exportWAV((blob) => {
|
159 |
+
let reader = new FileReader()
|
160 |
+
reader.onload = () => {
|
161 |
+
let result = new Uint8Array(reader.result)
|
162 |
+
|
163 |
+
let data = {
|
164 |
+
"config": {
|
165 |
+
"encoding": "LINEAR16",
|
166 |
+
"languageCode": "ja-JP",
|
167 |
+
"alternativeLanguageCodes": ["en-US"],//,"cmn-CN","ko-KR"],
|
168 |
+
"audio_channel_count": 2
|
169 |
+
},
|
170 |
+
"audio": {
|
171 |
+
"content": this.arrayBufferToBase64(result)
|
172 |
+
}
|
173 |
+
}
|
174 |
+
fetch('https://speech.googleapis.com/v1/speech:recognize?key=' + this.speechToTextOptions.api_key, {
|
175 |
+
method: 'POST',
|
176 |
+
headers: {
|
177 |
+
'Content-Type': 'application/json; charset=utf-8'
|
178 |
+
},
|
179 |
+
body: JSON.stringify(data)
|
180 |
+
}).then((response) => {
|
181 |
+
return response.text()
|
182 |
+
}).then((text) => {
|
183 |
+
let result_json = JSON.parse(text)
|
184 |
+
text = result_json.results[0].alternatives[0].transcript;
|
185 |
+
this.languageCode = result_json.results[0].languageCode;
|
186 |
+
//alert("154 ask")
|
187 |
+
this.ask(text)
|
188 |
+
})
|
189 |
+
}
|
190 |
+
reader.readAsArrayBuffer(blob)
|
191 |
+
})
|
192 |
+
}
|
193 |
+
// Chrome Only
|
194 |
+
autoRecognizeMessage(message) {
|
195 |
+
|
196 |
+
//handleLiveComment("test",message)
|
197 |
+
//live2d
|
198 |
+
if(_debug==1)
|
199 |
+
alert("start word test")
|
200 |
+
this.rec.continuous = false
|
201 |
+
this.rec.interimResults = false
|
202 |
+
this.rec.lang = 'ja-JP'
|
203 |
+
this.processing = true
|
204 |
+
this.playLoadingVideo();
|
205 |
+
|
206 |
+
this.rec.stop()
|
207 |
+
//alert("172 start================================= "+message)
|
208 |
+
let transcript = message
|
209 |
+
//alert("transcript"+transcript)
|
210 |
+
this.ask(transcript);
|
211 |
+
//for (var i = e.resultIndex; i < e.results.length; i++) {
|
212 |
+
// if (!e.results[i].isFinal) continue
|
213 |
+
//
|
214 |
+
// const { transcript } = e.results[i][0]
|
215 |
+
// this.ask(transcript);
|
216 |
+
//}
|
217 |
+
//this.rec.onresult = (e) => {
|
218 |
+
|
219 |
+
//}
|
220 |
+
|
221 |
+
this.rec.onend = () => { this.autoRecognizeRestart() }
|
222 |
+
this.rec.start()
|
223 |
+
}
|
224 |
+
// Chrome Only
|
225 |
+
autoRecognize() {
|
226 |
+
this.rec.continuous = false
|
227 |
+
this.rec.interimResults = false
|
228 |
+
this.rec.lang = 'ja-JP'
|
229 |
+
|
230 |
+
this.rec.onresult = (e) => {
|
231 |
+
this.processing = true
|
232 |
+
this.playLoadingVideo();
|
233 |
+
|
234 |
+
this.rec.stop()
|
235 |
+
if(_debug==1)
|
236 |
+
alert("start")
|
237 |
+
|
238 |
+
for (var i = e.resultIndex; i < e.results.length; i++) {
|
239 |
+
if (!e.results[i].isFinal) continue
|
240 |
+
|
241 |
+
const { transcript } = e.results[i][0]
|
242 |
+
this.ask(transcript);
|
243 |
+
}
|
244 |
+
}
|
245 |
+
|
246 |
+
this.rec.onend = () => { this.autoRecognizeRestart() }
|
247 |
+
this.rec.start()
|
248 |
+
}
|
249 |
+
|
250 |
+
autoRecognizeRestart() {
|
251 |
+
if (this.processing) {
|
252 |
+
setTimeout(() => {this.autoRecognizeRestart()}, 1000)
|
253 |
+
} else {
|
254 |
+
this.rec.start()
|
255 |
+
}
|
256 |
+
}
|
257 |
+
|
258 |
+
arrayBufferToBase64(buffer) {
|
259 |
+
let binary = ''
|
260 |
+
let bytes = new Uint8Array(buffer);
|
261 |
+
let len = bytes.byteLength
|
262 |
+
for (let i = 0; i < len; i++) {
|
263 |
+
binary += String.fromCharCode(bytes[i])
|
264 |
+
}
|
265 |
+
return window.btoa(binary)
|
266 |
+
}
|
267 |
+
|
268 |
+
ask(message) {
|
269 |
+
//youtubeからの質問
|
270 |
+
if(_debug==1)
|
271 |
+
alert("203 ask ========================== "+message);
|
272 |
+
//ask(start);
|
273 |
+
this.getMiiboResponse(message);
|
274 |
+
}
|
275 |
+
|
276 |
+
async getMiiboResponse(utterance) {
|
277 |
+
if(_debug==1)
|
278 |
+
alert("209 getMiiboResponse"+utterance)
|
279 |
+
const params = {
|
280 |
+
api_key: this.miiboOptions.api_key,
|
281 |
+
agent_id: this.miiboOptions.agent_id,
|
282 |
+
uid: this.miiboOptions.user_id,
|
283 |
+
stream: true,
|
284 |
+
utterance: utterance
|
285 |
+
};
|
286 |
+
|
287 |
+
try {
|
288 |
+
const res = await fetch("https://api-mebo.dev/api", {
|
289 |
+
method: "POST",
|
290 |
+
headers: { "Content-Type": "application/json" },
|
291 |
+
body: JSON.stringify(params),
|
292 |
+
});
|
293 |
+
|
294 |
+
const reader = res.body.getReader();
|
295 |
+
const decoder = new TextDecoder();
|
296 |
+
let output = "";
|
297 |
+
let sentences = [];
|
298 |
+
let current_index = 0;
|
299 |
+
|
300 |
+
const read = async () => {
|
301 |
+
const { done, value } = await reader.read();
|
302 |
+
if (done) return;
|
303 |
+
|
304 |
+
let dataString = decoder.decode(value).split("\n").filter(x => x != "");
|
305 |
+
|
306 |
+
try {
|
307 |
+
let responseData = JSON.parse(dataString[dataString.length - 1]);
|
308 |
+
|
309 |
+
output = responseData.bestResponse.utterance.split("\n").filter(x => x.trim() != "").join("\n");
|
310 |
+
sentences = output.replace(/[。、\.\,\!\?!?]/,".").split(".")
|
311 |
+
if (this.didOptions.priority == "speed" && current_index == 0 && current_index + 1 < sentences.length) {
|
312 |
+
this.startTalk(sentences[current_index++])
|
313 |
+
}
|
314 |
+
} catch(e) {
|
315 |
+
console.log(e);
|
316 |
+
}
|
317 |
+
|
318 |
+
return read();
|
319 |
+
};
|
320 |
+
|
321 |
+
await read();
|
322 |
+
reader.releaseLock();
|
323 |
+
|
324 |
+
this.startTalk(sentences.slice(current_index).join("。"));
|
325 |
+
} catch(error) {
|
326 |
+
console.log("Error fetching AI response: ", error);
|
327 |
+
}
|
328 |
+
}
|
329 |
+
|
330 |
+
async startTalk(input) {
|
331 |
+
if(_debug==1)
|
332 |
+
alert("start streaming id==============="+input)
|
333 |
+
if (this.peerConnection?.signalingState === 'stable' || this.peerConnection?.iceConnectionState === 'connected') {
|
334 |
+
|
335 |
+
const gender = this.didOptions.presenter.gender;
|
336 |
+
let voice_id = this.didOptions.presenter.voice_id || "";
|
337 |
+
|
338 |
+
if (voice_id == "") {
|
339 |
+
switch (this.languageCode) {
|
340 |
+
case "en-us":
|
341 |
+
voice_id = gender == "male" ? "en-US-GuyNeural" : "en-US-AriaNeural"
|
342 |
+
break;
|
343 |
+
case "ko-kr":
|
344 |
+
voice_id = gender == "male" ? "ko-KR-InJoonNeural" : "ko-KR-YuJinNeural"
|
345 |
+
break;
|
346 |
+
case "cmn-CN":
|
347 |
+
voice_id = gender == "male" ? "zh-CN-YunjianNeural" : "zh-CN-XiaohanNeural"
|
348 |
+
break;
|
349 |
+
default:
|
350 |
+
voice_id = gender == "male" ? "ja-JP-KeitaNeural" : "ja-JP-NanamiNeural"
|
351 |
+
}
|
352 |
+
}
|
353 |
+
alert("start tolak 330")
|
354 |
+
speak(input)
|
355 |
+
const requestOptions = {
|
356 |
+
method: 'POST',
|
357 |
+
headers: {
|
358 |
+
Authorization: `Basic ${this.didOptions.key}`,
|
359 |
+
'Content-Type': 'application/json',
|
360 |
+
},
|
361 |
+
body: JSON.stringify({
|
362 |
+
script: {
|
363 |
+
type: "text",
|
364 |
+
subtitles: false,
|
365 |
+
provider: {
|
366 |
+
type: "microsoft",
|
367 |
+
voice_id: voice_id
|
368 |
+
},
|
369 |
+
ssml: false,
|
370 |
+
input: input
|
371 |
+
},
|
372 |
+
config: {
|
373 |
+
fluent: false,
|
374 |
+
pad_audio: 0,
|
375 |
+
align_driver: false,
|
376 |
+
stitch: false,
|
377 |
+
auto_match: false,
|
378 |
+
sharpen: false,
|
379 |
+
normalization_factor: 0
|
380 |
+
},
|
381 |
+
session_id: this.sessionId,
|
382 |
+
}),
|
383 |
+
};
|
384 |
+
|
385 |
+
if (this.didOptions.service === 'clips') {
|
386 |
+
requestOptions.body.background = { color: '#FFFFFF' };
|
387 |
+
}
|
388 |
+
|
389 |
+
try {
|
390 |
+
//const playResponse = await this.fetchWithRetries(`https://api.d-id.com/${this.didOptions.service}/streams/${this.streamId}`, requestOptions);
|
391 |
+
//
|
392 |
+
startTyping({
|
393 |
+
el: "#aiResponseUtterance",
|
394 |
+
string: input,
|
395 |
+
speed: 50
|
396 |
+
});
|
397 |
+
|
398 |
+
// Handle response if needed
|
399 |
+
} catch (error) {
|
400 |
+
console.error('Error starting talk:', error);
|
401 |
+
// Handle error
|
402 |
+
}
|
403 |
+
}
|
404 |
+
}
|
405 |
+
|
406 |
+
async destoryTalk(input) {
|
407 |
+
try {
|
408 |
+
await fetch(`https://api.d-id.com/${this.didOptions.service}/streams/${this.streamId}`, {
|
409 |
+
method: 'DELETE',
|
410 |
+
headers: {
|
411 |
+
Authorization: `Basic ${this.didOptions.key}`,
|
412 |
+
'Content-Type': 'application/json',
|
413 |
+
},
|
414 |
+
body: JSON.stringify({ session_id: this.sessionId }),
|
415 |
+
});
|
416 |
+
|
417 |
+
await this.stopAllStreams();
|
418 |
+
await this.closePC();
|
419 |
+
} catch (error) {
|
420 |
+
console.error('Error destroying talk:', error);
|
421 |
+
// Handle error
|
422 |
+
}
|
423 |
+
}
|
424 |
+
|
425 |
+
onIceCandidate(event) {
|
426 |
+
if (event.candidate) {
|
427 |
+
const { candidate, sdpMid, sdpMLineIndex } = event.candidate;
|
428 |
+
|
429 |
+
fetch(`https://api.d-id.com/${this.didOptions.service}/streams/${this.streamId}/ice`, {
|
430 |
+
method: 'POST',
|
431 |
+
headers: {
|
432 |
+
Authorization: `Basic ${this.didOptions.key}`,
|
433 |
+
'Content-Type': 'application/json',
|
434 |
+
},
|
435 |
+
body: JSON.stringify({
|
436 |
+
candidate,
|
437 |
+
sdpMid,
|
438 |
+
sdpMLineIndex,
|
439 |
+
session_id: this.sessionId,
|
440 |
+
}),
|
441 |
+
}).catch((error) => {
|
442 |
+
console.error('Error sending ICE candidate:', error);
|
443 |
+
// Handle error
|
444 |
+
});
|
445 |
+
}
|
446 |
+
}
|
447 |
+
|
448 |
+
onIceConnectionStateChange() {
|
449 |
+
if (this.peerConnection.iceConnectionState === 'failed' || this.peerConnection.iceConnectionState === 'closed') {
|
450 |
+
this.stopAllStreams();
|
451 |
+
this.closePC();
|
452 |
+
}
|
453 |
+
}
|
454 |
+
|
455 |
+
onTrack(event) {
|
456 |
+
if (!event.track || !event.streams || event.streams.length === 0) return;
|
457 |
+
|
458 |
+
this.statsIntervalId = setInterval(async () => {
|
459 |
+
const stats = await this.peerConnection.getStats(event.track);
|
460 |
+
stats.forEach((report) => {
|
461 |
+
if (report.type === 'inbound-rtp' && report.mediaType === 'video') {
|
462 |
+
const videoStatusChanged = this.videoIsPlaying !== report.bytesReceived > this.lastBytesReceived;
|
463 |
+
|
464 |
+
if (videoStatusChanged) {
|
465 |
+
this.videoIsPlaying = report.bytesReceived > this.lastBytesReceived;
|
466 |
+
this.onVideoStatusChange(this.videoIsPlaying, event.streams[0]);
|
467 |
+
}
|
468 |
+
this.lastBytesReceived = report.bytesReceived;
|
469 |
+
}
|
470 |
+
});
|
471 |
+
}, 100);
|
472 |
+
}
|
473 |
+
|
474 |
+
onVideoStatusChange(videoIsPlaying, stream) {
|
475 |
+
let status;
|
476 |
+
if (videoIsPlaying) {
|
477 |
+
status = 'streaming';
|
478 |
+
const remoteStream = stream;
|
479 |
+
this.streams.push(remoteStream);
|
480 |
+
this.checkSpeaching();
|
481 |
+
} else {
|
482 |
+
status = 'empty';
|
483 |
+
this.speaching = false;
|
484 |
+
this.processing = false;
|
485 |
+
this.playIdleVideo();
|
486 |
+
}
|
487 |
+
}
|
488 |
+
|
489 |
+
checkSpeaching() {
|
490 |
+
if (this.speaching) {
|
491 |
+
setTimeout(() => {this.checkSpeaching()}, 20)
|
492 |
+
} else {
|
493 |
+
this.setVideoElement(this.streams.shift());
|
494 |
+
}
|
495 |
+
}
|
496 |
+
|
497 |
+
async createPeerConnection(offer, iceServers) {
|
498 |
+
if (!this.peerConnection) {
|
499 |
+
this.peerConnection = new RTCPeerConnection({ iceServers });
|
500 |
+
this.peerConnection.addEventListener('icecandidate', this.onIceCandidate.bind(this), true);
|
501 |
+
this.peerConnection.addEventListener('iceconnectionstatechange', this.onIceConnectionStateChange.bind(this), true);
|
502 |
+
this.peerConnection.addEventListener('track', this.onTrack.bind(this), true);
|
503 |
+
}
|
504 |
+
|
505 |
+
await this.peerConnection.setRemoteDescription(offer);
|
506 |
+
const sessionClientAnswer = await this.peerConnection.createAnswer();
|
507 |
+
await this.peerConnection.setLocalDescription(sessionClientAnswer);
|
508 |
+
return sessionClientAnswer;
|
509 |
+
}
|
510 |
+
|
511 |
+
|
512 |
+
setVideoElement(stream) {
|
513 |
+
if (!stream) return;
|
514 |
+
this.videoElement.srcObject = stream;
|
515 |
+
this.videoElement.loop = false;
|
516 |
+
|
517 |
+
// safari hotfix
|
518 |
+
if (this.videoElement.paused) {
|
519 |
+
this.videoElement
|
520 |
+
.play()
|
521 |
+
.then((_) => {})
|
522 |
+
.catch((e) => {});
|
523 |
+
}
|
524 |
+
}
|
525 |
+
|
526 |
+
playIdleVideo() {
|
527 |
+
this.videoElement.srcObject = undefined;
|
528 |
+
this.videoElement.src = this.didOptions.presenter.idle_movie;
|
529 |
+
this.videoElement.loop = true;
|
530 |
+
}
|
531 |
+
|
532 |
+
playLoadingVideo() {
|
533 |
+
this.videoElement.srcObject = undefined;
|
534 |
+
this.videoElement.src = this.didOptions.presenter.loading_movie;
|
535 |
+
this.videoElement.loop = false;
|
536 |
+
}
|
537 |
+
|
538 |
+
stopAllStreams() {
|
539 |
+
if (this.videoElement.srcObject) {
|
540 |
+
this.videoElement.srcObject.getTracks().forEach((track) => track.stop());
|
541 |
+
this.videoElement.srcObject = null;
|
542 |
+
}
|
543 |
+
}
|
544 |
+
|
545 |
+
closePC(pc = this.peerConnection) {
|
546 |
+
if (!pc) return;
|
547 |
+
pc.close();
|
548 |
+
pc.removeEventListener('icecandidate', this.onIceCandidate.bind(this), true);
|
549 |
+
pc.removeEventListener('iceconnectionstatechange', this.onIceConnectionStateChange.bind(this), true);
|
550 |
+
pc.removeEventListener('track', this.onTrack.bind(this), true);
|
551 |
+
clearInterval(this.statsIntervalId);
|
552 |
+
if (pc === this.peerConnection) {
|
553 |
+
this.peerConnection = null;
|
554 |
+
}
|
555 |
+
}
|
556 |
+
|
557 |
+
async fetchWithRetries(url, options, retries = 1) {
|
558 |
+
const maxRetryCount = 3;
|
559 |
+
const maxDelaySec = 4;
|
560 |
+
try {
|
561 |
+
return await fetch(url, options);
|
562 |
+
} catch (err) {
|
563 |
+
if (retries <= maxRetryCount) {
|
564 |
+
const delay = Math.min(Math.pow(2, retries) / 4 + Math.random(), maxDelaySec) * 1000;
|
565 |
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
566 |
+
return this.fetchWithRetries(url, options, retries + 1);
|
567 |
+
} else {
|
568 |
+
throw new Error(`Max retries exceeded. error: ${err}`);
|
569 |
+
}
|
570 |
+
}
|
571 |
+
}
|
572 |
+
}
|
573 |
+
|
574 |
+
const miiboAvatar = new MiiboAvatar({
|
575 |
+
container: "my-video",
|
576 |
+
option: {
|
577 |
+
miibo: {
|
578 |
+
api_key: "9206bb9d-be50-46b9-8ba2-4441fee3fe7b190cb511a0c78",
|
579 |
+
agent_id: "3cfb2749-a1fa-4a5b-a2c8-eb17aee77808190cb500dd1357",
|
580 |
+
user_id: "user",
|
581 |
+
},
|
582 |
+
d_id: {
|
583 |
+
key: "bWl5YXRha2VuOTk3QGdtYWlsLmNvbQ:m3WhwvxpDS6nrr4YdW9wX",
|
584 |
+
service: "talks",
|
585 |
+
priority: "cost",
|
586 |
+
presenter:{
|
587 |
+
gender: "male",
|
588 |
+
image_url: "https://kenken999-fastapi-django-main-live.hf.space/static/sugi.jpg",
|
589 |
+
idle_movie: "https://kenken999-fastapi-django-main-live.hf.space/static/miiboi.mp4",
|
590 |
+
loading_movie: "https://kenken999-fastapi-django-main-live.hf.space/static/miibo.mp4",
|
591 |
+
}
|
592 |
+
}
|
593 |
+
}
|
594 |
+
})
|
595 |
+
|
596 |
+
miiboAvatar.autoRecognize();
|
597 |
+
|
598 |
+
const onClickSends = () => {
|
599 |
+
//alert("start send ============================ ")
|
600 |
+
let utterance = document.querySelector("#utterances");
|
601 |
+
//alert(utterances.value)
|
602 |
+
|
603 |
+
miiboAvatar.autoRecognizeMessage(utterance.value)
|
604 |
+
//handleLiveComment(utterance.value, '匿名');
|
605 |
+
//onClickSend
|
606 |
+
utterances.value = "";
|
607 |
+
}
|
608 |
+
|
609 |
+
</script>
|
610 |
+
<canvas id="my-live2d"></canvas>
|
611 |
+
<script src="live2d.js"></script>
|
612 |
+
<script src="index.js"></script>
|
613 |
+
<div id="vtubers">
|
614 |
+
<!--
|
615 |
+
<iframe width="auto" height="315" src="https://www.youtube.com/embed/9BSgttQDyOw?si=ax2F4oAi2h9f9eGT" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
|
616 |
+
/-->
|
617 |
+
<!--<img id="charaImg" src="chara.png" width="auto" height="400" />/-->
|
618 |
+
</div>
|
619 |
+
<div id="aiResponse" class="aiResponseBox">
|
620 |
+
<p class="ai-response" id="aiResponseUtterance"></p>
|
621 |
+
</div>
|
622 |
+
<div class="bottomBox">
|
623 |
+
<p id="userComment"></p>
|
624 |
+
<button id="startLiveButton" onclick="startLive();">LIVE開始</button>
|
625 |
+
<div id="submit_forms">
|
626 |
+
<input type="text" id="utterances" />
|
627 |
+
<button id="sendButton" onclick="onClickSends();">送信</button>
|
628 |
+
</div>
|
629 |
+
</div>
|
630 |
+
</body>
|
631 |
+
|
632 |
+
</html>
|
staticfiles/livetest.html
CHANGED
@@ -387,8 +387,8 @@ async startTalk(input) {
|
|
387 |
}
|
388 |
|
389 |
try {
|
390 |
-
|
391 |
-
|
392 |
startTyping({
|
393 |
el: "#aiResponseUtterance",
|
394 |
string: input,
|
|
|
387 |
}
|
388 |
|
389 |
try {
|
390 |
+
const playResponse = await this.fetchWithRetries(`https://api.d-id.com/${this.didOptions.service}/streams/${this.streamId}`, requestOptions);
|
391 |
+
|
392 |
startTyping({
|
393 |
el: "#aiResponseUtterance",
|
394 |
string: input,
|