从小白到入门webrtc音视频通话
0. 写在前面
先会骑车，再研究为什么这么骑，才是我认为学习技术的思路，底部付了demo例子，根据例子上面的介绍即可运行。
1. 音视频通话要用到的技术简介
- websocket
- 介绍：1. 服务器可以向浏览器推送信息；2. 一次握手成功，可持续互相发送信息
- 在音视频通话钟的作用：1. 作为音视频两个通话终端的桥梁，传递彼此上下线、网络环境等消息，因此他们都叫websocket为“信令服务器”
- coturn
- 介绍：1. 包含stun服务和turn服务，stun可实现两个终端点对点语音通话；turn服务在无法点对点通话时，用作中转音视频流。
- webrtc
- 介绍：1. 开源项目；2. 用于音视频实时互动、游戏、即时通讯、文件传输。
2. webrtc音视频通话开发思路
2.1. webrtc调用时序图
- 下图简化了B客户端创建PeerConnection，具体流程要看下面“调用时序图介绍”
2.2. 调用时序图介绍
- 上图名词介绍
- client A：客户端A
- Stun Server：穿透服务器，也就是coturn服务器中的Stun
- Signal Server：信令服务器，也就是web socket搭建的服务器
- client B：客户端B
- PeerConnection（WebRtc的接口）
- 流程介绍
- A客户端先发送信息到信令服务器，信令服务器存储A客户端信息，等待其他客户端加入。
- B客户端再发送信息到信令服务器，信令服务器存储B客户端信息，并告知A已有新客户端加入。
- A客户端创建 PeerConnection（WebRtc的接口），用于获取本地的网络信息、以及保存对方的网络信息、传递音视频流、监听通话过程状态。（B客户端后面也需要创建PeerConnection）
- AddStreams：A客户端添加本地音视频流到PeerConnection
- CreateOffer：A客户端创建Offer并发送给信令服务器，由信令服务器转给B客户端。Offer中包含本地的网络信息（SDP）。
- CreateAnswer：B客户端收到Offer后，创建放到自己的PeerConnection中，并获取自己的网络信息（SDP），通过发送给信令服务器，由信令服务器转发给A客户端。
- 上面步骤进行完毕后，开始通过信令服务器（coturn），双方客户端获取自己的地址作为候选人“candidate”，然后通过websocket发送给对方。彼此拿到候选地址后，互相进行访问测试，建立链接。
- OnAddStream：获取对方音视频流，PeerConnection有ontrack监听器能拿到对方视频流数据。
2. 搭建WebSocket服务器
看例子中代码，使用nodejs启动
3. 搭建Coturn音视频穿透服务器
公司内网虚拟机中穿透服务器Coturn的搭建
4. 遇到的问题
后面再慢慢补吧，问题有点多
5. 例子
- 客户端代码使用html+js编写
- WebSocket代码使用js编写使用nodejs运行
- android端代码请下载：WebRtcAndroidDemo
5.1 客户端代码
- 引入adapter-latest.js文件，此文件如果过期了就自己百度找找吧。
- 将ws://192.168.1.60:9001/ws改为自己websocket服务所在电脑的ip地址，在本地启动则是本机地址
- 将iceServers中的ip改为coturn服务器所在ip地址
<html>
<head>
<title>Voice WebRTC demotitle>
head>
<h1>WebRTC demo 1v1h1>
<div id="buttons">
<input id="zero-roomId" type="text" placeholder="请输入房间ID" maxlength="40"/>
<button id="joinBtn" type="button">加入button>
<button id="leaveBtn" type="button">离开button>
div>
<div id="videos">
<video id="localVideo" autoplay muted playsinline>本地窗口video>
<video id="remoteVideo" autoplay playsinline>远端窗口video>
div>
<script src="js/main.js">script>
<script src="js/adapter-latest.js">script>
html>
'use strict';
// join 主动加入房间
// leave 主动离开房间
// new-peer 有人加入房间，通知已经在房间的人
// peer-leave 有人离开房间，通知已经在房间的人
// offer 发送offer给对端peer
// answer发送offer给对端peer
// candidate 发送candidate给对端peer
const SIGNAL_TYPE_JOIN = "join";
const SIGNAL_TYPE_RESP_JOIN = "resp-join"; // 告知加入者对方是谁
const SIGNAL_TYPE_LEAVE = "leave";
const SIGNAL_TYPE_NEW_PEER = "new-peer";
const SIGNAL_TYPE_PEER_LEAVE = "peer-leave";
const SIGNAL_TYPE_OFFER = "offer";
const SIGNAL_TYPE_ANSWER = "answer";
const SIGNAL_TYPE_CANDIDATE = "candidate";
var localUserId = Math.random().toString(36).substr(2); // 本地uid
var remoteUserId = -1; // 对端
var roomId = 0;
var localVideo = document.querySelector('#localVideo');
var remoteVideo = document.querySelector('#remoteVideo');
var localStream = null;
var remoteStream = null;
var pc = null;
var zeroRTCEngine;
function handleIceCandidate(event) {
console.info("handleIceCandidate");
if (event.candidate) {
var candidateJson = {
'label': event.candidate.sdpMLineIndex,
'id': event.candidate.sdpMid,
'candidate': event.candidate.candidate
};
var jsonMsg = {
'cmd': SIGNAL_TYPE_CANDIDATE,
'roomId': roomId,
'uid': localUserId,
'remoteUid':remoteUserId,
'msg': JSON.stringify(candidateJson)
};
var message = JSON.stringify(jsonMsg);
zeroRTCEngine.sendMessage(message);
console.info("handleIceCandidate message: " + message);
console.info("send candidate message");
} else {
console.warn("End of candidates");
}
}
function handleRemoteStreamAdd(event) {
console.info("handleRemoteStreamAdd");
remoteStream = event.streams[0];
// 视频轨道
// let videoTracks = remoteStream.getVideoTracks()
// 音频轨道
// let audioTracks = remoteStream.getAudioTracks()
remoteVideo.srcObject = remoteStream;
}
function handleConnectionStateChange() {
if(pc != null) {
console.info("ConnectionState -> " + pc.connectionState);
}
}
function handleIceConnectionStateChange() {
if(pc != null) {
console.info("IceConnectionState -> " + pc.iceConnectionState);
}
}
function createPeerConnection() {
var defaultConfiguration = {
bundlePolicy: "max-bundle",
rtcpMuxPolicy: "require",
iceTransportPolicy:"all",//relay 或者 all
// 修改ice数组测试效果，需要进行封装
iceServers: [
{
"urls": [
"turn:192.168.1.173:3478?transport=udp",
"turn:192.168.1.173:3478?transport=tcp" // 可以插入多个进行备选
],
"username": "lqf",
"credential": "123456"
},
{
"urls": [
"stun:192.168.1.173:3478"
]
}
]
};
pc = new RTCPeerConnection(defaultConfiguration); // 音视频通话的核心类
pc.onicecandidate = handleIceCandidate;
pc.ontrack = handleRemoteStreamAdd;
pc.onconnectionstatechange = handleConnectionStateChange;
pc.oniceconnectionstatechange = handleIceConnectionStateChange
localStream.getTracks().forEach((track) => pc.addTrack(track, localStream)); // 把本地流设置给RTCPeerConnection
}
function createOfferAndSendMessage(session) {
pc.setLocalDescription(session)
.then(function () {
var jsonMsg = {
'cmd': 'offer',
'roomId': roomId,
'uid': localUserId,
'remoteUid': remoteUserId,
'msg': JSON.stringify(session)
};
var message = JSON.stringify(jsonMsg);
zeroRTCEngine.sendMessage(message);
// console.info("send offer message: " + message);
console.info("send offer message");
})
.catch(function (error) {
console.error("offer setLocalDescription failed: " + error);
});
}
function handleCreateOfferError(error) {
console.error("handleCreateOfferError: " + error);
}
function createAnswerAndSendMessage(session) {
pc.setLocalDescription(session)
.then(function () {
var jsonMsg = {
'cmd': 'answer',
'roomId': roomId,
'uid': localUserId,
'remoteUid': remoteUserId,
'msg': JSON.stringify(session)
};
var message = JSON.stringify(jsonMsg);
zeroRTCEngine.sendMessage(message);
// console.info("send answer message: " + message);
console.info("send answer message");
})
.catch(function (error) {
console.error("answer setLocalDescription failed: " + error);
});
}
function handleCreateAnswerError(error) {
console.error("handleCreateAnswerError: " + error);
}
var ZeroRTCEngine = function (wsUrl) {
this.init(wsUrl);
zeroRTCEngine = this;
return this;
}
ZeroRTCEngine.prototype.init = function (wsUrl) {
// 设置websocket url
this.wsUrl = wsUrl;
/** websocket对象 */
this.signaling = null;
}
ZeroRTCEngine.prototype.createWebsocket = function () {
zeroRTCEngine = this;
zeroRTCEngine.signaling = new WebSocket(this.wsUrl);
zeroRTCEngine.signaling.onopen = function () {
zeroRTCEngine.onOpen();
}
zeroRTCEngine.signaling.onmessage = function (ev) {
zeroRTCEngine.onMessage(ev);
}
zeroRTCEngine.signaling.onerror = function (ev) {
zeroRTCEngine.onError(ev);
}
zeroRTCEngine.signaling.onclose = function (ev) {
zeroRTCEngine.onClose(ev);
}
}
ZeroRTCEngine.prototype.onOpen = function () {
console.log("websocket打开");
}
ZeroRTCEngine.prototype.onMessage = function (event) {
console.log("websocket收到信息: " + event.data);
var jsonMsg = null;
try {
jsonMsg = JSON.parse(event.data);
} catch(e) {
console.warn("onMessage parse Json failed:" + e);
return;
}
switch (jsonMsg.cmd) {
case SIGNAL_TYPE_NEW_PEER:
handleRemoteNewPeer(jsonMsg);
break;
case SIGNAL_TYPE_RESP_JOIN:
handleResponseJoin(jsonMsg);
break;
case SIGNAL_TYPE_PEER_LEAVE:
handleRemotePeerLeave(jsonMsg);
break;
case SIGNAL_TYPE_OFFER:
handleRemoteOffer(jsonMsg);
break;
case SIGNAL_TYPE_ANSWER:
handleRemoteAnswer(jsonMsg);
break;
case SIGNAL_TYPE_CANDIDATE:
handleRemoteCandidate(jsonMsg);
break;
}
}
ZeroRTCEngine.prototype.onError = function (event) {
console.log("onError: " + event.data);
}
ZeroRTCEngine.prototype.onClose = function (event) {
console.log("onClose -> code: " + event.code + ", reason:" + EventTarget.reason);
}
ZeroRTCEngine.prototype.sendMessage = function (message) {
this.signaling.send(message);
}
function handleResponseJoin(message) {
console.info("handleResponseJoin, remoteUid: " + message.remoteUid);
remoteUserId = message.remoteUid;
// doOffer();
}
function handleRemotePeerLeave(message) {
console.info("handleRemotePeerLeave, remoteUid: " + message.remoteUid);
remoteVideo.srcObject = null;
if(pc != null) {
pc.close();
pc = null;
}
}
function handleRemoteNewPeer(message) {
console.info("处理远端新加入链接，并发送offer, remoteUid: " + message.remoteUid);
remoteUserId = message.remoteUid;
doOffer();
}
function handleRemoteOffer(message) {
console.info("handleRemoteOffer");
if(pc == null) {
createPeerConnection();
}
var desc = JSON.parse(message.msg);
pc.setRemoteDescription(desc);
doAnswer();
}
function handleRemoteAnswer(message) {
console.info("handleRemoteAnswer");
var desc = JSON.parse(message.msg);
pc.setRemoteDescription(desc);
}
function handleRemoteCandidate(message) {
console.info("handleRemoteCandidate");
var jsonMsg = message.msg;
if(typeof message.msg === "string"){
jsonMsg = JSON.parse(message.msg);
}
var candidateMsg = {
'sdpMLineIndex': jsonMsg.label,
'sdpMid': jsonMsg.id,
'candidate': jsonMsg.candidate
};
var candidate = new RTCIceCandidate(candidateMsg);
pc.addIceCandidate(candidate).catch(e => {
console.error("addIceCandidate failed:" + e.name);
});
}
function doOffer() {
// 创建RTCPeerConnection
if (pc == null) {
createPeerConnection();
}
// let options = {offerToReceiveVideo:true}
// pc.createOffer(options).then(createOfferAndSendMessage).catch(handleCreateOfferError);
pc.createOffer().then(createOfferAndSendMessage).catch(handleCreateOfferError);
}
function doAnswer() {
pc.createAnswer().then(createAnswerAndSendMessage).catch(handleCreateAnswerError);
}
function doJoin(roomId) {
var jsonMsg = {
'cmd': 'join',
'roomId': roomId,
'uid': localUserId,
};
var message = JSON.stringify(jsonMsg);
zeroRTCEngine.sendMessage(message);
console.info("doJoin message: " + message);
}
function doLeave() {
var jsonMsg = {
'cmd': 'leave',
'roomId': roomId,
'uid': localUserId,
};
var message = JSON.stringify(jsonMsg);
zeroRTCEngine.sendMessage(message);
console.info("doLeave message: " + message);
hangup();
}
function hangup() {
localVideo.srcObject = null; // 0.关闭自己的本地显示
remoteVideo.srcObject = null; // 1.不显示对方
closeLocalStream(); // 2. 关闭本地流
if(pc != null) {
pc.close(); // 3.关闭RTCPeerConnection
pc = null;
}
}
function closeLocalStream() {
if(localStream != null) {
localStream.getTracks().forEach((track) => {
track.stop();
});
}
}
function openLocalStream(stream) {
console.log('Open local stream');
doJoin(roomId);
localVideo.srcObject = stream; // 显示画面
localStream = stream; // 保存本地流的句柄
}
function initLocalStream() {
navigator.mediaDevices.getUserMedia({
audio: true,
video: true
})
.then(openLocalStream)
.catch(function (e) {
alert("getUserMedia() error: " + e.name);
});
}
// zeroRTCEngine = new ZeroRTCEngine("wss://192.168.1.60:80/ws");
zeroRTCEngine = new ZeroRTCEngine("ws://192.168.1.60:9001/ws");
zeroRTCEngine.createWebsocket();
document.getElementById('joinBtn').onclick = function () {
roomId = document.getElementById('zero-roomId').value;
if (roomId == "" || roomId == "请输入房间ID") {
alert("请输入房间ID");
return;
}
console.log("第一步：加入按钮被点击, roomId: " + roomId);
// 初始化本地码流
initLocalStream();
}
document.getElementById('leaveBtn').onclick = function () {
console.log("离开按钮被点击");
doLeave();
}
5.2. 编写websocket服务
- 使用nodejs启动
var ws = require("nodejs-websocket")
var prort = 9001;
// join 主动加入房间
// leave 主动离开房间
// new-peer 有人加入房间，通知已经在房间的人
// peer-leave 有人离开房间，通知已经在房间的人
// offer 发送offer给对端peer
// answer发送offer给对端peer
// candidate 发送candidate给对端peer
const SIGNAL_TYPE_JOIN = "join";
const SIGNAL_TYPE_RESP_JOIN = "resp-join"; // 告知加入者对方是谁
const SIGNAL_TYPE_LEAVE = "leave";
const SIGNAL_TYPE_NEW_PEER = "new-peer";
const SIGNAL_TYPE_PEER_LEAVE = "peer-leave";
const SIGNAL_TYPE_OFFER = "offer";
const SIGNAL_TYPE_ANSWER = "answer";
const SIGNAL_TYPE_CANDIDATE = "candidate";
/** ----- ZeroRTCMap ----- */
var ZeroRTCMap = function () {
this._entrys = new Array();
this.put = function (key, value) {
if (key == null || key == undefined) {
return;
}
var index = this._getIndex(key);
if (index == -1) {
var entry = new Object();
entry.key = key;
entry.value = value;
this._entrys[this._entrys.length] = entry;
} else {
this._entrys[index].value = value;
}
};
this.get = function (key) {
var index = this._getIndex(key);
return (index != -1) ? this._entrys[index].value : null;
};
this.remove = function (key) {
var index = this._getIndex(key);
if (index != -1) {
this._entrys.splice(index, 1);
}
};
this.clear = function () {
this._entrys.length = 0;
};
this.contains = function (key) {
var index = this._getIndex(key);
return (index != -1) ? true : false;
};
this.size = function () {
return this._entrys.length;
};
this.getEntrys = function () {
return this._entrys;
};
this._getIndex = function (key) {
if (key == null || key == undefined) {
return -1;
}
var _length = this._entrys.length;
for (var i = 0; i < _length; i++) {
var entry = this._entrys[i];
if (entry == null || entry == undefined) {
continue;
}
if (entry.key === key) {// equal
return i;
}
}
return -1;
};
}
var roomTableMap = new ZeroRTCMap();
function Client(uid, conn, roomId) {
this.uid = uid; // 用户所属的id
this.conn = conn; // uid对应的websocket连接
this.roomId = roomId;
}
function handleJoin(message, conn) {
var roomId = message.roomId;
var uid = message.uid;
console.info("uid: " + uid + "try to join room " + roomId);
var roomMap = roomTableMap.get(roomId);
if (roomMap == null) {
roomMap = new ZeroRTCMap(); // 如果房间没有创建，则新创建一个房间
roomTableMap.put(roomId, roomMap);
}
if(roomMap.size() >= 2) {
console.error("roomId:" + roomId + " 已经有两人存在，请使用其他房间");
// 加信令通知客户端，房间已满
return null;
}
var client = new Client(uid, conn, roomId);
roomMap.put(uid, client);
if(roomMap.size() > 1) {
// 房间里面已经有人了，加上新进来的人，那就是>=2了，所以要通知对方
var clients = roomMap.getEntrys();
for(var i in clients) {
var remoteUid = clients[i].key;
if (remoteUid != uid) {
var jsonMsg = {
'cmd': SIGNAL_TYPE_NEW_PEER,
'remoteUid': uid
};
var msg = JSON.stringify(jsonMsg);
var remoteClient =roomMap.get(remoteUid);
console.info("new-peer: " + msg);
remoteClient.conn.sendText(msg);
jsonMsg = {
'cmd':SIGNAL_TYPE_RESP_JOIN,
'remoteUid': remoteUid
};
msg = JSON.stringify(jsonMsg);
console.info("resp-join: " + msg);
conn.sendText(msg);
}
}
}
return client;
}
function handleLeave(message) {
var roomId = message.roomId;
var uid = message.uid;
var roomMap = roomTableMap.get(roomId);
if (roomMap == null) {
console.error("handleLeave can't find then roomId " + roomId);
return;
}
if (!roomMap.contains(uid)) {
console.info("uid: " + uid +" have leave roomId " + roomId);
return;
}
console.info("uid: " + uid + " leave room " + roomId);
roomMap.remove(uid); // 删除发送者
if(roomMap.size() >= 1) {
var clients = roomMap.getEntrys();
for(var i in clients) {
var jsonMsg = {
'cmd': 'peer-leave',
'remoteUid': uid // 谁离开就填写谁
};
var msg = JSON.stringify(jsonMsg);
var remoteUid = clients[i].key;
var remoteClient = roomMap.get(remoteUid);
if(remoteClient) {
console.info("notify peer:" + remoteClient.uid + ", uid:" + uid + " leave");
remoteClient.conn.sendText(msg);
}
}
}
}
function handleForceLeave(client) {
var roomId = client.roomId;
var uid = client.uid;
// 1. 先查找房间号
var roomMap = roomTableMap.get(roomId);
if (roomMap == null) {
console.warn("handleForceLeave can't find then roomId " + roomId);
return;
}
// 2. 判别uid是否在房间
if (!roomMap.contains(uid)) {
console.info("uid: " + uid +" have leave roomId " + roomId);
return;
}
// 3.走到这一步，说明客户端没有正常离开，所以我们要执行离开程序
console.info("uid: " + uid + " force leave room " + roomId);
roomMap.remove(uid); // 删除发送者
if(roomMap.size() >= 1) {
var clients = roomMap.getEntrys();
for(var i in clients) {
var jsonMsg = {
'cmd': 'peer-leave',
'remoteUid': uid // 谁离开就填写谁
};
var msg = JSON.stringify(jsonMsg);
var remoteUid = clients[i].key;
var remoteClient = roomMap.get(remoteUid);
if(remoteClient) {
console.info("notify peer:" + remoteClient.uid + ", uid:" + uid + " leave");
remoteClient.conn.sendText(msg);
}
}
}
}
function handleOffer(message) {
var roomId = message.roomId;
var uid = message.uid;
var remoteUid = message.remoteUid;
console.info("handleOffer uid: " + uid + "transfer offer to remoteUid" + remoteUid);
var roomMap = roomTableMap.get(roomId);
if (roomMap == null) {
console.error("handleOffer can't find then roomId " + roomId);
return;
}
if(roomMap.get(uid) == null) {
console.error("handleOffer can't find then uid " + uid);
return;
}
var remoteClient = roomMap.get(remoteUid);
if(remoteClient) {
var msg = JSON.stringify(message);
remoteClient.conn.sendText(msg); //把数据发送给对方
} else {
console.error("can't find remoteUid： " + remoteUid);
}
}
function handleAnswer(message) {
var roomId = message.roomId;
var uid = message.uid;
var remoteUid = message.remoteUid;
console.info("handleAnswer uid: " + uid + "transfer answer to remoteUid" + remoteUid);
var roomMap = roomTableMap.get(roomId);
if (roomMap == null) {
console.error("handleAnswer can't find then roomId " + roomId);
return;
}
if(roomMap.get(uid) == null) {
console.error("handleAnswer can't find then uid " + uid);
return;
}
var remoteClient = roomMap.get(remoteUid);
if(remoteClient) {
var msg = JSON.stringify(message);
remoteClient.conn.sendText(msg);
} else {
console.error("can't find remoteUid： " + remoteUid);
}
}
function handleCandidate(message) {
var roomId = message.roomId;
var uid = message.uid;
var remoteUid = message.remoteUid;
console.info("处理Candidate uid: " + uid + "transfer candidate to remoteUid" + remoteUid);
var roomMap = roomTableMap.get(roomId);
if (roomMap == null) {
console.error("handleCandidate can't find then roomId " + roomId);
return;
}
if(roomMap.get(uid) == null) {
console.error("handleCandidate can't find then uid " + uid);
return;
}
var remoteClient = roomMap.get(remoteUid);
if(remoteClient) {
var msg = JSON.stringify(message);
remoteClient.conn.sendText(msg);
} else {
console.error("can't find remoteUid： " + remoteUid);
}
}
// 创建监听9001端口webSocket服务
var server = ws.createServer(function(conn){
console.log("创建一个新的连接--------")
conn.client = null; // 对应的客户端信息
// conn.sendText("我收到你的连接了....");
conn.on("text", function(str) {
// console.info("recv msg:" + str);
var jsonMsg = JSON.parse(str);
switch (jsonMsg.cmd) {
case SIGNAL_TYPE_JOIN:
conn.client = handleJoin(jsonMsg, conn);
break;
case SIGNAL_TYPE_LEAVE:
handleLeave(jsonMsg);
break;
case SIGNAL_TYPE_OFFER:
handleOffer(jsonMsg);
break;
case SIGNAL_TYPE_ANSWER:
handleAnswer(jsonMsg);
break;
case SIGNAL_TYPE_CANDIDATE:
handleCandidate(jsonMsg);
break;
}
});
conn.on("close", function(code, reason) {
console.info("连接关闭 code: " + code + ", reason: " + reason);
if(conn.client != null) {
// 强制让客户端从房间退出
handleForceLeave(conn.client);
}
});
conn.on("error", function(err) {
console.info("监听到错误:" + err);
});
}).listen(prort);
6. 参考文档
- WebRtc接口参考
- WebRTC 传输协议详解
- WebRTC的学习(java版本信令服务)
- Android webrtc实战（一）录制本地视频并播放，附带详细的基础知识讲解
- webSocket(wss)出现连接失败的问题解决方法
- 最重要的是这个，完整听了课程：2023最新Webrtc基础教程合集，涵盖所有核心内容（Nodejs+vscode+coturn