浏览器API
- MediaStream：通过MediaStream的API能够通过设备的摄像头及话筒获得视频、音频的同步流。
- RTCPeerConnection：RTCPeerConnection是WebRTC用于构建点对点之间稳定、高效的流传输的组件。
- RTCDataChannel：RTCDataChannel使得浏览器之间（点对点）建立一个高吞吐量、低延时的信道，用于传输任意数据。
WebRTC-internal-structure-cn.jpg
image.png
先用ajax和ws传递信息，建立信道
再用RTC点对点传输
视频压缩技术：H264/H265、VP8/VP9、AV1
webRTC：集成回音消除，视频编解码，跨平台
支持WebRTC的浏览器：Chrome、Firefox、Safari、Edge。支持情况具体：https://cloud.tencent.com/document/product/647/16863
image.png
终端：音视频采集，编解码，NAT穿越，音视频传输
signal服务器：信令处理
STUN /TURN：获取终端在公网的IP，以及NAT穿越失败后数据中转
track：视频轨，音频轨
stream：媒体流存放0音频视频轨，数据流存放0数据轨
API
var promise = navigator.mediaDevices.getUserMedia(constraints);
const mediaStreamContrains = {
video: true,
audio: true
};
const mediaStreamContrains = {
video: {
frameRate: {min: 20},
width: {min: 640, ideal: 1280},
height: {min: 360, ideal: 720},
aspectRatio: 16/9 // 宽高比
},
audio: {
echoCancellation: true, // 回音消除
noiseSuppression: true, // 降噪
autoGainControl: true // 自动增益
}
};
image.png
音频原理-采样率：人的听觉是20-20kHZ，大于最高频率两倍就算是无损所以平时采样率都是44.1k\48k等；
采样大小：8位（255）,16位
每个设备信息：id,label,kind
编码帧：压缩后的帧
H264：I帧（关键帧），P帧（参考帧），B帧（前后参考帧）
RTP协议
image.png
拆包组合
序号，时间戳，PT(音频和视频不一样)
image.png
RTP可能发生丢包、乱序、抖动，使用RTCP
RTCP:RR和SR收和发交换报文，来看网络质量
image.png
header：标识报文类型
sender info：发送多少包
report block： 接收包的情况
SDP(Session Description Protocal)
在webRTC中
image.png
image.png
会话元数据
网络描述
流描述
安全描述
服务质量描述
媒体协商
RTCPeerConnection
image.png
var pcConfig = null;
var pc = new RTCPeerConnection(pcConfig); // 创建peer对象
// 呼叫方创建offer
function doCall() {
console.log('Sending offer to peer');
pc.createOffer(setLocalAndSendMessage, handleCreateOfferError);
}
// 发送给被呼叫方
function setLocalAndSendMessage(sessionDescription) {
pc.setLocalDescription(sessionDescription);
sendMessage(sessionDescription);
}
// 呼叫方收到anser
socket.on('message', function(message) {
...
} else if (message.type === 'answer') {
pc.setRemoteDescription(new RTCSessionDescription(message));
} else if (...) {
...
}
....
});
ICE Candidate
本地IP/端口、候选者类型（host/srflx/relay）、优先级、传输协议、访问服务的用户名
{
IP: xxx.xxx.xxx.xxx,
port: number,
type: host/srflx/relay, //本机，内网主机映射的外网地址和端口，中继候选者
priority: number,
protocol: UDP/TCP,
usernameFragment: string
...
}
先对host类型进行联通测试，按照优先级试
ICE
本机收集所有host类型
STUN拿到公网IP，收集srflx
再用TURN收集relay
NAT
WeChatWorkScreenshot_9dc62a12-9267-4c9a-aaad-9dc9ce7dcac3.png
流程梳理
选择类型
- 登入rtc manager：返回streamID和信令服务器地址（可返回ws地址）
demo参数
config 设置相关配置
localVideo 本地流播放标签null
remoteVideo 远程流播放标签
nickname streamID
recevieDataCallback receiveMessageData方法
-
登入信令服务器，get方法，返回peers（可以用new ws建立连接）
WXWorkCapture_15935094097369.png 构造demo
构造函数里面:构造PeerConnSignaling,构造call
异步等待两个事件 newMessage，newNotification
ice过程
开发经验
改造谷歌原生代码
代码地址：https://github.com/webrtc/apprtc
先跑起来然后魔改源码，简化代码，抽取信令，改造信令。源码解读有空再开别的文章。
信令设计
msg_type字段定义
sign_in_event 前端登入
accept_in_event 后端登入
recv_src_id 接收源id
recv_dst_id 接收目标id
offer_sdp_event 转发sdp
answer_sdp_event 转发sdp
ice_event 转发ice
bye 发送断开请求
error 信令服务器发送错误信息
ping 向信令服务器发ping
pong 信令服务器返回pong
reconn_event 重新连接
update_session_event 切换事件
交互过程中前端状态机
1、建立连接
var ws = new WebSocket(url);
2、登入,message里面放原来的字段，结构不变，加一个msgType字段表示发送消息类型
signMsg = {
msg_type:'sign_in_event',
message:{
protocol: "webRTC",
media: {
audio: {
...
},
video: {
...
}
},
svrType: 0
}
}
ws.send(JSON.stringify(signMsg))
3、登入后返回自身id
{
msg_type: 'recv_src_id',
message:{
stream_id:'1594014948578356499vdptx',
src_id:1
}
}
4、等待返回对端id
// 成功
{
msg_type: 'recv_dst_id',
message:{
stream_id:'1594014948578356499vdptx',
dst_id:2
}
}
5、发送前端sdp,message里面和原sdp字段相同
sdpMsg = {
msg_type:'offer_sdp_event',
src_id:1,
dst_id:2,
message:{
type:'offer',
sdp:''
}
}
ws.send(JSON.stringify(sdpMsg))
6、发送anser sdp
{
msg_type: 'answer_sdp_event',
src_id: 2,
dst_id: 1,
message:{
type:'answer',
sdp:'...'
}
}
5、发送前端candidate,message里面和原candidate字段相同
iceMsg = {
msg_type:'ice_event',
src_id:1,
dst_id:2,
message:{
type:'candidate',
candidate:'...',
id:'...',
label:'...'
}
}
ws.send(JSON.stringify(iceMsg))
8、等待接收对端candidate
{
msg_type: "ice_event",
src_id: 2,
dst_id: 1,
message:{
type:'candidate',
candidate:'...',
id:0,
label:'...'
}
}
9、结束连接,先发送一个断开的消息，再断开ws
byeMsg = {
msg_type:'bye',
src_id:1,
dst_id:2,
}
ws.send(JSON.stringify(byeMsg))
10、错误消息
{
msg_type: "error",
src_id:2/null,
dst_id:1,
message:{
// 失败原因
}
}
11、ping
pingMsg = {
msg_type:'ping',
src_id:1,
dst_id:2,
}
ws.send(JSON.stringify(pingMsg))
12、回复ping
{
msg_type:'pong',
src_id:2,
dst_id:1/null,
}
13、重连
reconnMsg = {
msg_type:'reconn_event',
message: {
stream_id,
src_id:1,
dst_id:2,
}
}
ws.send(JSON.stringify(reconnMsg))
14、更新
updateMsg = {
msg_type:'update_session_event',
src_id:1,
dst_id:2,
message: {
protocol: "webRTC",
media: {
audio: {
...
},
video: {
...
}
},
svrType: 0
}
}
ws.send(JSON.stringify(updateMsg))
信令单独走websocket，关于WS的限制，浏览器无法ping pong
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-------+-+-------------+-------------------------------+
|F|R|R|R| opcode|M| Payload len | Extended payload length |
|I|S|S|S| (4) |A| (7) | (16/64) |
|N|V|V|V| |S| | (if payload len==126/127) |
| |1|2|3| |K| | |
+-+-+-+-+-------+-+-------------+ - - - - - - - - - - - - - - - +
| Extended payload length continued, if payload len == 127 |
+ - - - - - - - - - - - - - - - +-------------------------------+
| |Masking-key, if MASK set to 1 |
+-------------------------------+-------------------------------+
| Masking-key (continued) | Payload Data |
+-------------------------------- - - - - - - - - - - - - - - - +
: Payload Data continued ... :
+ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +
| Payload Data continued ... |
+---------------------------------------------------------------+
具体每一bit的意思
FIN 1bit 表示信息的最后一帧
RSV 1-3 1bit each 以后备用的 默认都为 0
Opcode 4bit 帧类型，操作码
Mask 1bit 掩码，是否加密数据，默认必须置为1
Payload 7bit 数据的长度
Masking-key 1 or 4 bit 掩码
Payload data (x + y) bytes 数据
Extension data x bytes 扩展数据
Application data y bytes 程序数据
OPCODE字段：4位
解释PayloadData，如果接收到未知的opcode，接收端必须关闭连接。
0x0表示附加数据帧
0x1表示文本数据帧
0x2表示二进制数据帧
0x3-7暂时无定义，为以后的非控制帧保留
0x8表示连接关闭
0x9表示ping
0xA表示pong
0xB-F暂时无定义，为以后的控制帧保留
验证浏览器自动回复pong
ws = require('ws');
wss = new ws.Server({port: 7777});
wss.on('connection', conn_ws => { console.log(conn_ws);
conn_ws.on('pong', msg => console.log('get cli pong', msg))
conn_ws.ping("clq");})
简单的get请求
const express = require('express')
const app = express()
const port = 3000
app.get('/v1/get123/', (req, res) => res.send('Hello World!'))
app.listen(port, () => console.log(`Example app listening on port ${port}!`))
错误码
http://blog.sina.com.cn/s/blog_145f07e7b0102x3x1.html
原生API再写一版最简版本
基础上看了一门课
image.png
代码地址： https://github.com/avdance/webrtc_web
找到官方文档API，跟着api写了一版SDK，可以自由嵌入任何前端框架，大致思路，此项目现在已废弃脱敏。
import { sendBye, offLine, sendPing, onLine, notifyIce, sendCallResponse, sendCall, sendCancelCall } from './wsSignal'
function webrtcPlayer(option) {
/**
* 私有变量，无需加this，模块变量外部无法获取
* 参数：内部websocket连接，本地流，远程流，远端连接，重新计数次数，websocket是否打开标识
*/
let _ws
let _localStream
let _remoteStream
let _remotePeerConnection
let _reconnCount = 0
let _wsOpen = true
let _iceCandidateCache = []
/**
* 传入参数
* readme有详细解释
*/
let handleMessage = option.handleMessage || null
let wsHost = option.wsHost
let stunHost = option.stunHost || ""
let pingTime = option.pingTime || 5
let reconnTimes = option.reconnTimes || 3
//set constraints
const mediaStreamConstraints = option.mediaStreamConstraints || {
video: true,
audio: true,
}
// Set up to exchange only video.
const offerOptions = option.offerOptions || {
offerToReceiveVideo: 1,
offerToReceiveAudio: 1,
}
// Define peer connections, streams and video elements.
let localVideo = option.localVideo || null
let remoteVideo = option.remoteVideo || null
let localId = option.localId || ''
let remoteIds = option.remoteId || []
let remoteId = ''
let sessionId = Date.parse(new Date()) + 's'
// Sets the MediaStream as the video element src.
const gotLocalMediaStream = (mediaStream) => {
return new Promise(function (resolve) {
// @ts-ignore
if (localVideo) {
localVideo.srcObject = mediaStream;
}
_localStream = mediaStream;
resolve();
})
}
function callAction() {
// Add local stream to connection and create offer to connect.
_remotePeerConnection.addStream(_localStream);
_remotePeerConnection.createOffer(offerOptions).then(createdOffer).catch();
}
// Handles remote MediaStream success by adding it as the remoteVideo src.
function gotRemoteMediaStream(event) {
const mediaStream = event.stream;
remoteVideo.srcObject = mediaStream;
_remoteStream = mediaStream;
}
function reconnect() {
if (_reconnCount >= reconnTimes - 1) {
_reconnCount = 0
wsClose()
} else {
_reconnCount++
wsOnline()
}
}
let sdp = ''
function receiveCall() {
console.log('receiveCall');
const servers = null; // Allows for RTC server configuration.
_remotePeerConnection = new RTCPeerConnection(servers)
navigator.mediaDevices
.getUserMedia(mediaStreamConstraints)
.then(gotLocalMediaStream)
.then(() => {
_remotePeerConnection.addEventListener('icecandidate', handleConnection)
_remotePeerConnection.addEventListener('addstream', gotRemoteMediaStream)
// 获取对端sdp,返回callResponse
let description = new RTCSessionDescription({
type: 'offer',
sdp: sdp,
});
_remotePeerConnection.addStream(_localStream);
_remotePeerConnection.setRemoteDescription(description)
_remotePeerConnection.createAnswer().then(createdAnswer)
})
}
function cancelCall() {
_ws.send(sendCancelCall(sessionId, localId, remoteId))
}
const wsClose = value => {
if (remoteVideo && remoteVideo.srcObject) {
remoteVideo.srcObject.getTracks().forEach(track => track.stop())
}
if (localVideo && localVideo.srcObject) {
localVideo.srcObject.getTracks().forEach(track => track.stop())
}
if (_remotePeerConnection) {
_remotePeerConnection.close()
_remotePeerConnection = null
}
if (value) {
_ws.send(sendBye(sessionId, localId, remoteId))
}
}
const wsOffline = () => {
// wsClose(true)
_wsOpen = false
_ws.send(offLine(localId))
_ws.close()
}
const wsOnline = (local_id, remote_id, token, extra_data) => {
localId = local_id
remoteIds = remote_id
console.log('connect')
_ws = new WebSocket(wsHost)
_ws.onopen = () => {
console.info('注册连接成功')
_wsOpen = true
// setInterval(() => {
// _ws.send(sendPing(localId));
// }, pingTime * 1000);
_ws.send(onLine(localId, token, pingTime + 2, extra_data))
};
_ws.onclose = () => {
if (_wsOpen) {
reconnect()
} else {
handleMessage({evt: 'onClose'})
console.log('连接已关闭')
}
};
_ws.onmessage = (evt) => {
let message = JSON.parse(evt.data);
console.log('---evt.data---', message)
if (message.msg_id === 'onlineResponse') {
// 获取对端sdp，收集ice
console.log('登录成功');
}
if (message.msg_id === 'callResponse') {
handleMessage({evt: 'onCallResponse'})
// 获取对端sdp，收集ice
let description = new RTCSessionDescription({
type: 'answer',
sdp: message.data.sdp,
});
_remotePeerConnection.setRemoteDescription(description);
remoteId = message.data.callee_ids[0]
_iceCandidateCache.forEach(iceCandidate => {
_ws.send(notifyIce(sessionId, localId, remoteId, iceCandidate))
})
}
if (message.msg_id === 'call') {
sdp = message.data.sdp
sessionId = message.data.session_id
remoteId = message.data.caller_id
handleMessage({evt: 'onCall', remoteId: remoteId})
}
if (message.msg_id === 'cancelCall') {
sessionId = message.data.session_id
handleMessage({evt: 'onCancelCall'})
}
if (message.msg_id === 'notifyIceCandidate') {
let iceCandidate = new RTCIceCandidate({
candidate: message.data.candidate,
sdpMid: message.data.sdp_mid,
sdpMLineIndex: message.data.sdp_mline_index,
});
if (iceCandidate && _remotePeerConnection && _remotePeerConnection.remoteDescription) {
console.log(iceCandidate)
console.log(_remotePeerConnection)
_remotePeerConnection.addIceCandidate(iceCandidate).catch(e => {
console.log("Failure during addIceCandidate(): " + e.name);
});
}
}
if (message.msg_id === 'bye') {
handleMessage({evt: 'onBye'})
wsClose()
}
};
};
const wsCall = (remote_id) => {
console.log('call');
if (remote_id) {
remoteIds = remote_id
}
_iceCandidateCache = []
const servers = null; // Allows for RTC server configuration.
_remotePeerConnection = new RTCPeerConnection(servers)
navigator.mediaDevices
.getUserMedia(mediaStreamConstraints)
.then(gotLocalMediaStream)
.then(() => {
_remotePeerConnection.addEventListener('icecandidate', handleConnectionCall)
_remotePeerConnection.addEventListener('addstream', gotRemoteMediaStream)
callAction()
})
};
function handleConnection(event) {
const peerConnection = event.target;
const iceCandidate = event.candidate;
if (iceCandidate && iceCandidate.candidate) {
_ws.send(notifyIce(sessionId, localId, remoteId, iceCandidate));
}
}
function handleConnectionCall(event) {
const peerConnection = event.target;
const iceCandidate = event.candidate;
if (iceCandidate && iceCandidate.candidate) {
_iceCandidateCache.push(iceCandidate)
}
}
// Logs offer creation and sets peer connection session descriptions.
function createdOffer(description) {
_ws.send(sendCall(sessionId, localId, remoteIds, description.sdp));
_remotePeerConnection.setLocalDescription(description);
}
// Logs answer to offer creation and sets peer connection session descriptions.
function createdAnswer(description) {
_ws.send(sendCallResponse(sessionId, localId, remoteId, description.sdp))
_remotePeerConnection.setLocalDescription(description);
}
return {
wsCall: wsCall,
wsClose: wsClose,
receiveCall: receiveCall,
cancelCall: cancelCall,
wsOnline: wsOnline,
wsOffline: wsOffline,
}
}
export default webrtcPlayer;
其它方面
1、用kurento写一版
2、端上只有小程序无法实现，需要用腾讯云TRTC，需要云上翻译才能对接，无法直接接webRTC
3、关于保活，很复杂也很重要
4、建立音视频，时序很重要，缓存做不好不行
项目已经不做一年了，记录下当时的一些过程
对于新技术
1、源码很重要，看懂入口出口，各种函数流转过程
2、官方API很重要，浏览器是一个黑盒子，封装了很多神奇的API，直接用。当然第三方库的思路差不多，区别就是可以看源码，白盒子。
3、快速入门能花钱的就要花，有前人总结过最好，当然是随缘，最近也做到很多无人涉及的板块。
4、所以英语好很重要，无中文资料的项目太多了。
