1. 简介

开源项目：
https://github.com/Genymobile/scrcpy

项目简介:
通过在手机端使用虚拟显示器进行录屏, 并直接使用手机自带的视频编码器将屏幕数据编码成视频流(格式H264), 并将其发送成PC端, 使用ffmpeg对视频流进行解码, 并通过SDL将手机屏幕镜像显示到电脑屏幕, 并且再通过控制流将PC端的鼠标手势等操作发送给APP端对手机进行远程遥控.

技术点:
该项目使用的技术和云游戏或手机直播使用的技术类似, 包括录屏, 视频流编码, 推流, 视频流解码, 控制流远程操控等. 扩展内容需查看其他笔记: ffmpeg, WebRTC

目录结构：

app：PC端，纯C语言开发, 基于ffmpeg和SDL开发, 作为client端.
server，APP端，Java语言开发, adb命令行下执行的Java进程(Dex格式)

2. APP端 (Java)

2.1 Server

Server.main()

createOptions()
- maxSize // 最大尺寸
- bitRate // 比特率
- maxFps // 限帧
- lockedVideoOrientation // 锁定视频方向
- tunnelForward // 默认false，app作为server，监听unix端口，adb forward到PC端口，等待PC端连接。true则反之，adb tunnel。
- crop // 视频截取尺寸
- sendFrameMeta // 是否发送FrameMeta（和视频流一起）
- control // 是否控制
- displayId // 屏幕ID
scrcpy(opts)
- final Device device = new Device(options);
- DesktopConnection connection = DesktopConnection.open(device, tunnelForward) // 建立与PC的连接，详见 2.2节
- ScreenEncoder screenEncoder = new ScreenEncoder(options.getSendFrameMeta(), options.getBitRate(), options.getMaxFps());
- if control:
  - Controller controller = new Controller(device, connection);
    - sender = new DeviceMessageSender(connection);
  - startController(controller); // 控制流
    - controller.control(); // 开线程调用, 详见 2.4节
  - startDeviceMessageSender(controller.getSender());
    - sender.loop(); // 开线程调用
    - while true:
      - connection.sendDeviceMessage(clipboardTextEvent); // 发送剪贴板内容给PC端
- screenEncoder.streamScreen(device, connection.getVideoFd()); // 发送视频流（阻塞）, 详见 2.3节

2.2 DesktopConnection

DesktopConnection.open(device, tunnelForward)

if tunnelForward:
- LocalServerSocket localServerSocket = new LocalServerSocket("scrcpy"); // app作为server，监听在unix端口，adb forward到PC端口，等待PC端来连接
- videoSocket = localServerSocket.accept(); // 视频流
- controlSocket = localServerSocket.accept(); // 控制流
else:
- videoSocket = connect("scrcpy"); // app作为client，连接unix “scrcpy”端 <- adb reverse PC端口
- controlSocket = connect("scrcpy");
DesktopConnection connection = new DesktopConnection(videoSocket, controlSocket);
connection.send(Device.getDeviceName(), videoSize.getWidth(), videoSize.getHeight()); // 像PC发送设备名称，视频长宽尺寸, PC端读取的代码在device_read_info
- buffer = new byte[64 + 4]
- 64: deviceNames.getBytes()
- 2: width
- 2: height

DesktopConnection.receiveControlMessage()

msg = controlMessageReader.next()
controlMessageReader.readFrom(controlInputStream)
- controlInputStream.read(rawBuffer, head, rawBuffer.length - head); // byte[] rawBuffer = new byte[1024];

2.3 ScreenEncoder

ScreenEncoder.streamScreen(device, videoFd)

Looper.prepareMainLooper();
Workarounds.fillAppInfo();
- new android.app.ActivityThread()
- Application app = Instrumentation.newApplication(Application.class, ctx);
- .....
createFormat()
- MediaFormat format = new MediaFormat();
- format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);
- format.setFloat(KEY_MAX_FPS_TO_ENCODER, maxFps);
MediaCodec codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC); // 创建视频编码器
IBinder display = SurfaceControl.createDisplay("scrcpy", true); // 创建虚拟屏幕，详见 2.5小节
setSize()
- format.setInteger(MediaFormat.KEY_WIDTH, width);
- format.setInteger(MediaFormat.KEY_HEIGHT, height);
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
Surface surface = codec.createInputSurface();
setDisplaySurface()
- SurfaceControl.openTransaction();
- SurfaceControl.setDisplaySurface(display, surface);
- SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect);
- SurfaceControl.setDisplayLayerStack(display, layerStack);
- SurfaceControl.closeTransaction();
codec.start();
alive = encode(codec, fd);
- int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1); // 获取输出的编码bufferID
- ByteBuffer codecBuffer = codec.getOutputBuffer(outputBufferId); // 获取编码后的视频buffer
- if sendFrameMeta: // 总是true
  - writeFrameMeta(fd, bufferInfo, codecBuffer.remaining()); // 发送FrameMeta
- IO.writeFully(fd, codecBuffer); // 发送buffer到视频流
codec.stop();

2.4 Controller

Controller.control()

if not device.isScreenOn(): // 如果屏幕没亮，则点击电源键点亮屏幕
- injectKeyCode(KeyEvent.KEYCODE_POWER)
while true:
- handleEvent();
  - msg = connection.receiveControlMessage()
  - switch msg.type:
    - case TYPE_INJECT_KEYCODE:
      - if device.supportsInputEvents():
        
        injectKeycode(msg.getAction(), msg.getKeycode(), msg.getMetaState());
        
        KeyEvent event = new KeyEvent();
        
        device.injectInputEvent(event)
        
        android.hardware.input.InputManager.injectInputEvent(event, int)
    - case TYPE_INJECT_TEXT
    - case TYPE_INJECT_TOUCH_EVENT
    - case TYPE_INJECT_SCROLL_EVENT
    - case TYPE_BACK_OR_SCREEN_ON
    - case TYPE_EXPAND_NOTIFICATION_PANEL
    - case TYPE_COLLAPSE_NOTIFICATION_PANEL
    - case TYPE_GET_CLIPBOARD
      - sender.pushClipboardText(serviceManager.getClipboardManager().getText());
    - case TYPE_SET_CLIPBOARD
      - device.setClipboardText(msg.getText());
        
        serviceManager.getClipboardManager().setText(text);
    - case TYPE_SET_SCREEN_POWER_MODE
      - device.setScreenPowerMode(msg.getAction());
        
        SurfaceControl.setDisplayPowerMode()
    - case TYPE_ROTATE_DEVICE

2.5 SurfaceControl

SurfaceControl

createDisplay() // 创建虚拟显示器
- android.view.SurfaceControl.createDisplay(name, secure)
setDisplaySurface()
- android.view.SurfaceControl.setDisplaySurface(display, surface)
setDisplayProjection()
- android.view.SurfaceControl.setDisplayProjection(display, orientaion, layerStackRect, displayRect)
setDisplayLayerStack()
- android.view.SurfaceControl.setDisplayLayerStack(display, layerStack)
openTransaction()
- android.view.SurfaceControl.openTransaction()
closeTransaction()
- android.view.SurfaceControl.closeTransaction()
getBuiltInDisplay()
- android.view.SurfaceControl.getBuiltInDisplay() // or getInternalDisplayToken() if sdk >= android Q
setDisplayPowerMode()
- android.view.SurfaceControl.setDisplayPowerMode()
destroyDisplay()
- android.view.SurfaceControl.destroyDisplay()

为什么用反射去调用 android.view.SurfaceControl 接口，而不是使用如下的接口：

import android.hardware.display.DisplayManager;
import android.hardware.display.VirtualDisplay;
import android.media.projection.MediaProjection;
import android.media.projection.MediaProjectionManager;


virtualDisplay = mediaProjection. **createVirtualDisplay** ("WebRTC_ScreenCapture", width, height,
VIRTUAL_DISPLAY_DPI, DISPLAY_FLAGS, new Surface(surfaceTextureHelper.getSurfaceTexture()),
null /* callback */, null /* callback handler */);

这些接口可能需要权限，以及Context，而在命令行运行的dex没有这些。

3. PC端 (C语言)

3.1 main

main()

scrcpy_parse_args() // 解析参数, TODO
- serial // 多台adb device时指定需要连接的serial
av_register_all() // ffmpeg注册所有视频编码格式
avformat_network_init() // ffmpeg初始化网络格式, TODO
scrcpy(args.opts)
- server_start() // 开启本地服务
  - push_server() // 将APP端的server文件推送(adb push)到手机
  - enable_tunnel_any_port()
    - enable_tunnel_reverse_any_port() // PC端作为server，监听在local_port, 等待APP端来连接
      - adb reverse tcp:<local_port> localabstract:scrcpy
      - server_socket = listen_on_port(port)
        
        net_connect("localhost", port)
        
        sock = socket(AF_INET, SOCK_STREAM, 0)
        
        setsockopt(sock, ...)
        
        bind(sock)
        
        listen(sock)
    - or enable_tunnel_forward_any_port(） // PC端作为client，通过adb forward去连接监听在scrcpy端口的APP端的server
      - adb forward tcp:<local_port> localabstract:scrcpy
  - server->process = execute_server(server, params) // 拉起PC端的server
    - adb shell CLASSPATH=/data/local/tmp/scrcpy-server.jar app_process
  - SDL_CreateThread(run_wait_server)
    - cmd_simple_wait(server->process)
- sdl_init_and_configure(display, render_driver) // SDL初始化
  - SDL_Init()
  - SDL_SetHint(SDL_HINT_RENER_DRIVER,options->render_driver) // "direct3d", "opengl", "opengles2", "opengles", "metal" and "software"
  - SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "1")
  - SDL_SetHint(SDL_HINT_MOUSE_FOCUS_CLICKTHROUGH, "1")
  - SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0")
  - SDL_SetHint(SDL_HINT_VIDEO_MINIMIZE_ON_FOCUS_LOSS, "0")
  - SDL_EnableScreenSaver()
- server_connect_to() // 连接APP端
  - if tunnel_forward:
    - erver->video_socket = net_accept(server->server_socket);
    - server->control_socket = net_accept(server->server_socket);
  - else:
    - server->video_socket = connect_to_server(server->local_port, attempts, delay);
    - server->control_socket = net_connect(IPV4_LOCALHOST, server->local_port);
- device_read_info(server.video_socket) // 读取手机设备名, 对应 2.2小节
- fps_counter_init() // ALT+I 在控制台显示FPS
- video_buffer_init(video_buffer) // 初始化视频buffer
- if control:
  - file_handler_init() // 初始化控制流
- decoder_init() // 初始化解码器
- recorder_init() // 初始化录制器(直接录制视频到文件)
- av_log_set_callback(av_log_callback); // ffmpeg日志回调, TODO
- stream_init() // 初始化视频流
- stream_start()
- controller_init() // 初始化控制流流
- controller_start()
- screen_init_rendering() // 详见 screen.c, 初始化渲染
- if opts.turn_screen_off()
  - controller_push_msg(screen_power_mode_off_msg) // 关闭手机屏幕
- if opts.fullscreen():
  - screen_switch_fullscreen() // 切换PC端全屏
- if opts.show_touches:
  - wait_show_touches() // 显示触摸
- event_loop()
  - while SDL_WaitEvent(&event): // SDL事件驱动主循环
    - handle_event(&event, control)
      - switch event.type:
        
        case EVENT_NEW_FRAME
        
        case SDL_WINDOWEVENT
        
        case SDL_TEXTINPUT
        
        case SDL_KEYDOWN
        
        case SDL_KEYUP
        
        case SDL_MOUSEMOTION
        
        case SDL_MOUSEWHEEL
        
        case SDL_MOUSEBUTTONDOWN
        
        case SDL_MOUSEBUTTONUP
        
        case SDL_FINGERMOTION
        
        case SDL_FINGERDOWN
        
        case SDL_FINGERUP
        
        case SDL_DROPFILE
- screen_destroy()
avformat_network_deinit()

execute_server: 拉起PC端的命令及参数说明：

adb shell CLASSPATH=/data/local/tmp/scrcpy-server.jar app_process
/                                   // unused
com.genymobile.scrcpy.Server        // java main class
1.13                                // version
0                                   // max_size
8000000                             // bit_rate
0                                   // max_fps
-1                                  // lock_video_orientation
false                               // trunel_forward
-                                   // crop
true                                // send frame meta
true                                // iscontrol

NOTE ATTRIBUTES

Created Date: 2020-05-18 04:41:29
Last Evernote Update Date: 2020-05-20 03:22:46

Scrcpy源码阅读