askrobots / network-computer-flutter
Public
One Flutter client for network-computer: drive a remote Linux/Mac desktop from Android, iPhone, iPad, or desktop over WebRTC.
Dan Quellhorst
No colour (compare) in ⋯: the desk's picture in grey on this device
c74471a
1 month, 3 weeks ago
History
peer.dart · 15.3 KB · 417 lines
| 1 | import 'dart:async'; |
| 2 | import 'dart:convert'; |
| 3 | import 'dart:io'; |
| 4 | import 'dart:typed_data'; |
| 5 | import 'package:path_provider/path_provider.dart'; |
| 6 | import 'package:flutter/foundation.dart'; |
| 7 | import 'package:flutter_webrtc/flutter_webrtc.dart'; |
| 8 | import 'protocol.dart'; |
| 9 | |
| 10 | class PeerStats { |
| 11 | String path = '?'; |
| 12 | double rttMs = 0, fps = 0, mbps = 0, jitterMs = 0; |
| 13 | int width = 0, height = 0, lost = 0; |
| 14 | bool relayed = false; |
| 15 | String codec = '?'; |
| 16 | } |
| 17 | |
| 18 | /// One WebRTC session to a host: receives video+audio, sends input on an |
| 19 | /// unreliable "input" channel, and uses a reliable "control" channel for |
| 20 | /// everything that must arrive: screen size, keyboard layout, clipboard, |
| 21 | /// voice, the launcher (the same messages as the browser client). |
| 22 | class PeerClient { |
| 23 | final List<IceServer> ice; |
| 24 | final bool relayOnly; |
| 25 | final RTCVideoRenderer renderer; |
| 26 | |
| 27 | RTCPeerConnection? _pc; |
| 28 | RTCDataChannel? _input; |
| 29 | RTCDataChannel? _ctl; |
| 30 | RTCDataChannel? _file; // opened with the others: later ones never report open here |
| 31 | RTCDataChannel? _fileIn; // files from the desk |
| 32 | String? _inName; |
| 33 | int _inSize = 0; |
| 34 | bool _inPrint = false; |
| 35 | BytesBuilder? _inData; |
| 36 | Completer<String>? _fileAnswer; |
| 37 | Future<void> _fileQueue = Future.value(); |
| 38 | MediaStream? _mic; |
| 39 | // the camera: its own sending video transceiver, opened empty at connect; |
| 40 | // the camera is attached only while it is on (the camera light tells the truth) |
| 41 | RTCRtpTransceiver? _camTx; |
| 42 | MediaStream? _cam; |
| 43 | bool _camFront = true; |
| 44 | Timer? _statsTimer; |
| 45 | int _lastBytes = 0, _lastFrames = 0; |
| 46 | DateTime _lastAt = DateTime.now(); |
| 47 | |
| 48 | void Function(Map<String, dynamic> candidate)? onIceCandidate; |
| 49 | void Function(RTCPeerConnectionState)? onState; |
| 50 | void Function(PeerStats)? onStats; |
| 51 | void Function()? onControlOpen; |
| 52 | /// A file from the desk was saved (its path), or failed (null, reason). |
| 53 | void Function(String? path, String note)? onFileReceived; |
| 54 | /// A PDF the desk printed to "My device": this device's print dialog. |
| 55 | void Function(String path, String name)? onPrint; |
| 56 | void Function(Map<String, dynamic>)? onControl; |
| 57 | |
| 58 | PeerClient(this.ice, this.relayOnly, this.renderer); |
| 59 | |
| 60 | Future<void> start() async { |
| 61 | final config = { |
| 62 | 'iceServers': ice.map((e) => e.toMap()).toList(), |
| 63 | 'sdpSemantics': 'unified-plan', |
| 64 | if (relayOnly) 'iceTransportPolicy': 'relay', |
| 65 | }; |
| 66 | final pc = await createPeerConnection(config); |
| 67 | _pc = pc; |
| 68 | |
| 69 | pc.onTrack = (event) async { |
| 70 | debugPrint('nc track: ${event.track.kind} streams=${event.streams.length}'); |
| 71 | if (event.track.kind != 'video') return; |
| 72 | MediaStream stream; |
| 73 | if (event.streams.isNotEmpty) { |
| 74 | stream = event.streams[0]; |
| 75 | } else { |
| 76 | // some platforms deliver the track without its stream: wrap it ourselves |
| 77 | stream = await createLocalMediaStream('remote-video'); |
| 78 | await stream.addTrack(event.track); |
| 79 | } |
| 80 | renderer.srcObject = stream; |
| 81 | debugPrint('nc renderer: textureId=${renderer.textureId} src=${renderer.srcObject?.id}'); |
| 82 | }; |
| 83 | pc.onIceCandidate = (c) { |
| 84 | final m = c.toMap(); |
| 85 | if (m['candidate'] != null) onIceCandidate?.call(Map<String, dynamic>.from(m)); |
| 86 | }; |
| 87 | pc.onConnectionState = (s) { |
| 88 | onState?.call(s); |
| 89 | if (s == RTCPeerConnectionState.RTCPeerConnectionStateConnected) _startStats(); |
| 90 | }; |
| 91 | |
| 92 | await pc.addTransceiver( |
| 93 | kind: RTCRtpMediaType.RTCRtpMediaTypeVideo, |
| 94 | init: RTCRtpTransceiverInit(direction: TransceiverDirection.RecvOnly)); |
| 95 | // The microphone is opened with the connection and attached muted; the mic |
| 96 | // button only unmutes it. Attaching a track later (replaceTrack) left the |
| 97 | // iPhone sending no audio at all. No permission: the call works without. |
| 98 | MediaStreamTrack? micTrack; |
| 99 | try { |
| 100 | final mic = await navigator.mediaDevices.getUserMedia({ |
| 101 | 'audio': { |
| 102 | 'echoCancellation': true, // keep the desktop's sound out of the mic |
| 103 | 'noiseSuppression': true, |
| 104 | 'autoGainControl': true, |
| 105 | }, |
| 106 | 'video': false, |
| 107 | }); |
| 108 | _mic = mic; |
| 109 | micTrack = mic.getAudioTracks().first; |
| 110 | micTrack.enabled = false; |
| 111 | } catch (e) { |
| 112 | debugPrint('nc mic: unavailable ($e)'); |
| 113 | } |
| 114 | final init = RTCRtpTransceiverInit( |
| 115 | direction: TransceiverDirection.SendRecv, streams: _mic == null ? [] : [_mic!]); |
| 116 | if (micTrack != null) { |
| 117 | await pc.addTransceiver(track: micTrack, kind: RTCRtpMediaType.RTCRtpMediaTypeAudio, init: init); |
| 118 | } else { |
| 119 | await pc.addTransceiver(kind: RTCRtpMediaType.RTCRtpMediaTypeAudio, init: init); |
| 120 | } |
| 121 | _camTx = await pc.addTransceiver( |
| 122 | kind: RTCRtpMediaType.RTCRtpMediaTypeVideo, |
| 123 | init: RTCRtpTransceiverInit(direction: TransceiverDirection.SendOnly)); |
| 124 | await _loudspeaker(); // capturing puts iOS in call mode: earpiece otherwise |
| 125 | |
| 126 | _input = await pc.createDataChannel( |
| 127 | 'input', RTCDataChannelInit()..ordered = false..maxRetransmits = 0); |
| 128 | final file = await pc.createDataChannel('file', RTCDataChannelInit()); // reliable, ordered |
| 129 | _file = file; |
| 130 | file.onMessage = (m) { |
| 131 | final a = _fileAnswer; |
| 132 | if (a != null && !a.isCompleted && !m.isBinary) a.complete(m.text); |
| 133 | }; |
| 134 | // files from the desk ("Send to my device") arrive on a channel we open |
| 135 | final fileIn = await pc.createDataChannel('file-in', RTCDataChannelInit()); |
| 136 | _fileIn = fileIn; |
| 137 | fileIn.onMessage = _onFileIn; |
| 138 | final ctl = await pc.createDataChannel('control', RTCDataChannelInit()); // reliable, ordered |
| 139 | _ctl = ctl; |
| 140 | ctl.onDataChannelState = (s) { |
| 141 | if (s == RTCDataChannelState.RTCDataChannelOpen) onControlOpen?.call(); |
| 142 | }; |
| 143 | ctl.onMessage = (m) { |
| 144 | if (m.isBinary) return; |
| 145 | try { |
| 146 | final v = jsonDecode(m.text); |
| 147 | if (v is Map<String, dynamic>) onControl?.call(v); |
| 148 | } catch (_) {} |
| 149 | }; |
| 150 | } |
| 151 | |
| 152 | /// Send a file to the desk (saved on its Desktop): its own reliable channel, |
| 153 | /// a JSON header, 16 KB pieces with flow control, then "end"; the desk |
| 154 | /// answers "ok NAME" or "error: WHY". Same as the browser's file drop. |
| 155 | Future<String> sendFile(String name, Uint8List data, void Function(double) progress) { |
| 156 | // one file at a time on the file channel |
| 157 | final done = _fileQueue.then((_) => _sendFileNow(name, data, progress)); |
| 158 | _fileQueue = done.then((_) {}, onError: (_) {}); |
| 159 | return done; |
| 160 | } |
| 161 | |
| 162 | Future<String> _sendFileNow(String name, Uint8List data, void Function(double) progress) async { |
| 163 | final ch = _file; |
| 164 | if (ch == null || ch.state != RTCDataChannelState.RTCDataChannelOpen) { |
| 165 | throw 'the file channel is not open'; |
| 166 | } |
| 167 | final answer = Completer<String>(); |
| 168 | _fileAnswer = answer; |
| 169 | await ch.send(RTCDataChannelMessage(jsonEncode({'name': name, 'size': data.length}))); |
| 170 | const piece = 16 * 1024; |
| 171 | for (var i = 0; i < data.length && !answer.isCompleted; i += piece) { |
| 172 | while ((ch.bufferedAmount ?? 0) > 4 * 1024 * 1024) { |
| 173 | await Future.delayed(const Duration(milliseconds: 20)); |
| 174 | } |
| 175 | final end = i + piece < data.length ? i + piece : data.length; |
| 176 | await ch.send(RTCDataChannelMessage.fromBinary(data.sublist(i, end))); |
| 177 | progress(end / data.length); |
| 178 | } |
| 179 | if (!answer.isCompleted) await ch.send(RTCDataChannelMessage('end')); |
| 180 | return answer.future.timeout(const Duration(minutes: 2)); |
| 181 | } |
| 182 | |
| 183 | /// A file from the desk: header, pieces, "end"; saved in the app's Documents |
| 184 | /// (on iPhone: Files › On My iPhone › Network Computer); answers "ok NAME". |
| 185 | Future<void> _onFileIn(RTCDataChannelMessage m) async { |
| 186 | final ch = _fileIn; |
| 187 | if (ch == null) return; |
| 188 | if (m.isBinary) { |
| 189 | _inData?.add(m.binary); |
| 190 | return; |
| 191 | } |
| 192 | if (m.text != 'end') { |
| 193 | try { |
| 194 | final h = jsonDecode(m.text) as Map<String, dynamic>; |
| 195 | _inName = '${h['name']}'.split('/').last.split('\\').last; |
| 196 | _inSize = (h['size'] as num).toInt(); |
| 197 | _inPrint = h['print'] == true; |
| 198 | _inData = BytesBuilder(copy: false); |
| 199 | } catch (_) { |
| 200 | await ch.send(RTCDataChannelMessage('error: bad header')); |
| 201 | } |
| 202 | return; |
| 203 | } |
| 204 | final name = _inName, data = _inData, print = _inPrint; |
| 205 | _inName = null; _inData = null; _inPrint = false; |
| 206 | if (name == null || data == null) return; |
| 207 | if (data.length != _inSize) { |
| 208 | await ch.send(RTCDataChannelMessage('error: got ${data.length} of $_inSize bytes')); |
| 209 | onFileReceived?.call(null, '$name did not arrive whole'); |
| 210 | return; |
| 211 | } |
| 212 | if (print) { // kept only while its dialog is up: a print job, not a document |
| 213 | try { |
| 214 | final path = '${(await getTemporaryDirectory()).path}/$name'; |
| 215 | await File(path).writeAsBytes(data.takeBytes()); |
| 216 | await ch.send(RTCDataChannelMessage('ok $name')); |
| 217 | onPrint?.call(path, name); |
| 218 | } catch (e) { |
| 219 | await ch.send(RTCDataChannelMessage('error: $e')); |
| 220 | onFileReceived?.call(null, '$name: $e'); |
| 221 | } |
| 222 | return; |
| 223 | } |
| 224 | try { |
| 225 | final dir = await getApplicationDocumentsDirectory(); |
| 226 | var path = '${dir.path}/$name'; |
| 227 | final dot = name.lastIndexOf('.'); |
| 228 | final stem = dot > 0 ? name.substring(0, dot) : name, ext = dot > 0 ? name.substring(dot) : ''; |
| 229 | for (var i = 2; File(path).existsSync(); i++) { |
| 230 | path = '${dir.path}/$stem ($i)$ext'; |
| 231 | } |
| 232 | await File(path).writeAsBytes(data.takeBytes()); |
| 233 | await ch.send(RTCDataChannelMessage('ok ${path.split('/').last}')); |
| 234 | onFileReceived?.call(path, path.split('/').last); |
| 235 | } catch (e) { |
| 236 | await ch.send(RTCDataChannelMessage('error: $e')); |
| 237 | onFileReceived?.call(null, '$name: $e'); |
| 238 | } |
| 239 | } |
| 240 | |
| 241 | bool get controlOpen => _ctl?.state == RTCDataChannelState.RTCDataChannelOpen; |
| 242 | |
| 243 | /// Send on the reliable control channel (in order, never dropped). |
| 244 | void sendControl(Map<String, dynamic> m) { |
| 245 | if (controlOpen) _ctl!.send(RTCDataChannelMessage(jsonEncode(m))); |
| 246 | } |
| 247 | |
| 248 | Future<String> createOffer() async { |
| 249 | final offer = await _pc!.createOffer(); |
| 250 | await _pc!.setLocalDescription(offer); |
| 251 | return offer.sdp!; |
| 252 | } |
| 253 | |
| 254 | Future<void> setAnswer(String sdp) async => |
| 255 | _pc!.setRemoteDescription(RTCSessionDescription(sdp, 'answer')); |
| 256 | |
| 257 | Future<void> addCandidate(Map<String, dynamic> c) async => _pc!.addCandidate( |
| 258 | RTCIceCandidate(c['candidate'], c['sdpMid'], c['sdpMLineIndex'])); |
| 259 | |
| 260 | void send(InputEvent e) { |
| 261 | final dc = _input; |
| 262 | if (dc != null && dc.state == RTCDataChannelState.RTCDataChannelOpen) { |
| 263 | dc.send(RTCDataChannelMessage(jsonEncode(e.toJson()))); |
| 264 | } |
| 265 | } |
| 266 | |
| 267 | /// Mute or unmute the microphone (opened with the connection). |
| 268 | Future<void> setMic(bool on) async { |
| 269 | final tracks = _mic?.getAudioTracks() ?? <MediaStreamTrack>[]; |
| 270 | if (tracks.isEmpty) { |
| 271 | if (on) throw 'no microphone (allow it in Settings, then reconnect)'; |
| 272 | return; |
| 273 | } |
| 274 | for (final t in tracks) { |
| 275 | t.enabled = on; |
| 276 | } |
| 277 | debugPrint('nc mic: ${on ? 'on' : 'off'}'); |
| 278 | } |
| 279 | |
| 280 | /// Turn this device's camera on or off. On the desk it is the webcam |
| 281 | /// "network-computer camera" for any app (a video call in the browser). |
| 282 | Future<void> setCamera(bool on) async { |
| 283 | final tx = _camTx; |
| 284 | if (tx == null) throw 'not connected'; |
| 285 | if (!on) { |
| 286 | await tx.sender.replaceTrack(null); |
| 287 | await _closeCamera(); |
| 288 | debugPrint('nc camera: off'); |
| 289 | return; |
| 290 | } |
| 291 | if (_cam != null) return; |
| 292 | final cam = await navigator.mediaDevices.getUserMedia({ |
| 293 | 'audio': false, |
| 294 | 'video': { |
| 295 | 'facingMode': _camFront ? 'user' : 'environment', |
| 296 | 'width': {'ideal': 1280}, |
| 297 | 'height': {'ideal': 720}, |
| 298 | 'frameRate': {'ideal': 30}, |
| 299 | }, |
| 300 | }); |
| 301 | _cam = cam; |
| 302 | await tx.sender.replaceTrack(cam.getVideoTracks().first); |
| 303 | debugPrint('nc camera: on (${_camFront ? 'front' : 'back'})'); |
| 304 | } |
| 305 | |
| 306 | bool get cameraOn => _cam != null; |
| 307 | |
| 308 | /// Front to back camera and back (phones, tablets). |
| 309 | Future<void> flipCamera() async { |
| 310 | _camFront = !_camFront; |
| 311 | final t = _cam?.getVideoTracks(); |
| 312 | if (t == null || t.isEmpty) return; |
| 313 | await Helper.switchCamera(t.first); |
| 314 | } |
| 315 | |
| 316 | Future<void> _closeCamera() async { |
| 317 | for (final t in _cam?.getTracks() ?? <MediaStreamTrack>[]) { |
| 318 | await t.stop(); |
| 319 | } |
| 320 | await _cam?.dispose(); |
| 321 | _cam = null; |
| 322 | } |
| 323 | |
| 324 | Future<void> _closeMic() async { |
| 325 | for (final t in _mic?.getTracks() ?? <MediaStreamTrack>[]) { |
| 326 | await t.stop(); |
| 327 | } |
| 328 | await _mic?.dispose(); |
| 329 | _mic = null; |
| 330 | } |
| 331 | |
| 332 | /// The desk's sound on the loudspeaker: iOS otherwise picks the earpiece for |
| 333 | /// a call-style session. |
| 334 | Future<void> _loudspeaker() async { |
| 335 | if (defaultTargetPlatform != TargetPlatform.iOS) return; |
| 336 | try { |
| 337 | await Helper.setSpeakerphoneOn(true); |
| 338 | } catch (e) { |
| 339 | debugPrint('nc audio route: $e'); |
| 340 | } |
| 341 | } |
| 342 | |
| 343 | void _startStats() { |
| 344 | _statsTimer = Timer.periodic(const Duration(seconds: 1), (_) => _poll()); |
| 345 | } |
| 346 | |
| 347 | Future<void> _poll() async { |
| 348 | final pc = _pc; |
| 349 | if (pc == null) return; |
| 350 | final reports = await pc.getStats(); |
| 351 | final s = PeerStats(); |
| 352 | final cand = <String, StatsReport>{}; |
| 353 | StatsReport? pair, inbound; |
| 354 | for (final r in reports) { |
| 355 | switch (r.type) { |
| 356 | case 'candidate-pair': |
| 357 | if (r.values['nominated'] == true && r.values['state'] == 'succeeded') pair = r; |
| 358 | break; |
| 359 | case 'local-candidate': |
| 360 | case 'remote-candidate': |
| 361 | cand[r.id] = r; |
| 362 | break; |
| 363 | case 'inbound-rtp': |
| 364 | if (r.values['kind'] == 'video') inbound = r; |
| 365 | break; |
| 366 | case 'outbound-rtp': |
| 367 | if (r.values['kind'] == 'audio') _micSent = r.values['bytesSent']; |
| 368 | break; |
| 369 | } |
| 370 | } |
| 371 | if (pair != null) { |
| 372 | final l = cand[pair.values['localCandidateId']]?.values['candidateType']; |
| 373 | final rc = cand[pair.values['remoteCandidateId']]?.values['candidateType']; |
| 374 | s.relayed = l == 'relay' || rc == 'relay'; |
| 375 | s.path = '$l → $rc'; |
| 376 | s.rttMs = ((pair.values['currentRoundTripTime'] as num?) ?? 0).toDouble() * 1000; |
| 377 | } |
| 378 | if (inbound != null) { |
| 379 | final now = DateTime.now(); |
| 380 | final dt = now.difference(_lastAt).inMilliseconds / 1000.0; |
| 381 | final bytes = (inbound.values['bytesReceived'] as num?)?.toInt() ?? 0; |
| 382 | final frames = (inbound.values['framesDecoded'] as num?)?.toInt() ?? 0; |
| 383 | s.lost = (inbound.values['packetsLost'] as num?)?.toInt() ?? 0; |
| 384 | if (dt > 0) { |
| 385 | s.mbps = (bytes - _lastBytes) * 8 / dt / 1e6; |
| 386 | s.fps = (frames - _lastFrames) / dt; |
| 387 | } |
| 388 | _lastBytes = bytes; _lastFrames = frames; _lastAt = now; |
| 389 | s.width = (inbound.values['frameWidth'] as num?)?.toInt() ?? 0; |
| 390 | s.height = (inbound.values['frameHeight'] as num?)?.toInt() ?? 0; |
| 391 | s.jitterMs = ((inbound.values['jitter'] as num?) ?? 0).toDouble() * 1000; |
| 392 | } |
| 393 | onStats?.call(s); |
| 394 | if (!kReleaseMode && ++_polls % 5 == 0 && inbound != null) { // every 5 s in debug/profile runs |
| 395 | final v = inbound.values; |
| 396 | debugPrint('nc video: bytes=${v['bytesReceived']} packets=${v['packetsReceived']} ' |
| 397 | 'framesReceived=${v['framesReceived']} decoded=${v['framesDecoded']} dropped=${v['framesDropped']} ' |
| 398 | 'size=${v['frameWidth']}x${v['frameHeight']} decoder=${v['decoderImplementation']} ' |
| 399 | 'keyframes=${v['keyFramesDecoded']} pli=${v['pliCount']} path=${s.path} ' |
| 400 | 'view=${renderer.videoWidth}x${renderer.videoHeight} mic-bytes-sent=$_micSent'); |
| 401 | } |
| 402 | } |
| 403 | int _polls = 0; |
| 404 | Object? _micSent; |
| 405 | |
| 406 | Future<void> close() async { |
| 407 | _statsTimer?.cancel(); |
| 408 | await _closeMic(); |
| 409 | await _closeCamera(); |
| 410 | await _input?.close(); |
| 411 | await _file?.close(); |
| 412 | await _fileIn?.close(); |
| 413 | await _ctl?.close(); |
| 414 | await _pc?.close(); |
| 415 | renderer.srcObject = null; |
| 416 | } |
| 417 | } |