One Flutter client for network-computer: drive a remote Linux/Mac desktop from Android, iPhone, iPad, or desktop over WebRTC.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/network-computer-flutter.git
HTTPS
https://gitstable.com/git/askrobots/network-computer-flutter.git
SSH
ssh://git@gitstable.com:2222/askrobots/network-computer-flutter.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

Dan Quellhorst No colour (compare) in ⋯: the desk's picture in grey on this device c74471a 1 month, 3 weeks ago History
peer.dart · 15.3 KB · 417 lines
1import 'dart:async';
2import 'dart:convert';
3import 'dart:io';
4import 'dart:typed_data';
5import 'package:path_provider/path_provider.dart';
6import 'package:flutter/foundation.dart';
7import 'package:flutter_webrtc/flutter_webrtc.dart';
8import 'protocol.dart';
9
10class PeerStats {
11 String path = '?';
12 double rttMs = 0, fps = 0, mbps = 0, jitterMs = 0;
13 int width = 0, height = 0, lost = 0;
14 bool relayed = false;
15 String codec = '?';
16}
17
18/// One WebRTC session to a host: receives video+audio, sends input on an
19/// unreliable "input" channel, and uses a reliable "control" channel for
20/// everything that must arrive: screen size, keyboard layout, clipboard,
21/// voice, the launcher (the same messages as the browser client).
22class PeerClient {
23 final List<IceServer> ice;
24 final bool relayOnly;
25 final RTCVideoRenderer renderer;
26
27 RTCPeerConnection? _pc;
28 RTCDataChannel? _input;
29 RTCDataChannel? _ctl;
30 RTCDataChannel? _file; // opened with the others: later ones never report open here
31 RTCDataChannel? _fileIn; // files from the desk
32 String? _inName;
33 int _inSize = 0;
34 bool _inPrint = false;
35 BytesBuilder? _inData;
36 Completer<String>? _fileAnswer;
37 Future<void> _fileQueue = Future.value();
38 MediaStream? _mic;
39 // the camera: its own sending video transceiver, opened empty at connect;
40 // the camera is attached only while it is on (the camera light tells the truth)
41 RTCRtpTransceiver? _camTx;
42 MediaStream? _cam;
43 bool _camFront = true;
44 Timer? _statsTimer;
45 int _lastBytes = 0, _lastFrames = 0;
46 DateTime _lastAt = DateTime.now();
47
48 void Function(Map<String, dynamic> candidate)? onIceCandidate;
49 void Function(RTCPeerConnectionState)? onState;
50 void Function(PeerStats)? onStats;
51 void Function()? onControlOpen;
52 /// A file from the desk was saved (its path), or failed (null, reason).
53 void Function(String? path, String note)? onFileReceived;
54 /// A PDF the desk printed to "My device": this device's print dialog.
55 void Function(String path, String name)? onPrint;
56 void Function(Map<String, dynamic>)? onControl;
57
58 PeerClient(this.ice, this.relayOnly, this.renderer);
59
60 Future<void> start() async {
61 final config = {
62 'iceServers': ice.map((e) => e.toMap()).toList(),
63 'sdpSemantics': 'unified-plan',
64 if (relayOnly) 'iceTransportPolicy': 'relay',
65 };
66 final pc = await createPeerConnection(config);
67 _pc = pc;
68
69 pc.onTrack = (event) async {
70 debugPrint('nc track: ${event.track.kind} streams=${event.streams.length}');
71 if (event.track.kind != 'video') return;
72 MediaStream stream;
73 if (event.streams.isNotEmpty) {
74 stream = event.streams[0];
75 } else {
76 // some platforms deliver the track without its stream: wrap it ourselves
77 stream = await createLocalMediaStream('remote-video');
78 await stream.addTrack(event.track);
79 }
80 renderer.srcObject = stream;
81 debugPrint('nc renderer: textureId=${renderer.textureId} src=${renderer.srcObject?.id}');
82 };
83 pc.onIceCandidate = (c) {
84 final m = c.toMap();
85 if (m['candidate'] != null) onIceCandidate?.call(Map<String, dynamic>.from(m));
86 };
87 pc.onConnectionState = (s) {
88 onState?.call(s);
89 if (s == RTCPeerConnectionState.RTCPeerConnectionStateConnected) _startStats();
90 };
91
92 await pc.addTransceiver(
93 kind: RTCRtpMediaType.RTCRtpMediaTypeVideo,
94 init: RTCRtpTransceiverInit(direction: TransceiverDirection.RecvOnly));
95 // The microphone is opened with the connection and attached muted; the mic
96 // button only unmutes it. Attaching a track later (replaceTrack) left the
97 // iPhone sending no audio at all. No permission: the call works without.
98 MediaStreamTrack? micTrack;
99 try {
100 final mic = await navigator.mediaDevices.getUserMedia({
101 'audio': {
102 'echoCancellation': true, // keep the desktop's sound out of the mic
103 'noiseSuppression': true,
104 'autoGainControl': true,
105 },
106 'video': false,
107 });
108 _mic = mic;
109 micTrack = mic.getAudioTracks().first;
110 micTrack.enabled = false;
111 } catch (e) {
112 debugPrint('nc mic: unavailable ($e)');
113 }
114 final init = RTCRtpTransceiverInit(
115 direction: TransceiverDirection.SendRecv, streams: _mic == null ? [] : [_mic!]);
116 if (micTrack != null) {
117 await pc.addTransceiver(track: micTrack, kind: RTCRtpMediaType.RTCRtpMediaTypeAudio, init: init);
118 } else {
119 await pc.addTransceiver(kind: RTCRtpMediaType.RTCRtpMediaTypeAudio, init: init);
120 }
121 _camTx = await pc.addTransceiver(
122 kind: RTCRtpMediaType.RTCRtpMediaTypeVideo,
123 init: RTCRtpTransceiverInit(direction: TransceiverDirection.SendOnly));
124 await _loudspeaker(); // capturing puts iOS in call mode: earpiece otherwise
125
126 _input = await pc.createDataChannel(
127 'input', RTCDataChannelInit()..ordered = false..maxRetransmits = 0);
128 final file = await pc.createDataChannel('file', RTCDataChannelInit()); // reliable, ordered
129 _file = file;
130 file.onMessage = (m) {
131 final a = _fileAnswer;
132 if (a != null && !a.isCompleted && !m.isBinary) a.complete(m.text);
133 };
134 // files from the desk ("Send to my device") arrive on a channel we open
135 final fileIn = await pc.createDataChannel('file-in', RTCDataChannelInit());
136 _fileIn = fileIn;
137 fileIn.onMessage = _onFileIn;
138 final ctl = await pc.createDataChannel('control', RTCDataChannelInit()); // reliable, ordered
139 _ctl = ctl;
140 ctl.onDataChannelState = (s) {
141 if (s == RTCDataChannelState.RTCDataChannelOpen) onControlOpen?.call();
142 };
143 ctl.onMessage = (m) {
144 if (m.isBinary) return;
145 try {
146 final v = jsonDecode(m.text);
147 if (v is Map<String, dynamic>) onControl?.call(v);
148 } catch (_) {}
149 };
150 }
151
152 /// Send a file to the desk (saved on its Desktop): its own reliable channel,
153 /// a JSON header, 16 KB pieces with flow control, then "end"; the desk
154 /// answers "ok NAME" or "error: WHY". Same as the browser's file drop.
155 Future<String> sendFile(String name, Uint8List data, void Function(double) progress) {
156 // one file at a time on the file channel
157 final done = _fileQueue.then((_) => _sendFileNow(name, data, progress));
158 _fileQueue = done.then((_) {}, onError: (_) {});
159 return done;
160 }
161
162 Future<String> _sendFileNow(String name, Uint8List data, void Function(double) progress) async {
163 final ch = _file;
164 if (ch == null || ch.state != RTCDataChannelState.RTCDataChannelOpen) {
165 throw 'the file channel is not open';
166 }
167 final answer = Completer<String>();
168 _fileAnswer = answer;
169 await ch.send(RTCDataChannelMessage(jsonEncode({'name': name, 'size': data.length})));
170 const piece = 16 * 1024;
171 for (var i = 0; i < data.length && !answer.isCompleted; i += piece) {
172 while ((ch.bufferedAmount ?? 0) > 4 * 1024 * 1024) {
173 await Future.delayed(const Duration(milliseconds: 20));
174 }
175 final end = i + piece < data.length ? i + piece : data.length;
176 await ch.send(RTCDataChannelMessage.fromBinary(data.sublist(i, end)));
177 progress(end / data.length);
178 }
179 if (!answer.isCompleted) await ch.send(RTCDataChannelMessage('end'));
180 return answer.future.timeout(const Duration(minutes: 2));
181 }
182
183 /// A file from the desk: header, pieces, "end"; saved in the app's Documents
184 /// (on iPhone: Files › On My iPhone › Network Computer); answers "ok NAME".
185 Future<void> _onFileIn(RTCDataChannelMessage m) async {
186 final ch = _fileIn;
187 if (ch == null) return;
188 if (m.isBinary) {
189 _inData?.add(m.binary);
190 return;
191 }
192 if (m.text != 'end') {
193 try {
194 final h = jsonDecode(m.text) as Map<String, dynamic>;
195 _inName = '${h['name']}'.split('/').last.split('\\').last;
196 _inSize = (h['size'] as num).toInt();
197 _inPrint = h['print'] == true;
198 _inData = BytesBuilder(copy: false);
199 } catch (_) {
200 await ch.send(RTCDataChannelMessage('error: bad header'));
201 }
202 return;
203 }
204 final name = _inName, data = _inData, print = _inPrint;
205 _inName = null; _inData = null; _inPrint = false;
206 if (name == null || data == null) return;
207 if (data.length != _inSize) {
208 await ch.send(RTCDataChannelMessage('error: got ${data.length} of $_inSize bytes'));
209 onFileReceived?.call(null, '$name did not arrive whole');
210 return;
211 }
212 if (print) { // kept only while its dialog is up: a print job, not a document
213 try {
214 final path = '${(await getTemporaryDirectory()).path}/$name';
215 await File(path).writeAsBytes(data.takeBytes());
216 await ch.send(RTCDataChannelMessage('ok $name'));
217 onPrint?.call(path, name);
218 } catch (e) {
219 await ch.send(RTCDataChannelMessage('error: $e'));
220 onFileReceived?.call(null, '$name: $e');
221 }
222 return;
223 }
224 try {
225 final dir = await getApplicationDocumentsDirectory();
226 var path = '${dir.path}/$name';
227 final dot = name.lastIndexOf('.');
228 final stem = dot > 0 ? name.substring(0, dot) : name, ext = dot > 0 ? name.substring(dot) : '';
229 for (var i = 2; File(path).existsSync(); i++) {
230 path = '${dir.path}/$stem ($i)$ext';
231 }
232 await File(path).writeAsBytes(data.takeBytes());
233 await ch.send(RTCDataChannelMessage('ok ${path.split('/').last}'));
234 onFileReceived?.call(path, path.split('/').last);
235 } catch (e) {
236 await ch.send(RTCDataChannelMessage('error: $e'));
237 onFileReceived?.call(null, '$name: $e');
238 }
239 }
240
241 bool get controlOpen => _ctl?.state == RTCDataChannelState.RTCDataChannelOpen;
242
243 /// Send on the reliable control channel (in order, never dropped).
244 void sendControl(Map<String, dynamic> m) {
245 if (controlOpen) _ctl!.send(RTCDataChannelMessage(jsonEncode(m)));
246 }
247
248 Future<String> createOffer() async {
249 final offer = await _pc!.createOffer();
250 await _pc!.setLocalDescription(offer);
251 return offer.sdp!;
252 }
253
254 Future<void> setAnswer(String sdp) async =>
255 _pc!.setRemoteDescription(RTCSessionDescription(sdp, 'answer'));
256
257 Future<void> addCandidate(Map<String, dynamic> c) async => _pc!.addCandidate(
258 RTCIceCandidate(c['candidate'], c['sdpMid'], c['sdpMLineIndex']));
259
260 void send(InputEvent e) {
261 final dc = _input;
262 if (dc != null && dc.state == RTCDataChannelState.RTCDataChannelOpen) {
263 dc.send(RTCDataChannelMessage(jsonEncode(e.toJson())));
264 }
265 }
266
267 /// Mute or unmute the microphone (opened with the connection).
268 Future<void> setMic(bool on) async {
269 final tracks = _mic?.getAudioTracks() ?? <MediaStreamTrack>[];
270 if (tracks.isEmpty) {
271 if (on) throw 'no microphone (allow it in Settings, then reconnect)';
272 return;
273 }
274 for (final t in tracks) {
275 t.enabled = on;
276 }
277 debugPrint('nc mic: ${on ? 'on' : 'off'}');
278 }
279
280 /// Turn this device's camera on or off. On the desk it is the webcam
281 /// "network-computer camera" for any app (a video call in the browser).
282 Future<void> setCamera(bool on) async {
283 final tx = _camTx;
284 if (tx == null) throw 'not connected';
285 if (!on) {
286 await tx.sender.replaceTrack(null);
287 await _closeCamera();
288 debugPrint('nc camera: off');
289 return;
290 }
291 if (_cam != null) return;
292 final cam = await navigator.mediaDevices.getUserMedia({
293 'audio': false,
294 'video': {
295 'facingMode': _camFront ? 'user' : 'environment',
296 'width': {'ideal': 1280},
297 'height': {'ideal': 720},
298 'frameRate': {'ideal': 30},
299 },
300 });
301 _cam = cam;
302 await tx.sender.replaceTrack(cam.getVideoTracks().first);
303 debugPrint('nc camera: on (${_camFront ? 'front' : 'back'})');
304 }
305
306 bool get cameraOn => _cam != null;
307
308 /// Front to back camera and back (phones, tablets).
309 Future<void> flipCamera() async {
310 _camFront = !_camFront;
311 final t = _cam?.getVideoTracks();
312 if (t == null || t.isEmpty) return;
313 await Helper.switchCamera(t.first);
314 }
315
316 Future<void> _closeCamera() async {
317 for (final t in _cam?.getTracks() ?? <MediaStreamTrack>[]) {
318 await t.stop();
319 }
320 await _cam?.dispose();
321 _cam = null;
322 }
323
324 Future<void> _closeMic() async {
325 for (final t in _mic?.getTracks() ?? <MediaStreamTrack>[]) {
326 await t.stop();
327 }
328 await _mic?.dispose();
329 _mic = null;
330 }
331
332 /// The desk's sound on the loudspeaker: iOS otherwise picks the earpiece for
333 /// a call-style session.
334 Future<void> _loudspeaker() async {
335 if (defaultTargetPlatform != TargetPlatform.iOS) return;
336 try {
337 await Helper.setSpeakerphoneOn(true);
338 } catch (e) {
339 debugPrint('nc audio route: $e');
340 }
341 }
342
343 void _startStats() {
344 _statsTimer = Timer.periodic(const Duration(seconds: 1), (_) => _poll());
345 }
346
347 Future<void> _poll() async {
348 final pc = _pc;
349 if (pc == null) return;
350 final reports = await pc.getStats();
351 final s = PeerStats();
352 final cand = <String, StatsReport>{};
353 StatsReport? pair, inbound;
354 for (final r in reports) {
355 switch (r.type) {
356 case 'candidate-pair':
357 if (r.values['nominated'] == true && r.values['state'] == 'succeeded') pair = r;
358 break;
359 case 'local-candidate':
360 case 'remote-candidate':
361 cand[r.id] = r;
362 break;
363 case 'inbound-rtp':
364 if (r.values['kind'] == 'video') inbound = r;
365 break;
366 case 'outbound-rtp':
367 if (r.values['kind'] == 'audio') _micSent = r.values['bytesSent'];
368 break;
369 }
370 }
371 if (pair != null) {
372 final l = cand[pair.values['localCandidateId']]?.values['candidateType'];
373 final rc = cand[pair.values['remoteCandidateId']]?.values['candidateType'];
374 s.relayed = l == 'relay' || rc == 'relay';
375 s.path = '$l → $rc';
376 s.rttMs = ((pair.values['currentRoundTripTime'] as num?) ?? 0).toDouble() * 1000;
377 }
378 if (inbound != null) {
379 final now = DateTime.now();
380 final dt = now.difference(_lastAt).inMilliseconds / 1000.0;
381 final bytes = (inbound.values['bytesReceived'] as num?)?.toInt() ?? 0;
382 final frames = (inbound.values['framesDecoded'] as num?)?.toInt() ?? 0;
383 s.lost = (inbound.values['packetsLost'] as num?)?.toInt() ?? 0;
384 if (dt > 0) {
385 s.mbps = (bytes - _lastBytes) * 8 / dt / 1e6;
386 s.fps = (frames - _lastFrames) / dt;
387 }
388 _lastBytes = bytes; _lastFrames = frames; _lastAt = now;
389 s.width = (inbound.values['frameWidth'] as num?)?.toInt() ?? 0;
390 s.height = (inbound.values['frameHeight'] as num?)?.toInt() ?? 0;
391 s.jitterMs = ((inbound.values['jitter'] as num?) ?? 0).toDouble() * 1000;
392 }
393 onStats?.call(s);
394 if (!kReleaseMode && ++_polls % 5 == 0 && inbound != null) { // every 5 s in debug/profile runs
395 final v = inbound.values;
396 debugPrint('nc video: bytes=${v['bytesReceived']} packets=${v['packetsReceived']} '
397 'framesReceived=${v['framesReceived']} decoded=${v['framesDecoded']} dropped=${v['framesDropped']} '
398 'size=${v['frameWidth']}x${v['frameHeight']} decoder=${v['decoderImplementation']} '
399 'keyframes=${v['keyFramesDecoded']} pli=${v['pliCount']} path=${s.path} '
400 'view=${renderer.videoWidth}x${renderer.videoHeight} mic-bytes-sent=$_micSent');
401 }
402 }
403 int _polls = 0;
404 Object? _micSent;
405
406 Future<void> close() async {
407 _statsTimer?.cancel();
408 await _closeMic();
409 await _closeCamera();
410 await _input?.close();
411 await _file?.close();
412 await _fileIn?.close();
413 await _ctl?.close();
414 await _pc?.close();
415 renderer.srcObject = null;
416 }
417}