One Flutter client for network-computer: drive a remote Linux/Mac desktop from Android, iPhone, iPad, or desktop over WebRTC.

0 downloads

network-computer-flutter/lib/session_store.dart
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/network-computer-flutter.git
HTTPS
https://gitstable.com/git/askrobots/network-computer-flutter.git
SSH
ssh://git@gitstable.com:2222/askrobots/network-computer-flutter.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

Dan Quellhorst No colour (compare) in ⋯: the desk's picture in grey on this device c74471a 1 month, 3 weeks ago History
session_store.dart · 15.4 KB · 459 lines
1import 'dart:async';
2import 'dart:io';
3import 'package:flutter/foundation.dart';
4import 'package:flutter/services.dart';
5import 'package:file_picker/file_picker.dart';
6import 'package:flutter_webrtc/flutter_webrtc.dart';
7import 'package:printing/printing.dart';
8import 'package:shared_preferences/shared_preferences.dart';
9import 'protocol.dart';
10import 'signaling.dart';
11import 'secrets.dart';
12import 'peer.dart';
13
14enum ConnState { idle, connecting, connected, failed }
15
16/// App-wide connection state. Handles connect, auto-reconnect and input send.
17class SessionStore extends ChangeNotifier {
18 final renderer = RTCVideoRenderer();
19 ConnState state = ConnState.idle;
20 String status = '';
21 List<String> hosts = [];
22 PeerStats stats = PeerStats();
23 bool micOn = false;
24 bool cameraOn = false;
25
26 // voice: the desk listens (nc-voice); here only the button and the transcript
27 bool voiceOn = false, voicePanel = false, keepListening = false;
28 String voiceState = 'Voice off';
29 final List<(String, String)> voiceLines = []; // (kind, text): heard, said, did, error
30 bool _micByVoice = false;
31 Timer? _micOffTimer;
32
33 /// A short message for the user (shown briefly by the session screen).
34 String notice = '';
35 int noticeSeq = 0;
36
37 // screen size chosen for the desk: null keeps the host's default
38 int? displayW, displayH;
39 double displayScale = 1;
40 String _clipIn = '';
41
42 SignalingClient? _sig;
43 PeerClient? _peer;
44 Endpoint? _ep;
45 String _host = '', _pin = '';
46 bool _relayOnly = false, _wantRetry = false;
47 int _retries = 0;
48
49 Future<void> init() async => renderer.initialize();
50
51 String mode = 'insecure';
52
53 Future<void> refreshHosts(Endpoint ep) async {
54 try {
55 final c = SignalingClient(ep);
56 await c.authenticate();
57 mode = c.mode;
58 hosts = await c.fetchHosts();
59 notifyListeners();
60 } catch (e) {
61 _fail('$e');
62 }
63 }
64
65 Future<void> connect(Endpoint ep, String host, String pin, bool relayOnly) async {
66 _ep = ep; _host = host; _pin = pin; _relayOnly = relayOnly;
67 _wantRetry = true; _retries = 0;
68 await _open();
69 }
70
71 Future<void> _open() async {
72 _set(ConnState.connecting, 'fetching config');
73 try {
74 final sig = SignalingClient(_ep!);
75 await sig.authenticate();
76 mode = sig.mode;
77 final ice = await sig.fetchIceServers();
78 final peer = PeerClient(ice, _relayOnly, renderer);
79 _sig = sig; _peer = peer;
80
81 peer.onIceCandidate = (c) =>
82 sig.send(SignalMessage(type: 'ice', to: _host, candidate: c));
83 peer.onState = (s) {
84 if (s == RTCPeerConnectionState.RTCPeerConnectionStateConnected) {
85 _retries = 0;
86 _set(ConnState.connected, '');
87 } else if (s == RTCPeerConnectionState.RTCPeerConnectionStateFailed ||
88 s == RTCPeerConnectionState.RTCPeerConnectionStateDisconnected) {
89 _dropped();
90 }
91 };
92 peer.onStats = (st) { stats = st; notifyListeners(); };
93 peer.onControlOpen = _controlOpen;
94 peer.onFileReceived = (path, note) {
95 debugPrint('nc file in: $note -> $path');
96 _say(path == null ? 'Not saved: $note'
97 : defaultTargetPlatform == TargetPlatform.iOS ? 'From the desk: $note (Files › Network Computer)'
98 : 'From the desk: $note ($path)');
99 };
100 peer.onPrint = _print;
101 peer.onControl = _onControl;
102
103 // Candidates can arrive before the answer is applied (saving the pairing
104 // token is async): hold them until it is, or they are rejected and lost.
105 var answered = false;
106 final early = <Map<String, dynamic>>[];
107 sig.onMessage = (m) async {
108 switch (m.type) {
109 case 'answer':
110 if (m.sdp != null) await peer.setAnswer(m.sdp!);
111 answered = true;
112 for (final c in early) {
113 await peer.addCandidate(c);
114 }
115 early.clear();
116 if (m.pair != null) await _savePair(m.pair!);
117 break;
118 case 'ice':
119 if (m.candidate == null) break;
120 if (answered) {
121 await peer.addCandidate(m.candidate!);
122 } else {
123 early.add(m.candidate!);
124 }
125 break;
126 case 'error':
127 final err = (m.error ?? '').toLowerCase();
128 if (err.contains('pairing')) await _clearPair();
129 _fail(m.error ?? 'rendezvous error');
130 if (err.contains('pin')) _wantRetry = false;
131 break;
132 }
133 };
134 sig.onClose = (_) {
135 if (state == ConnState.connecting) _dropped();
136 };
137
138 await peer.start();
139 _set(ConnState.connecting, 'signaling');
140 await sig.connect();
141 final sdp = await peer.createOffer();
142 final pair = await _loadPair();
143 sig.send(SignalMessage(
144 type: 'offer', to: _host, sdp: sdp,
145 pin: _pin.isEmpty ? null : _pin, pair: pair));
146 _set(ConnState.connecting, 'waiting for $_host');
147 } catch (e) {
148 _fail('$e');
149 await _teardown();
150 }
151 }
152
153 void _dropped() {
154 if (!_wantRetry) return;
155 _set(ConnState.connecting, 'reconnecting…');
156 _teardown();
157 _retries++;
158 final delay = Duration(milliseconds: (1000 * _retries).clamp(1000, 5000));
159 Future.delayed(delay, () { if (_wantRetry) _open(); });
160 }
161
162 Future<void> disconnect() async {
163 _wantRetry = false;
164 await _teardown();
165 _set(ConnState.idle, '');
166 }
167
168 Future<void> _teardown() async {
169 micOn = false;
170 cameraOn = false;
171 voiceOn = false;
172 _micByVoice = false;
173 _micOffTimer?.cancel();
174 await _peer?.close();
175 _peer = null;
176 _sig?.close();
177 _sig = null;
178 }
179
180 void send(InputEvent e) => _peer?.send(e);
181
182 /// The desk printed to "My device": this device's own print dialog
183 /// (AirPrint on an iPhone or iPad, the print panel on a Mac).
184 Future<void> _print(String path, String name) async {
185 final file = File(path);
186 debugPrint('nc print: $name');
187 try {
188 final bytes = await file.readAsBytes();
189 final done = await Printing.layoutPdf(
190 onLayout: (_) async => bytes, name: name,
191 // the desk already laid it out; dynamic layout also deadlocks on macOS
192 dynamicLayout: false);
193 debugPrint('nc print: $name done=$done');
194 _say(done ? 'Printed $name' : 'Not printed: $name');
195 } catch (e) {
196 debugPrint('nc print: $name failed: $e');
197 _say('Could not print $name: $e');
198 } finally {
199 if (await file.exists()) await file.delete();
200 }
201 }
202
203 void _say(String msg) { notice = msg; noticeSeq++; notifyListeners(); }
204
205 String? _inputKind;
206
207 /// Tells the desk whether the person uses touch or a mouse/trackpad right
208 /// now (the dbbasic apps size their controls by it); only changes are sent.
209 void reportInput(String kind) {
210 if (kind == _inputKind) return;
211 _inputKind = kind;
212 _peer?.sendControl({'t': 'input', 'kind': kind, 'device': deviceClass});
213 }
214
215 /// phone, tablet or desktop: the desk lays itself out for it (nc-device).
216 /// Set by the session screen from its size (shortest side under 600 is a phone).
217 String deviceClass = Platform.isIOS || Platform.isAndroid ? 'phone' : 'desktop';
218
219 Future<void> _controlOpen() async {
220 // keys arrive as US positions (see the session screen's key map)
221 _peer?.sendControl({'t': 'keyboard', 'layout': 'us'});
222 // the desk's clock follows this device (Etc/GMT+4 is UTC-4: the sign is inverted)
223 final off = DateTime.now().timeZoneOffset;
224 if (off.inMinutes % 60 == 0) {
225 final h = -off.inHours;
226 _peer?.sendControl({'t': 'tz', 'text': h == 0 ? 'Etc/UTC' : 'Etc/GMT${h > 0 ? '+' : ''}$h'});
227 }
228 // what the person is using, until the first press says otherwise
229 _inputKind = null;
230 reportInput(Platform.isIOS || Platform.isAndroid ? 'touch' : 'pointer');
231 final p = await SharedPreferences.getInstance();
232 keepListening = p.getBool('voiceKeep') ?? false;
233 final w = p.getInt('displayW'), h = p.getInt('displayH');
234 if (w != null && h != null) {
235 setDisplay(w, h, p.getDouble('displayScale') ?? 1, save: false);
236 }
237 voiceOn = false;
238 notifyListeners();
239 // Developer test hook (--dart-define=NC_AUTO_VOICE=true): press the voice
240 // button once, 3 s after connecting, so the mic path can be checked
241 // end to end without a hand on the screen.
242 if (const bool.fromEnvironment('NC_AUTO_VOICE')) {
243 Future.delayed(const Duration(seconds: 3), toggleVoice);
244 }
245 // (--dart-define=NC_AUTO_SEND=/path/file): send that file after connecting.
246 const autoSend = String.fromEnvironment('NC_AUTO_SEND');
247 if (autoSend.isNotEmpty) {
248 Future.delayed(const Duration(seconds: 2), () async {
249 final data = File(autoSend).readAsBytesSync();
250 await _sendBytes(autoSend.split('/').last, data); // twice: the channel is reused
251 await _sendBytes(autoSend.split('/').last, data);
252 });
253 }
254 }
255
256 void _onControl(Map<String, dynamic> m) {
257 switch (m['t']) {
258 case 'voice':
259 _onVoice('${m['kind'] ?? ''}', '${m['text'] ?? ''}');
260 break;
261 case 'clip': // the desk's clipboard, in pieces
262 _clipIn += '${m['text'] ?? ''}';
263 if (m['more'] == true) return;
264 final t = _clipIn;
265 _clipIn = '';
266 Clipboard.setData(ClipboardData(text: t));
267 _say('Copied from the desk');
268 break;
269 case 'clipnone':
270 _say("The desk's clipboard has no text");
271 break;
272 }
273 }
274
275 // ---------- voice ----------
276 static const _states = {
277 'listening': 'Listening…', 'hearing': 'Hearing you…',
278 'thinking': 'Thinking…', 'off': 'Voice off',
279 };
280
281 void _onVoice(String kind, String text) {
282 voicePanel = true;
283 if (kind == 'state') {
284 voiceState = _states[text] ?? text;
285 if (text == 'off') _voiceStopped();
286 } else {
287 voiceLines.add((kind, text));
288 if (voiceLines.length > 40) voiceLines.removeAt(0);
289 if (kind == 'error' && text.contains('not running')) _voiceStopped();
290 }
291 notifyListeners();
292 }
293
294 void _voiceStopped() {
295 voiceOn = false;
296 voiceState = 'Voice off';
297 if (_micByVoice) {
298 // the reply's last words are still on their way: stop the mic a bit later
299 _micOffTimer?.cancel();
300 _micOffTimer = Timer(const Duration(milliseconds: 1500), () async {
301 if (!voiceOn && _micByVoice && micOn) {
302 _micByVoice = false;
303 await toggleMic();
304 }
305 });
306 }
307 }
308
309 /// One tap, one phrase (or continuous with keepListening). Turns the mic on if needed.
310 Future<void> toggleVoice() async {
311 final peer = _peer;
312 if (peer == null || !peer.controlOpen) return;
313 _micOffTimer?.cancel();
314 if (!voiceOn) {
315 if (!micOn) {
316 await toggleMic();
317 if (!micOn) return;
318 _micByVoice = true;
319 // the phone's audio takes a moment to start flowing: don't let the
320 // desk start listening (and its 8 s timeout) before it does
321 await Future.delayed(const Duration(milliseconds: 600));
322 }
323 voiceOn = true;
324 voicePanel = true;
325 voiceState = keepListening ? 'Starting…' : 'Say one thing…';
326 peer.sendControl({'t': 'voice', 'on': true, 'once': !keepListening});
327 } else {
328 peer.sendControl({'t': 'voice', 'on': false});
329 _voiceStopped();
330 }
331 notifyListeners();
332 }
333
334 Future<void> setKeepListening(bool v) async {
335 keepListening = v;
336 (await SharedPreferences.getInstance()).setBool('voiceKeep', v);
337 notifyListeners();
338 }
339
340 void hideVoicePanel() { voicePanel = false; notifyListeners(); }
341
342 // ---------- launcher, screen, clipboard ----------
343 void launch() => _peer?.sendControl({'t': 'launch'});
344
345 Future<void> setDisplay(int w, int h, double scale, {bool save = true}) async {
346 displayW = w; displayH = h; displayScale = scale;
347 _peer?.sendControl({'t': 'display', 'w': w, 'h': h, 's': scale});
348 if (save) {
349 final p = await SharedPreferences.getInstance();
350 await p.setInt('displayW', w);
351 await p.setInt('displayH', h);
352 await p.setDouble('displayScale', scale);
353 _say('Screen $w×$h at ${(scale * 100).round()}%');
354 }
355 notifyListeners();
356 }
357
358 /// This device's clipboard to the desk (then paste there as usual).
359 Future<void> sendClipboard() async {
360 final data = await Clipboard.getData(Clipboard.kTextPlain);
361 final text = data?.text ?? '';
362 if (text.isEmpty) { _say('No text on this clipboard'); return; }
363 const piece = 4096;
364 var i = 0;
365 while (i < text.length) {
366 var end = i + piece < text.length ? i + piece : text.length;
367 // keep surrogate pairs together
368 if (end < text.length && (text.codeUnitAt(end - 1) & 0xFC00) == 0xD800) end--;
369 _peer?.sendControl({'t': 'clip', 'text': text.substring(i, end), 'more': end < text.length});
370 i = end;
371 }
372 _say('Clipboard sent to the desk');
373 }
374
375 /// Photos, videos or files from this device to the desk's Desktop.
376 Future<void> sendFiles() async {
377 final peer = _peer;
378 if (peer == null) return;
379 final picked = await FilePicker.pickFiles(allowMultiple: true, withData: true);
380 if (picked == null) return;
381 for (final f in picked.files) {
382 final data = f.bytes;
383 if (data == null) { _say('${f.name}: could not read it'); continue; }
384 await _sendBytes(f.name, data);
385 }
386 }
387
388 Future<void> _sendBytes(String name, Uint8List data) async {
389 final peer = _peer;
390 if (peer == null) return;
391 try {
392 var shown = 0;
393 final r = await peer.sendFile(name, data, (p) {
394 final pct = (p * 100).round();
395 if (pct >= shown + 20) { shown = pct; _say('Sending $name: $pct%'); }
396 });
397 debugPrint('nc file: $name -> $r');
398 _say(r.startsWith('ok ') ? "On the desk's Desktop: ${r.substring(3)}" : '$name: $r');
399 } catch (e) {
400 debugPrint('nc file: $name failed: $e');
401 _say('$name: $e');
402 }
403 }
404
405 /// The desk's clipboard to this device.
406 void getClipboard() => _peer?.sendControl({'t': 'clipnow'});
407
408 /// This device's camera as the desk's webcam ("network-computer camera").
409 Future<void> toggleCamera() async {
410 final want = !cameraOn;
411 try {
412 await _peer?.setCamera(want);
413 cameraOn = want;
414 if (want) _say('Camera on: pick "network-computer camera" in the app on the desk');
415 } catch (e) {
416 cameraOn = false;
417 await _peer?.setCamera(false).catchError((_) {});
418 _say('Camera unavailable: $e');
419 }
420 notifyListeners();
421 }
422
423 Future<void> flipCamera() async {
424 try {
425 await _peer?.flipCamera();
426 } catch (e) {
427 _say('Could not switch cameras: $e');
428 }
429 }
430
431 Future<void> toggleMic() async {
432 final want = !micOn;
433 try {
434 await _peer?.setMic(want);
435 micOn = want;
436 } catch (e) {
437 micOn = false;
438 status = 'Microphone unavailable: $e';
439 }
440 notifyListeners();
441 }
442
443 // Pairing: after a correct PIN the host returns a signed token; keep one per
444 // rendezvous+host and offer it next time so the PIN is typed once.
445 String _pairKey() => 'pair:${_ep!.base}|$_host';
446 Future<String?> _loadPair() => Secrets.read(_pairKey());
447 Future<void> _savePair(String t) => Secrets.write(_pairKey(), t);
448 Future<void> _clearPair() => Secrets.delete(_pairKey());
449
450 /// Whether a pairing token is stored for [ep] + [host] (for the UI hint).
451 static Future<bool> isPaired(Endpoint ep, String host) async =>
452 await Secrets.read('pair:${ep.base}|$host') != null;
453
454 void _set(ConnState s, String msg) { state = s; status = msg; notifyListeners(); }
455 void _fail(String msg) { state = ConnState.failed; status = msg; notifyListeners(); }
456
457 @override
458 void dispose() { _teardown(); renderer.dispose(); super.dispose(); }
459}