askrobots / network-computer-flutter
Public
One Flutter client for network-computer: drive a remote Linux/Mac desktop from Android, iPhone, iPad, or desktop over WebRTC.
Dan Quellhorst
No colour (compare) in ⋯: the desk's picture in grey on this device
c74471a
1 month, 3 weeks ago
History
session_store.dart · 15.4 KB · 459 lines
| 1 | import 'dart:async'; |
| 2 | import 'dart:io'; |
| 3 | import 'package:flutter/foundation.dart'; |
| 4 | import 'package:flutter/services.dart'; |
| 5 | import 'package:file_picker/file_picker.dart'; |
| 6 | import 'package:flutter_webrtc/flutter_webrtc.dart'; |
| 7 | import 'package:printing/printing.dart'; |
| 8 | import 'package:shared_preferences/shared_preferences.dart'; |
| 9 | import 'protocol.dart'; |
| 10 | import 'signaling.dart'; |
| 11 | import 'secrets.dart'; |
| 12 | import 'peer.dart'; |
| 13 | |
| 14 | enum ConnState { idle, connecting, connected, failed } |
| 15 | |
| 16 | /// App-wide connection state. Handles connect, auto-reconnect and input send. |
| 17 | class SessionStore extends ChangeNotifier { |
| 18 | final renderer = RTCVideoRenderer(); |
| 19 | ConnState state = ConnState.idle; |
| 20 | String status = ''; |
| 21 | List<String> hosts = []; |
| 22 | PeerStats stats = PeerStats(); |
| 23 | bool micOn = false; |
| 24 | bool cameraOn = false; |
| 25 | |
| 26 | // voice: the desk listens (nc-voice); here only the button and the transcript |
| 27 | bool voiceOn = false, voicePanel = false, keepListening = false; |
| 28 | String voiceState = 'Voice off'; |
| 29 | final List<(String, String)> voiceLines = []; // (kind, text): heard, said, did, error |
| 30 | bool _micByVoice = false; |
| 31 | Timer? _micOffTimer; |
| 32 | |
| 33 | /// A short message for the user (shown briefly by the session screen). |
| 34 | String notice = ''; |
| 35 | int noticeSeq = 0; |
| 36 | |
| 37 | // screen size chosen for the desk: null keeps the host's default |
| 38 | int? displayW, displayH; |
| 39 | double displayScale = 1; |
| 40 | String _clipIn = ''; |
| 41 | |
| 42 | SignalingClient? _sig; |
| 43 | PeerClient? _peer; |
| 44 | Endpoint? _ep; |
| 45 | String _host = '', _pin = ''; |
| 46 | bool _relayOnly = false, _wantRetry = false; |
| 47 | int _retries = 0; |
| 48 | |
| 49 | Future<void> init() async => renderer.initialize(); |
| 50 | |
| 51 | String mode = 'insecure'; |
| 52 | |
| 53 | Future<void> refreshHosts(Endpoint ep) async { |
| 54 | try { |
| 55 | final c = SignalingClient(ep); |
| 56 | await c.authenticate(); |
| 57 | mode = c.mode; |
| 58 | hosts = await c.fetchHosts(); |
| 59 | notifyListeners(); |
| 60 | } catch (e) { |
| 61 | _fail('$e'); |
| 62 | } |
| 63 | } |
| 64 | |
| 65 | Future<void> connect(Endpoint ep, String host, String pin, bool relayOnly) async { |
| 66 | _ep = ep; _host = host; _pin = pin; _relayOnly = relayOnly; |
| 67 | _wantRetry = true; _retries = 0; |
| 68 | await _open(); |
| 69 | } |
| 70 | |
| 71 | Future<void> _open() async { |
| 72 | _set(ConnState.connecting, 'fetching config'); |
| 73 | try { |
| 74 | final sig = SignalingClient(_ep!); |
| 75 | await sig.authenticate(); |
| 76 | mode = sig.mode; |
| 77 | final ice = await sig.fetchIceServers(); |
| 78 | final peer = PeerClient(ice, _relayOnly, renderer); |
| 79 | _sig = sig; _peer = peer; |
| 80 | |
| 81 | peer.onIceCandidate = (c) => |
| 82 | sig.send(SignalMessage(type: 'ice', to: _host, candidate: c)); |
| 83 | peer.onState = (s) { |
| 84 | if (s == RTCPeerConnectionState.RTCPeerConnectionStateConnected) { |
| 85 | _retries = 0; |
| 86 | _set(ConnState.connected, ''); |
| 87 | } else if (s == RTCPeerConnectionState.RTCPeerConnectionStateFailed || |
| 88 | s == RTCPeerConnectionState.RTCPeerConnectionStateDisconnected) { |
| 89 | _dropped(); |
| 90 | } |
| 91 | }; |
| 92 | peer.onStats = (st) { stats = st; notifyListeners(); }; |
| 93 | peer.onControlOpen = _controlOpen; |
| 94 | peer.onFileReceived = (path, note) { |
| 95 | debugPrint('nc file in: $note -> $path'); |
| 96 | _say(path == null ? 'Not saved: $note' |
| 97 | : defaultTargetPlatform == TargetPlatform.iOS ? 'From the desk: $note (Files › Network Computer)' |
| 98 | : 'From the desk: $note ($path)'); |
| 99 | }; |
| 100 | peer.onPrint = _print; |
| 101 | peer.onControl = _onControl; |
| 102 | |
| 103 | // Candidates can arrive before the answer is applied (saving the pairing |
| 104 | // token is async): hold them until it is, or they are rejected and lost. |
| 105 | var answered = false; |
| 106 | final early = <Map<String, dynamic>>[]; |
| 107 | sig.onMessage = (m) async { |
| 108 | switch (m.type) { |
| 109 | case 'answer': |
| 110 | if (m.sdp != null) await peer.setAnswer(m.sdp!); |
| 111 | answered = true; |
| 112 | for (final c in early) { |
| 113 | await peer.addCandidate(c); |
| 114 | } |
| 115 | early.clear(); |
| 116 | if (m.pair != null) await _savePair(m.pair!); |
| 117 | break; |
| 118 | case 'ice': |
| 119 | if (m.candidate == null) break; |
| 120 | if (answered) { |
| 121 | await peer.addCandidate(m.candidate!); |
| 122 | } else { |
| 123 | early.add(m.candidate!); |
| 124 | } |
| 125 | break; |
| 126 | case 'error': |
| 127 | final err = (m.error ?? '').toLowerCase(); |
| 128 | if (err.contains('pairing')) await _clearPair(); |
| 129 | _fail(m.error ?? 'rendezvous error'); |
| 130 | if (err.contains('pin')) _wantRetry = false; |
| 131 | break; |
| 132 | } |
| 133 | }; |
| 134 | sig.onClose = (_) { |
| 135 | if (state == ConnState.connecting) _dropped(); |
| 136 | }; |
| 137 | |
| 138 | await peer.start(); |
| 139 | _set(ConnState.connecting, 'signaling'); |
| 140 | await sig.connect(); |
| 141 | final sdp = await peer.createOffer(); |
| 142 | final pair = await _loadPair(); |
| 143 | sig.send(SignalMessage( |
| 144 | type: 'offer', to: _host, sdp: sdp, |
| 145 | pin: _pin.isEmpty ? null : _pin, pair: pair)); |
| 146 | _set(ConnState.connecting, 'waiting for $_host'); |
| 147 | } catch (e) { |
| 148 | _fail('$e'); |
| 149 | await _teardown(); |
| 150 | } |
| 151 | } |
| 152 | |
| 153 | void _dropped() { |
| 154 | if (!_wantRetry) return; |
| 155 | _set(ConnState.connecting, 'reconnecting…'); |
| 156 | _teardown(); |
| 157 | _retries++; |
| 158 | final delay = Duration(milliseconds: (1000 * _retries).clamp(1000, 5000)); |
| 159 | Future.delayed(delay, () { if (_wantRetry) _open(); }); |
| 160 | } |
| 161 | |
| 162 | Future<void> disconnect() async { |
| 163 | _wantRetry = false; |
| 164 | await _teardown(); |
| 165 | _set(ConnState.idle, ''); |
| 166 | } |
| 167 | |
| 168 | Future<void> _teardown() async { |
| 169 | micOn = false; |
| 170 | cameraOn = false; |
| 171 | voiceOn = false; |
| 172 | _micByVoice = false; |
| 173 | _micOffTimer?.cancel(); |
| 174 | await _peer?.close(); |
| 175 | _peer = null; |
| 176 | _sig?.close(); |
| 177 | _sig = null; |
| 178 | } |
| 179 | |
| 180 | void send(InputEvent e) => _peer?.send(e); |
| 181 | |
| 182 | /// The desk printed to "My device": this device's own print dialog |
| 183 | /// (AirPrint on an iPhone or iPad, the print panel on a Mac). |
| 184 | Future<void> _print(String path, String name) async { |
| 185 | final file = File(path); |
| 186 | debugPrint('nc print: $name'); |
| 187 | try { |
| 188 | final bytes = await file.readAsBytes(); |
| 189 | final done = await Printing.layoutPdf( |
| 190 | onLayout: (_) async => bytes, name: name, |
| 191 | // the desk already laid it out; dynamic layout also deadlocks on macOS |
| 192 | dynamicLayout: false); |
| 193 | debugPrint('nc print: $name done=$done'); |
| 194 | _say(done ? 'Printed $name' : 'Not printed: $name'); |
| 195 | } catch (e) { |
| 196 | debugPrint('nc print: $name failed: $e'); |
| 197 | _say('Could not print $name: $e'); |
| 198 | } finally { |
| 199 | if (await file.exists()) await file.delete(); |
| 200 | } |
| 201 | } |
| 202 | |
| 203 | void _say(String msg) { notice = msg; noticeSeq++; notifyListeners(); } |
| 204 | |
| 205 | String? _inputKind; |
| 206 | |
| 207 | /// Tells the desk whether the person uses touch or a mouse/trackpad right |
| 208 | /// now (the dbbasic apps size their controls by it); only changes are sent. |
| 209 | void reportInput(String kind) { |
| 210 | if (kind == _inputKind) return; |
| 211 | _inputKind = kind; |
| 212 | _peer?.sendControl({'t': 'input', 'kind': kind, 'device': deviceClass}); |
| 213 | } |
| 214 | |
| 215 | /// phone, tablet or desktop: the desk lays itself out for it (nc-device). |
| 216 | /// Set by the session screen from its size (shortest side under 600 is a phone). |
| 217 | String deviceClass = Platform.isIOS || Platform.isAndroid ? 'phone' : 'desktop'; |
| 218 | |
| 219 | Future<void> _controlOpen() async { |
| 220 | // keys arrive as US positions (see the session screen's key map) |
| 221 | _peer?.sendControl({'t': 'keyboard', 'layout': 'us'}); |
| 222 | // the desk's clock follows this device (Etc/GMT+4 is UTC-4: the sign is inverted) |
| 223 | final off = DateTime.now().timeZoneOffset; |
| 224 | if (off.inMinutes % 60 == 0) { |
| 225 | final h = -off.inHours; |
| 226 | _peer?.sendControl({'t': 'tz', 'text': h == 0 ? 'Etc/UTC' : 'Etc/GMT${h > 0 ? '+' : ''}$h'}); |
| 227 | } |
| 228 | // what the person is using, until the first press says otherwise |
| 229 | _inputKind = null; |
| 230 | reportInput(Platform.isIOS || Platform.isAndroid ? 'touch' : 'pointer'); |
| 231 | final p = await SharedPreferences.getInstance(); |
| 232 | keepListening = p.getBool('voiceKeep') ?? false; |
| 233 | final w = p.getInt('displayW'), h = p.getInt('displayH'); |
| 234 | if (w != null && h != null) { |
| 235 | setDisplay(w, h, p.getDouble('displayScale') ?? 1, save: false); |
| 236 | } |
| 237 | voiceOn = false; |
| 238 | notifyListeners(); |
| 239 | // Developer test hook (--dart-define=NC_AUTO_VOICE=true): press the voice |
| 240 | // button once, 3 s after connecting, so the mic path can be checked |
| 241 | // end to end without a hand on the screen. |
| 242 | if (const bool.fromEnvironment('NC_AUTO_VOICE')) { |
| 243 | Future.delayed(const Duration(seconds: 3), toggleVoice); |
| 244 | } |
| 245 | // (--dart-define=NC_AUTO_SEND=/path/file): send that file after connecting. |
| 246 | const autoSend = String.fromEnvironment('NC_AUTO_SEND'); |
| 247 | if (autoSend.isNotEmpty) { |
| 248 | Future.delayed(const Duration(seconds: 2), () async { |
| 249 | final data = File(autoSend).readAsBytesSync(); |
| 250 | await _sendBytes(autoSend.split('/').last, data); // twice: the channel is reused |
| 251 | await _sendBytes(autoSend.split('/').last, data); |
| 252 | }); |
| 253 | } |
| 254 | } |
| 255 | |
| 256 | void _onControl(Map<String, dynamic> m) { |
| 257 | switch (m['t']) { |
| 258 | case 'voice': |
| 259 | _onVoice('${m['kind'] ?? ''}', '${m['text'] ?? ''}'); |
| 260 | break; |
| 261 | case 'clip': // the desk's clipboard, in pieces |
| 262 | _clipIn += '${m['text'] ?? ''}'; |
| 263 | if (m['more'] == true) return; |
| 264 | final t = _clipIn; |
| 265 | _clipIn = ''; |
| 266 | Clipboard.setData(ClipboardData(text: t)); |
| 267 | _say('Copied from the desk'); |
| 268 | break; |
| 269 | case 'clipnone': |
| 270 | _say("The desk's clipboard has no text"); |
| 271 | break; |
| 272 | } |
| 273 | } |
| 274 | |
| 275 | // ---------- voice ---------- |
| 276 | static const _states = { |
| 277 | 'listening': 'Listening…', 'hearing': 'Hearing you…', |
| 278 | 'thinking': 'Thinking…', 'off': 'Voice off', |
| 279 | }; |
| 280 | |
| 281 | void _onVoice(String kind, String text) { |
| 282 | voicePanel = true; |
| 283 | if (kind == 'state') { |
| 284 | voiceState = _states[text] ?? text; |
| 285 | if (text == 'off') _voiceStopped(); |
| 286 | } else { |
| 287 | voiceLines.add((kind, text)); |
| 288 | if (voiceLines.length > 40) voiceLines.removeAt(0); |
| 289 | if (kind == 'error' && text.contains('not running')) _voiceStopped(); |
| 290 | } |
| 291 | notifyListeners(); |
| 292 | } |
| 293 | |
| 294 | void _voiceStopped() { |
| 295 | voiceOn = false; |
| 296 | voiceState = 'Voice off'; |
| 297 | if (_micByVoice) { |
| 298 | // the reply's last words are still on their way: stop the mic a bit later |
| 299 | _micOffTimer?.cancel(); |
| 300 | _micOffTimer = Timer(const Duration(milliseconds: 1500), () async { |
| 301 | if (!voiceOn && _micByVoice && micOn) { |
| 302 | _micByVoice = false; |
| 303 | await toggleMic(); |
| 304 | } |
| 305 | }); |
| 306 | } |
| 307 | } |
| 308 | |
| 309 | /// One tap, one phrase (or continuous with keepListening). Turns the mic on if needed. |
| 310 | Future<void> toggleVoice() async { |
| 311 | final peer = _peer; |
| 312 | if (peer == null || !peer.controlOpen) return; |
| 313 | _micOffTimer?.cancel(); |
| 314 | if (!voiceOn) { |
| 315 | if (!micOn) { |
| 316 | await toggleMic(); |
| 317 | if (!micOn) return; |
| 318 | _micByVoice = true; |
| 319 | // the phone's audio takes a moment to start flowing: don't let the |
| 320 | // desk start listening (and its 8 s timeout) before it does |
| 321 | await Future.delayed(const Duration(milliseconds: 600)); |
| 322 | } |
| 323 | voiceOn = true; |
| 324 | voicePanel = true; |
| 325 | voiceState = keepListening ? 'Starting…' : 'Say one thing…'; |
| 326 | peer.sendControl({'t': 'voice', 'on': true, 'once': !keepListening}); |
| 327 | } else { |
| 328 | peer.sendControl({'t': 'voice', 'on': false}); |
| 329 | _voiceStopped(); |
| 330 | } |
| 331 | notifyListeners(); |
| 332 | } |
| 333 | |
| 334 | Future<void> setKeepListening(bool v) async { |
| 335 | keepListening = v; |
| 336 | (await SharedPreferences.getInstance()).setBool('voiceKeep', v); |
| 337 | notifyListeners(); |
| 338 | } |
| 339 | |
| 340 | void hideVoicePanel() { voicePanel = false; notifyListeners(); } |
| 341 | |
| 342 | // ---------- launcher, screen, clipboard ---------- |
| 343 | void launch() => _peer?.sendControl({'t': 'launch'}); |
| 344 | |
| 345 | Future<void> setDisplay(int w, int h, double scale, {bool save = true}) async { |
| 346 | displayW = w; displayH = h; displayScale = scale; |
| 347 | _peer?.sendControl({'t': 'display', 'w': w, 'h': h, 's': scale}); |
| 348 | if (save) { |
| 349 | final p = await SharedPreferences.getInstance(); |
| 350 | await p.setInt('displayW', w); |
| 351 | await p.setInt('displayH', h); |
| 352 | await p.setDouble('displayScale', scale); |
| 353 | _say('Screen $w×$h at ${(scale * 100).round()}%'); |
| 354 | } |
| 355 | notifyListeners(); |
| 356 | } |
| 357 | |
| 358 | /// This device's clipboard to the desk (then paste there as usual). |
| 359 | Future<void> sendClipboard() async { |
| 360 | final data = await Clipboard.getData(Clipboard.kTextPlain); |
| 361 | final text = data?.text ?? ''; |
| 362 | if (text.isEmpty) { _say('No text on this clipboard'); return; } |
| 363 | const piece = 4096; |
| 364 | var i = 0; |
| 365 | while (i < text.length) { |
| 366 | var end = i + piece < text.length ? i + piece : text.length; |
| 367 | // keep surrogate pairs together |
| 368 | if (end < text.length && (text.codeUnitAt(end - 1) & 0xFC00) == 0xD800) end--; |
| 369 | _peer?.sendControl({'t': 'clip', 'text': text.substring(i, end), 'more': end < text.length}); |
| 370 | i = end; |
| 371 | } |
| 372 | _say('Clipboard sent to the desk'); |
| 373 | } |
| 374 | |
| 375 | /// Photos, videos or files from this device to the desk's Desktop. |
| 376 | Future<void> sendFiles() async { |
| 377 | final peer = _peer; |
| 378 | if (peer == null) return; |
| 379 | final picked = await FilePicker.pickFiles(allowMultiple: true, withData: true); |
| 380 | if (picked == null) return; |
| 381 | for (final f in picked.files) { |
| 382 | final data = f.bytes; |
| 383 | if (data == null) { _say('${f.name}: could not read it'); continue; } |
| 384 | await _sendBytes(f.name, data); |
| 385 | } |
| 386 | } |
| 387 | |
| 388 | Future<void> _sendBytes(String name, Uint8List data) async { |
| 389 | final peer = _peer; |
| 390 | if (peer == null) return; |
| 391 | try { |
| 392 | var shown = 0; |
| 393 | final r = await peer.sendFile(name, data, (p) { |
| 394 | final pct = (p * 100).round(); |
| 395 | if (pct >= shown + 20) { shown = pct; _say('Sending $name: $pct%'); } |
| 396 | }); |
| 397 | debugPrint('nc file: $name -> $r'); |
| 398 | _say(r.startsWith('ok ') ? "On the desk's Desktop: ${r.substring(3)}" : '$name: $r'); |
| 399 | } catch (e) { |
| 400 | debugPrint('nc file: $name failed: $e'); |
| 401 | _say('$name: $e'); |
| 402 | } |
| 403 | } |
| 404 | |
| 405 | /// The desk's clipboard to this device. |
| 406 | void getClipboard() => _peer?.sendControl({'t': 'clipnow'}); |
| 407 | |
| 408 | /// This device's camera as the desk's webcam ("network-computer camera"). |
| 409 | Future<void> toggleCamera() async { |
| 410 | final want = !cameraOn; |
| 411 | try { |
| 412 | await _peer?.setCamera(want); |
| 413 | cameraOn = want; |
| 414 | if (want) _say('Camera on: pick "network-computer camera" in the app on the desk'); |
| 415 | } catch (e) { |
| 416 | cameraOn = false; |
| 417 | await _peer?.setCamera(false).catchError((_) {}); |
| 418 | _say('Camera unavailable: $e'); |
| 419 | } |
| 420 | notifyListeners(); |
| 421 | } |
| 422 | |
| 423 | Future<void> flipCamera() async { |
| 424 | try { |
| 425 | await _peer?.flipCamera(); |
| 426 | } catch (e) { |
| 427 | _say('Could not switch cameras: $e'); |
| 428 | } |
| 429 | } |
| 430 | |
| 431 | Future<void> toggleMic() async { |
| 432 | final want = !micOn; |
| 433 | try { |
| 434 | await _peer?.setMic(want); |
| 435 | micOn = want; |
| 436 | } catch (e) { |
| 437 | micOn = false; |
| 438 | status = 'Microphone unavailable: $e'; |
| 439 | } |
| 440 | notifyListeners(); |
| 441 | } |
| 442 | |
| 443 | // Pairing: after a correct PIN the host returns a signed token; keep one per |
| 444 | // rendezvous+host and offer it next time so the PIN is typed once. |
| 445 | String _pairKey() => 'pair:${_ep!.base}|$_host'; |
| 446 | Future<String?> _loadPair() => Secrets.read(_pairKey()); |
| 447 | Future<void> _savePair(String t) => Secrets.write(_pairKey(), t); |
| 448 | Future<void> _clearPair() => Secrets.delete(_pairKey()); |
| 449 | |
| 450 | /// Whether a pairing token is stored for [ep] + [host] (for the UI hint). |
| 451 | static Future<bool> isPaired(Endpoint ep, String host) async => |
| 452 | await Secrets.read('pair:${ep.base}|$host') != null; |
| 453 | |
| 454 | void _set(ConnState s, String msg) { state = s; status = msg; notifyListeners(); } |
| 455 | void _fail(String msg) { state = ConnState.failed; status = msg; notifyListeners(); } |
| 456 | |
| 457 | @override |
| 458 | void dispose() { _teardown(); renderer.dispose(); super.dispose(); } |
| 459 | } |