Your phone is the terminal. Stream a Linux/Mac desktop to an iPhone over WebRTC through NAT, with audio and input. Open source, single binaries.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/network-computer.git
HTTPS
https://gitstable.com/git/askrobots/network-computer.git
SSH
ssh://git@gitstable.com:2222/askrobots/network-computer.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

Dan Quellhorst Firefox policy: no WebRTC mDNS (a page using WebRTC made Firefox listen on UDP 5353) a1d92f3 1 month, 3 weeks ago History
main.go · 17.9 KB · 573 lines
1// nc-host: registers with nc-rendezvous, answers WebRTC offers, streams the
2// screen (ffmpeg -> Pion) and injects input from the data channel.
3package main
4
5import (
6 "context"
7 "crypto/rand"
8 "crypto/subtle"
9 "encoding/base64"
10 "encoding/json"
11 "flag"
12 "fmt"
13 "log"
14 "math/big"
15 "net/http"
16 "os"
17 "path/filepath"
18 "runtime"
19 "strings"
20 "sync"
21 "time"
22
23 "github.com/coder/websocket"
24 "github.com/pion/ice/v4"
25 "github.com/pion/sdp/v3"
26 "github.com/pion/webrtc/v4"
27 "github.com/pion/webrtc/v4/pkg/media"
28
29 "github.com/askrobots/network-computer/internal/input"
30 "github.com/askrobots/network-computer/internal/proto"
31 "github.com/askrobots/network-computer/internal/tlspin"
32)
33
34type host struct {
35 name string
36 ice []webrtc.ICEServer
37 capture captureOpts
38 audio string
39 mic string
40 camera string // v4l2loopback device for the client's camera ('' = ignore it)
41 video, sound *hub // one screen encoder and one sound encoder, shared by every device
42 cam camOwner
43 dryRun bool
44 api *webrtc.API
45 user, password, pin string
46 httpc *http.Client // pinned when -tls-fingerprint is set
47 pairs *pairing
48 display displayControl
49 filesDir string
50 idleFile string // "busy N" or "idle-since UNIX", for auto-stop (next to the send socket)
51 inputFile string // input=touch|pointer, device=phone|tablet|desktop: for the apps and voice
52 inputKind, device string
53 idleSince time.Time // when the last client left
54 voice voiceRelay
55 injOnce sync.Once
56 inj input.Injector
57
58 mu sync.Mutex
59 ws *websocket.Conn
60 sessions map[string]*session
61}
62
63type session struct {
64 pc *webrtc.PeerConnection
65 cancel context.CancelFunc
66 ctl *webrtc.DataChannel // the client's "control" channel, once open
67 fileIn *fileOut // the client's "file-in" channel (the app), for files to it
68}
69
70func main() {
71 rz := flag.String("rendezvous", "http://127.0.0.1:8080", "rendezvous base URL (http or https)")
72 name := flag.String("name", hostname(), "name to register as")
73 display := flag.Int("display", 0, "screen index to capture")
74 fps := flag.Int("fps", 60, "capture frame rate")
75 size := flag.String("size", "1920x1080", "scale output to WxH ('native' for no scaling). iPhone 12 class target: 1080p60")
76 bitrate := flag.String("bitrate", "8M", "video bitrate")
77 encoder := flag.String("encoder", "", "ffmpeg video encoder (default: h264_videotoolbox on mac, libx264 elsewhere)")
78 extra := flag.String("ffmpeg-extra", "", "extra ffmpeg args inserted before the output")
79 custom := flag.String("ffmpeg-args", "", "full ffmpeg argument string; must end with '-f h264 pipe:1'")
80 audio := flag.String("audio-device", "", "capture audio from this device (mac: avfoundation index; linux: pulse source; '' = no audio)")
81 camera := flag.String("camera-device", "", "show the client's camera as this webcam (linux: a v4l2loopback device such as /dev/video10; '' = ignore)")
82 mic := flag.String("mic-device", "", "play the client's microphone into this device (linux: pulse sink such as nc-mic; mac: output device index, e.g. BlackHole; '' = ignore)")
83 user := flag.String("user", envOr("NC_USER", "nc"), "rendezvous basic auth username (env NC_USER)")
84 password := flag.String("password", os.Getenv("NC_PASSWORD"), "rendezvous basic auth password (env NC_PASSWORD)")
85 pin := flag.String("pin", os.Getenv("NC_PIN"), "PIN a client must present to connect to this host (env NC_PIN); generated and printed if empty")
86 stateDir := flag.String("state-dir", defaultHostStateDir(), "where the pairing secret is kept")
87 resetPairs := flag.Bool("reset-pairings", false, "rotate the pairing secret, revoking every paired client")
88 tlsFP := flag.String("tls-fingerprint", os.Getenv("NC_TLS_FP"), "pin the rendezvous certificate by SHA-256 (env NC_TLS_FP); for a self-signed rendezvous")
89 filesDir := flag.String("files-dir", "", "save files dropped on the client window here ('' = refuse them); e.g. the desk user's Desktop")
90 sendSocket := flag.String("send-socket", "", "unix socket where nc-send hands over files for the connected client ('' = off)")
91 mdns := flag.Bool("mdns", true, "resolve clients' .local ICE candidates (mDNS, port 5353); off on a cloud desk")
92 dryRun := flag.Bool("dry-run", false, "log input events instead of injecting them")
93 flag.Parse()
94
95 if *pin == "" {
96 *pin = randomPIN()
97 }
98 log.Printf("host PIN: %s (clients must enter this to connect)", *pin)
99 h := &host{video: newHub("video", true, streamVideo), name: *name, audio: *audio, mic: *mic, camera: *camera, dryRun: *dryRun, sessions: map[string]*session{}, user: *user, password: *password, pin: *pin, filesDir: *filesDir}
100 h.sound = newHub("sound", false, func(ctx context.Context, _ captureOpts, w sampleWriter, _ func()) {
101 streamAudio(ctx, h.audio, w)
102 })
103 if *sendSocket != "" {
104 go h.serveSend(*sendSocket)
105 h.idleFile = filepath.Join(filepath.Dir(*sendSocket), "idle")
106 h.inputFile = filepath.Join(filepath.Dir(*sendSocket), "input")
107 }
108 h.idleSince = time.Now()
109 h.writeIdle()
110 h.httpc = tlspin.Client(*tlsFP)
111 pairs, err := loadPairing(*name, *stateDir, *resetPairs)
112 if err != nil {
113 log.Fatalf("pairing secret: %v", err)
114 }
115 h.pairs = pairs
116 if *resetPairs {
117 log.Printf("pairing secret rotated: every previously paired client must enter the PIN again")
118 }
119 h.capture = captureOpts{Display: *display, FPS: *fps, Bitrate: *bitrate, Encoder: *encoder, Extra: *extra, Custom: *custom}
120 if *size != "" && *size != "native" {
121 fmt.Sscanf(*size, "%dx%d", &h.capture.Width, &h.capture.Height)
122 }
123 if runtime.GOOS == "darwin" && *custom == "" {
124 idx, err := resolveDarwinScreen(*display)
125 if err != nil {
126 log.Fatal(err)
127 }
128 h.capture.Display = idx
129 }
130
131 // ICE servers come from the rendezvous so host and client agree.
132 var cfg proto.Config
133 req, _ := http.NewRequest("GET", *rz+"/config", nil)
134 req.SetBasicAuth(h.user, h.password)
135 if resp, err := h.httpc.Do(req); err == nil {
136 if resp.StatusCode == http.StatusUnauthorized {
137 log.Fatalf("rendezvous rejected the password (use -password or NC_PASSWORD)")
138 }
139 json.NewDecoder(resp.Body).Decode(&cfg)
140 resp.Body.Close()
141 } else {
142 log.Fatalf("GET %s/config: %v", *rz, err)
143 }
144 for _, s := range cfg.ICEServers {
145 h.ice = append(h.ice, webrtc.ICEServer{URLs: s.URLs, Username: s.Username, Credential: s.Credential})
146 }
147 log.Printf("ice servers: %+v", cfg.ICEServers)
148
149 m := &webrtc.MediaEngine{}
150 if err := m.RegisterDefaultCodecs(); err != nil {
151 log.Fatal(err)
152 }
153 // the MID header extension ties incoming RTP to its m-line: the camera's
154 // packets start after the offer (the camera is attached on demand) and must
155 // still find their transceiver
156 for _, k := range []webrtc.RTPCodecType{webrtc.RTPCodecTypeAudio, webrtc.RTPCodecTypeVideo} {
157 if err := m.RegisterHeaderExtension(webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESMidURI}, k); err != nil {
158 log.Fatal(err)
159 }
160 }
161 var se webrtc.SettingEngine
162 if !*mdns {
163 // a cloud desk cannot reach a client's .local names anyway, and the
164 // resolver would listen on 5353 on every interface
165 se.SetICEMulticastDNSMode(ice.MulticastDNSModeDisabled)
166 }
167 h.api = webrtc.NewAPI(webrtc.WithMediaEngine(m), webrtc.WithSettingEngine(se))
168
169 wsURL := strings.Replace(strings.Replace(*rz, "https://", "wss://", 1), "http://", "ws://", 1) + "/ws"
170 for {
171 if err := h.runSignaling(context.Background(), wsURL); err != nil {
172 log.Printf("signaling: %v (reconnecting in 3s)", err)
173 }
174 time.Sleep(3 * time.Second)
175 }
176}
177
178func (h *host) runSignaling(ctx context.Context, url string) error {
179 hdr := http.Header{}
180 hdr.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(h.user+":"+h.password)))
181 conn, _, err := websocket.Dial(ctx, url, &websocket.DialOptions{HTTPHeader: hdr, HTTPClient: h.httpc})
182 if err != nil {
183 return err
184 }
185 defer conn.CloseNow()
186 conn.SetReadLimit(4 << 20)
187 h.mu.Lock()
188 h.ws = conn
189 h.mu.Unlock()
190 if err := h.send(ctx, proto.Message{Type: "register", Name: h.name}); err != nil {
191 return err
192 }
193 log.Printf("registered as %q at %s", h.name, url)
194 for {
195 _, data, err := conn.Read(ctx)
196 if err != nil {
197 return err
198 }
199 var m proto.Message
200 if err := json.Unmarshal(data, &m); err != nil {
201 continue
202 }
203 switch m.Type {
204 case "offer":
205 go h.handleOffer(ctx, m)
206 case "ice":
207 h.mu.Lock()
208 s := h.sessions[m.From]
209 h.mu.Unlock()
210 if s != nil {
211 var c webrtc.ICECandidateInit
212 if json.Unmarshal(m.Candidate, &c) == nil {
213 s.pc.AddICECandidate(c)
214 }
215 }
216 case "error":
217 log.Printf("rendezvous error: %s", m.Error)
218 }
219 }
220}
221
222func (h *host) send(ctx context.Context, m proto.Message) error {
223 b, _ := json.Marshal(m)
224 h.mu.Lock()
225 defer h.mu.Unlock()
226 if h.ws == nil {
227 return fmt.Errorf("no signaling connection")
228 }
229 return h.ws.Write(ctx, websocket.MessageText, b)
230}
231
232func (h *host) handleOffer(ctx context.Context, m proto.Message) {
233 peer := m.From
234 pinOK := m.PIN != "" && subtle.ConstantTimeCompare([]byte(m.PIN), []byte(h.pin)) == 1
235 pairOK := m.Pair != "" && h.pairs.valid(m.Pair)
236 if !pinOK && !pairOK {
237 why := "wrong PIN"
238 if m.Pair != "" && m.PIN == "" {
239 why = "pairing expired or revoked: enter the PIN"
240 }
241 log.Printf("[%s] offer rejected: %s", peer, why)
242 h.send(ctx, proto.Message{Type: "error", To: peer, Error: why})
243 return
244 }
245 if pairOK {
246 log.Printf("[%s] offer received, paired client", peer)
247 } else {
248 log.Printf("[%s] offer received, PIN ok", peer)
249 }
250
251 pc, err := h.api.NewPeerConnection(webrtc.Configuration{ICEServers: h.ice})
252 if err != nil {
253 log.Printf("[%s] pc: %v", peer, err)
254 return
255 }
256 sctx, cancel := context.WithCancel(ctx)
257 s := &session{pc: pc, cancel: cancel}
258 h.mu.Lock()
259 if old := h.sessions[peer]; old != nil {
260 old.cancel()
261 old.pc.Close()
262 }
263 h.sessions[peer] = s
264 h.mu.Unlock()
265
266 closeSession := func() {
267 cancel()
268 pc.Close()
269 h.mu.Lock()
270 if h.sessions[peer] == s {
271 delete(h.sessions, peer)
272 }
273 nobody := len(h.sessions) == 0
274 h.mu.Unlock()
275 if nobody {
276 h.voice.command(false, false) // no one to listen to
277 }
278 h.writeIdle()
279 }
280
281 video, err := webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264}, "video", "nc")
282 if err != nil {
283 log.Printf("track: %v", err)
284 closeSession()
285 return
286 }
287 vsender, err := pc.AddTrack(video)
288 if err != nil {
289 log.Printf("addtrack: %v", err)
290 closeSession()
291 return
292 }
293 // Drain RTCP (PLI/NACK). We cannot ask ffmpeg for a keyframe on demand,
294 // so PLIs are only logged; GOP is 2s so recovery is bounded anyway.
295 go func() {
296 buf := make([]byte, 1500)
297 for {
298 if _, _, err := vsender.Read(buf); err != nil {
299 return
300 }
301 }
302 }()
303
304 // The client's microphone arrives as an incoming audio track, its camera
305 // (when the person turns it on) as an incoming video track.
306 pc.OnTrack(func(track *webrtc.TrackRemote, _ *webrtc.RTPReceiver) {
307 if track.Kind() == webrtc.RTPCodecTypeAudio {
308 go playMic(sctx, track, h.mic)
309 return
310 }
311 go h.playCamera(sctx, pc, track)
312 })
313
314 var audioTrack *webrtc.TrackLocalStaticSample
315 if h.audio != "" {
316 audioTrack, _ = webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2}, "audio", "nc")
317 if asender, err := pc.AddTrack(audioTrack); err == nil {
318 go func() {
319 buf := make([]byte, 1500)
320 for {
321 if _, _, err := asender.Read(buf); err != nil {
322 return
323 }
324 }
325 }()
326 }
327 }
328
329 inj := h.injector()
330 clip := newClipSync()
331 pc.OnDataChannel(func(dc *webrtc.DataChannel) {
332 log.Printf("[%s] data channel %q", peer, dc.Label())
333 if dc.Label() == "file" {
334 receiveFile(peer, dc, h.filesDir)
335 return
336 }
337 if dc.Label() == "file-in" { // the app receives files on a channel it opened
338 o := newFileOut(dc)
339 h.mu.Lock()
340 s.fileIn = o
341 h.mu.Unlock()
342 return
343 }
344 if dc.Label() == "control" {
345 clip.dc = dc
346 h.mu.Lock()
347 s.ctl = dc
348 h.mu.Unlock()
349 dc.OnOpen(func() { go clip.run(sctx) })
350 }
351 dc.OnMessage(func(msg webrtc.DataChannelMessage) {
352 var ev proto.InputEvent
353 if err := json.Unmarshal(msg.Data, &ev); err != nil {
354 return
355 }
356 switch ev.T {
357 case "keyboard":
358 go setKeyboard(ev.Layout)
359 return
360 case "clip":
361 clip.receive(ev) // not in a goroutine: the paste keystroke that follows must find it
362 return
363 case "clipget":
364 clip.get()
365 return
366 case "clipnow":
367 go clip.now()
368 return
369 case "launch":
370 go launch()
371 return
372 case "tz":
373 go setTimezone(ev.Text)
374 return
375 case "input":
376 h.writeInput(ev.Kind, ev.Device)
377 return
378 case "voice":
379 if !h.voice.command(ev.On, ev.Once) {
380 b, _ := json.Marshal(proto.InputEvent{T: "voice", Kind: "error", Text: "voice is not running on this desk"})
381 dc.SendText(string(b))
382 }
383 return
384 }
385 if ev.T == "display" {
386 go func() {
387 req := normalize(ev)
388 if h.display.apply(req) {
389 opts := h.capture
390 opts.Width, opts.Height = req.W, req.H
391 opts.Bitrate = bitrateFor(h.capture.Bitrate, req.W, req.H)
392 h.video.restart(opts) // the screen changed for every device
393 }
394 }()
395 return
396 }
397 if (ev.T == "md" || ev.T == "mu") && ev.At {
398 inj.Handle(proto.InputEvent{T: "mm", X: ev.X, Y: ev.Y}) // click where it was pressed
399 }
400 inj.Handle(ev)
401 })
402 })
403
404 pc.OnICECandidate(func(c *webrtc.ICECandidate) {
405 if c == nil {
406 return
407 }
408 b, _ := json.Marshal(c.ToJSON())
409 h.send(ctx, proto.Message{Type: "ice", To: peer, Candidate: b})
410 })
411 pc.OnICEConnectionStateChange(func(st webrtc.ICEConnectionState) {
412 log.Printf("[%s] ice: %s", peer, st)
413 })
414 pc.OnConnectionStateChange(func(st webrtc.PeerConnectionState) {
415 log.Printf("[%s] pc: %s", peer, st)
416 switch st {
417 case webrtc.PeerConnectionStateConnected:
418 go logSelectedPair(peer, pc)
419 h.writeIdle()
420 for _, c := range vsender.GetParameters().Codecs { // what the client agreed to decode
421 log.Printf("[%s] video codec: %s %s (pt %d)", peer, c.MimeType, c.SDPFmtpLine, c.PayloadType)
422 }
423 go func() {
424 opts := h.capture
425 if last := h.display.current(); last.W > 0 {
426 opts.Width, opts.Height = last.W, last.H
427 opts.Bitrate = bitrateFor(h.capture.Bitrate, last.W, last.H)
428 }
429 leave := h.video.join(video, opts)
430 var leaveSound func()
431 if audioTrack != nil {
432 leaveSound = h.sound.join(audioTrack, opts)
433 }
434 <-sctx.Done()
435 leave()
436 if leaveSound != nil {
437 leaveSound()
438 }
439 }()
440 case webrtc.PeerConnectionStateFailed, webrtc.PeerConnectionStateClosed, webrtc.PeerConnectionStateDisconnected:
441 inj.ReleaseAll()
442 closeSession()
443 }
444 })
445
446 if err := pc.SetRemoteDescription(webrtc.SessionDescription{Type: webrtc.SDPTypeOffer, SDP: m.SDP}); err != nil {
447 log.Printf("[%s] set remote: %v", peer, err)
448 closeSession()
449 return
450 }
451 answer, err := pc.CreateAnswer(nil)
452 if err != nil {
453 log.Printf("[%s] answer: %v", peer, err)
454 closeSession()
455 return
456 }
457 if err := pc.SetLocalDescription(answer); err != nil {
458 log.Printf("[%s] set local: %v", peer, err)
459 closeSession()
460 return
461 }
462 // every successful offer gets a fresh pairing token, so the PIN is typed once
463 h.send(ctx, proto.Message{Type: "answer", To: peer, SDP: answer.SDP, Pair: h.pairs.mint()})
464}
465
466func logSelectedPair(peer string, pc *webrtc.PeerConnection) {
467 time.Sleep(500 * time.Millisecond)
468 for _, t := range pc.GetTransceivers() {
469 if t.Sender() == nil || t.Sender().Transport() == nil {
470 continue
471 }
472 p, err := t.Sender().Transport().ICETransport().GetSelectedCandidatePair()
473 if err == nil && p != nil {
474 log.Printf("[%s] path: local %s %s:%d <-> remote %s %s:%d", peer,
475 p.Local.Typ, p.Local.Address, p.Local.Port, p.Remote.Typ, p.Remote.Address, p.Remote.Port)
476 }
477 return
478 }
479}
480
481func randomPIN() string {
482 n, _ := rand.Int(rand.Reader, big.NewInt(1000000))
483 return fmt.Sprintf("%06d", n.Int64())
484}
485
486func envOr(k, def string) string {
487 if v := os.Getenv(k); v != "" {
488 return v
489 }
490 return def
491}
492
493func hostname() string {
494 n, _ := os.Hostname()
495 if i := strings.Index(n, "."); i > 0 {
496 n = n[:i]
497 }
498 return n
499}
500
501var _ = media.Sample{}
502
503// injector creates the virtual keyboard and mouse once per host process and
504// reuses them for every session: devices created per connection were never
505// closed, and each new keyboard came up with the default layout.
506func (h *host) injector() input.Injector {
507 h.injOnce.Do(func() {
508 inj, err := input.New(h.capture.Display, h.dryRun)
509 if err != nil {
510 log.Printf("input: %v (falling back to dry-run)", err)
511 inj, _ = input.New(h.capture.Display, true)
512 }
513 h.inj = inj
514 runInputHook()
515 })
516 return h.inj
517}
518
519// writeInput records what the person is using right now, as the client
520// reports it: input "touch" (fingers on the glass) or "pointer" (a mouse or
521// trackpad), and device "phone", "tablet" or "desktop". The dbbasic apps read
522// it (dbbasic_app_kit InputMode) to size and arrange their controls; voice
523// reads it to know what "prepare this for my phone" means. The last report
524// wins; a report without a device keeps the one before.
525func (h *host) writeInput(kind, device string) {
526 if h.inputFile == "" {
527 return
528 }
529 h.mu.Lock()
530 if kind == "touch" || kind == "pointer" {
531 h.inputKind = kind
532 }
533 if device == "phone" || device == "tablet" || device == "desktop" {
534 h.device = device
535 }
536 line := ""
537 if h.inputKind != "" {
538 line += "input=" + h.inputKind + "\n"
539 }
540 if h.device != "" {
541 line += "device=" + h.device + "\n"
542 }
543 h.mu.Unlock()
544 if line != "" {
545 os.WriteFile(h.inputFile, []byte(line), 0o644)
546 }
547}
548
549// writeIdle records whether anyone is connected, and since when nobody is:
550// the Mac's auto-stop (infra/droplet.sh autostop) reads it over ssh.
551func (h *host) writeIdle() {
552 if h.idleFile == "" {
553 return
554 }
555 h.mu.Lock()
556 n := 0
557 for _, s := range h.sessions {
558 if s.pc.ConnectionState() == webrtc.PeerConnectionStateConnected {
559 n++
560 }
561 }
562 if n > 0 {
563 h.idleSince = time.Time{}
564 } else if h.idleSince.IsZero() {
565 h.idleSince = time.Now()
566 }
567 line := fmt.Sprintf("busy %d\n", n)
568 if n == 0 {
569 line = fmt.Sprintf("idle-since %d\n", h.idleSince.Unix())
570 }
571 h.mu.Unlock()
572 os.WriteFile(h.idleFile, []byte(line), 0o644)
573}