Your phone is the terminal. Stream a Linux/Mac desktop to an iPhone over WebRTC through NAT, with audio and input. Open source, single binaries.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/network-computer.git
HTTPS
https://gitstable.com/git/askrobots/network-computer.git
SSH
ssh://git@gitstable.com:2222/askrobots/network-computer.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

Dan Quellhorst Firefox policy: no WebRTC mDNS (a page using WebRTC made Firefox listen on UDP 5353) a1d92f3 1 month, 3 weeks ago History
camera.go · 6.3 KB · 237 lines
1package main
2
3import (
4 "context"
5 "errors"
6 "io"
7 "log"
8 "os"
9 "os/exec"
10 "strings"
11 "sync"
12 "time"
13
14 "github.com/pion/rtcp"
15 "github.com/pion/rtp"
16 "github.com/pion/webrtc/v4"
17 "github.com/pion/webrtc/v4/pkg/media/h264writer"
18 "github.com/pion/webrtc/v4/pkg/media/ivfwriter"
19)
20
21// The client's camera as a webcam on this machine. The client opens a second
22// video transceiver (sending) at connect with no camera attached, and attaches
23// one when the person taps 📷, so the camera light is only on while it is in
24// use. Its frames are decoded by ffmpeg into a v4l2loopback device
25// ("network-computer camera", -camera-device), which Firefox, Jitsi, Zoom or
26// any other app on the desk picks like a real webcam.
27//
28// Only one camera at a time: the newest one to start sending takes the device.
29// Frames stop (camera off, the app in the background): after a short wait
30// ffmpeg is closed and the device released; the next frame starts it again.
31const (
32 camWidth, camHeight = 1280, 720
33 camIdle = 2 * time.Second
34)
35
36// camArgs decodes a stream of the given format from stdin into the device,
37// letterboxed to a fixed size so a phone turning sideways does not change the
38// device's format under the apps using it.
39func camArgs(format, device string) []string {
40 vf := "scale=1280:720:force_original_aspect_ratio=decrease,pad=1280:720:(ow-iw)/2:(oh-ih)/2,format=yuv420p"
41 return []string{"-hide_banner", "-loglevel", "error",
42 "-fflags", "nobuffer", "-flags", "low_delay", "-probesize", "32", "-analyzeduration", "0",
43 "-f", format, "-i", "pipe:0",
44 "-vf", vf, "-r", "30", "-f", "v4l2", device}
45}
46
47// camWriter depacketizes RTP into a stream ffmpeg can read.
48type camWriter interface {
49 WriteRTP(*rtp.Packet) error
50 Close() error
51}
52
53func camFormat(mime string) string {
54 switch strings.ToLower(mime) {
55 case strings.ToLower(webrtc.MimeTypeH264):
56 return "h264"
57 case strings.ToLower(webrtc.MimeTypeVP8), strings.ToLower(webrtc.MimeTypeVP9), strings.ToLower(webrtc.MimeTypeAV1):
58 return "ivf"
59 }
60 return ""
61}
62
63func newCamWriter(mime string, w io.Writer) (camWriter, error) {
64 if camFormat(mime) == "h264" {
65 return h264writer.NewWith(w), nil
66 }
67 return ivfwriter.NewWith(w, ivfwriter.WithCodec(mime))
68}
69
70// camOwner lets only the newest camera write to the device.
71type camOwner struct {
72 mu sync.Mutex
73 cancel context.CancelFunc
74 id int
75}
76
77func (c *camOwner) take(cancel context.CancelFunc) int {
78 c.mu.Lock()
79 defer c.mu.Unlock()
80 if c.cancel != nil {
81 c.cancel()
82 }
83 c.id++
84 c.cancel = cancel
85 return c.id
86}
87
88func (c *camOwner) current() int {
89 c.mu.Lock()
90 defer c.mu.Unlock()
91 return c.id
92}
93
94func (c *camOwner) release(id int) {
95 c.mu.Lock()
96 defer c.mu.Unlock()
97 if c.id == id {
98 c.cancel = nil
99 }
100}
101
102// playCamera copies one incoming video track into the camera device until the
103// session ends. Without a device it just drains the track.
104func (h *host) playCamera(ctx context.Context, pc *webrtc.PeerConnection, track *webrtc.TrackRemote) {
105 if h.camera == "" {
106 drain(track)
107 return
108 }
109 mime := track.Codec().MimeType
110 log.Printf("camera: client camera track (%s)", mime)
111 for ctx.Err() == nil {
112 // wait for the camera to start (or start again)
113 track.SetReadDeadline(time.Time{})
114 first, _, err := track.ReadRTP()
115 if err != nil {
116 log.Printf("camera: track ended (%v)", err)
117 return
118 }
119 if len(first.Payload) == 0 {
120 continue
121 }
122 err = h.cameraRun(ctx, pc, track, mime, first)
123 if errors.Is(err, errPreempted) {
124 // another device's camera has the webcam: ignore this one until it
125 // stops sending, or it would take the device straight back
126 log.Printf("camera: another device's camera took over")
127 for {
128 track.SetReadDeadline(time.Now().Add(camIdle))
129 if _, _, err := track.ReadRTP(); err != nil {
130 if isTimeout(err) {
131 break
132 }
133 return
134 }
135 }
136 continue
137 }
138 if err != nil {
139 log.Printf("camera: %v", err)
140 if strings.Contains(err.Error(), "unsupported") {
141 drain(track)
142 return
143 }
144 time.Sleep(time.Second)
145 }
146 }
147}
148
149var errPreempted = errors.New("preempted")
150
151func isTimeout(err error) bool {
152 var t interface{ Timeout() bool }
153 return errors.As(err, &t) && t.Timeout()
154}
155
156// cameraRun runs ffmpeg for one stretch of frames: until they stop for
157// camIdle, the session ends, or a newer camera takes over.
158func (h *host) cameraRun(ctx context.Context, pc *webrtc.PeerConnection, track *webrtc.TrackRemote, mime string, first *rtp.Packet) error {
159 rctx, cancel := context.WithCancel(ctx)
160 defer cancel()
161 id := h.cam.take(cancel)
162 defer h.cam.release(id)
163
164 // provisioning sets it too; without it this v4l2loopback refuses every
165 // writer after the first one closes
166 exec.Command("v4l2-ctl", "-d", h.camera, "-c", "keep_format=1").Run()
167 format := camFormat(mime)
168 if format == "" {
169 return errors.New("unsupported camera codec " + mime)
170 }
171 // ffmpeg first: the IVF writer writes its header into the pipe at once,
172 // which blocks until someone reads it
173 pr, pw := io.Pipe()
174 args := camArgs(format, h.camera)
175 cmd := exec.CommandContext(rctx, "ffmpeg", args...)
176 cmd.Stdin = pr
177 cmd.Stderr = os.Stderr
178 log.Printf("ffmpeg(camera) %s", strings.Join(args, " "))
179 if err := cmd.Start(); err != nil {
180 return err
181 }
182 w, err := newCamWriter(mime, pw)
183 if err != nil {
184 pw.Close()
185 cmd.Wait()
186 return err
187 }
188 log.Printf("camera: on -> %s (%s)", h.camera, mime)
189 done := make(chan struct{})
190 go func() { cmd.Wait(); pr.CloseWithError(io.EOF); close(done) }()
191
192 // a keyframe now, and again a few times while ffmpeg starts: the camera
193 // may have begun mid-stream and nothing decodes before one
194 pli := func() {
195 pc.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(track.SSRC())}})
196 }
197 pli()
198 go func() {
199 for i := 0; i < 3; i++ {
200 select {
201 case <-time.After(time.Second):
202 pli()
203 case <-rctx.Done():
204 return
205 }
206 }
207 }()
208
209 frames := 0
210 pkt := first
211 for {
212 if len(pkt.Payload) > 0 {
213 if err := w.WriteRTP(pkt); err != nil {
214 break // ffmpeg is gone
215 }
216 if pkt.Marker {
217 frames++
218 }
219 }
220 if rctx.Err() != nil {
221 break
222 }
223 track.SetReadDeadline(time.Now().Add(camIdle))
224 if pkt, _, err = track.ReadRTP(); err != nil {
225 break // stopped sending, or the session closed
226 }
227 }
228 w.Close()
229 pw.Close()
230 cancel()
231 <-done
232 log.Printf("camera: off after %d frames", frames)
233 if ctx.Err() == nil && h.cam.current() != id {
234 return errPreempted
235 }
236 return nil
237}