askrobots / network-computer
Public
Your phone is the terminal. Stream a Linux/Mac desktop to an iPhone over WebRTC through NAT, with audio and input. Open source, single binaries.
Dan Quellhorst
Firefox policy: no WebRTC mDNS (a page using WebRTC made Firefox listen on UDP 5353)
a1d92f3
1 month, 3 weeks ago
History
camera.go · 6.3 KB · 237 lines
| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "io" |
| 7 | "log" |
| 8 | "os" |
| 9 | "os/exec" |
| 10 | "strings" |
| 11 | "sync" |
| 12 | "time" |
| 13 | |
| 14 | "github.com/pion/rtcp" |
| 15 | "github.com/pion/rtp" |
| 16 | "github.com/pion/webrtc/v4" |
| 17 | "github.com/pion/webrtc/v4/pkg/media/h264writer" |
| 18 | "github.com/pion/webrtc/v4/pkg/media/ivfwriter" |
| 19 | ) |
| 20 | |
| 21 | // The client's camera as a webcam on this machine. The client opens a second |
| 22 | // video transceiver (sending) at connect with no camera attached, and attaches |
| 23 | // one when the person taps 📷, so the camera light is only on while it is in |
| 24 | // use. Its frames are decoded by ffmpeg into a v4l2loopback device |
| 25 | // ("network-computer camera", -camera-device), which Firefox, Jitsi, Zoom or |
| 26 | // any other app on the desk picks like a real webcam. |
| 27 | // |
| 28 | // Only one camera at a time: the newest one to start sending takes the device. |
| 29 | // Frames stop (camera off, the app in the background): after a short wait |
| 30 | // ffmpeg is closed and the device released; the next frame starts it again. |
| 31 | const ( |
| 32 | camWidth, camHeight = 1280, 720 |
| 33 | camIdle = 2 * time.Second |
| 34 | ) |
| 35 | |
| 36 | // camArgs decodes a stream of the given format from stdin into the device, |
| 37 | // letterboxed to a fixed size so a phone turning sideways does not change the |
| 38 | // device's format under the apps using it. |
| 39 | func camArgs(format, device string) []string { |
| 40 | vf := "scale=1280:720:force_original_aspect_ratio=decrease,pad=1280:720:(ow-iw)/2:(oh-ih)/2,format=yuv420p" |
| 41 | return []string{"-hide_banner", "-loglevel", "error", |
| 42 | "-fflags", "nobuffer", "-flags", "low_delay", "-probesize", "32", "-analyzeduration", "0", |
| 43 | "-f", format, "-i", "pipe:0", |
| 44 | "-vf", vf, "-r", "30", "-f", "v4l2", device} |
| 45 | } |
| 46 | |
| 47 | // camWriter depacketizes RTP into a stream ffmpeg can read. |
| 48 | type camWriter interface { |
| 49 | WriteRTP(*rtp.Packet) error |
| 50 | Close() error |
| 51 | } |
| 52 | |
| 53 | func camFormat(mime string) string { |
| 54 | switch strings.ToLower(mime) { |
| 55 | case strings.ToLower(webrtc.MimeTypeH264): |
| 56 | return "h264" |
| 57 | case strings.ToLower(webrtc.MimeTypeVP8), strings.ToLower(webrtc.MimeTypeVP9), strings.ToLower(webrtc.MimeTypeAV1): |
| 58 | return "ivf" |
| 59 | } |
| 60 | return "" |
| 61 | } |
| 62 | |
| 63 | func newCamWriter(mime string, w io.Writer) (camWriter, error) { |
| 64 | if camFormat(mime) == "h264" { |
| 65 | return h264writer.NewWith(w), nil |
| 66 | } |
| 67 | return ivfwriter.NewWith(w, ivfwriter.WithCodec(mime)) |
| 68 | } |
| 69 | |
| 70 | // camOwner lets only the newest camera write to the device. |
| 71 | type camOwner struct { |
| 72 | mu sync.Mutex |
| 73 | cancel context.CancelFunc |
| 74 | id int |
| 75 | } |
| 76 | |
| 77 | func (c *camOwner) take(cancel context.CancelFunc) int { |
| 78 | c.mu.Lock() |
| 79 | defer c.mu.Unlock() |
| 80 | if c.cancel != nil { |
| 81 | c.cancel() |
| 82 | } |
| 83 | c.id++ |
| 84 | c.cancel = cancel |
| 85 | return c.id |
| 86 | } |
| 87 | |
| 88 | func (c *camOwner) current() int { |
| 89 | c.mu.Lock() |
| 90 | defer c.mu.Unlock() |
| 91 | return c.id |
| 92 | } |
| 93 | |
| 94 | func (c *camOwner) release(id int) { |
| 95 | c.mu.Lock() |
| 96 | defer c.mu.Unlock() |
| 97 | if c.id == id { |
| 98 | c.cancel = nil |
| 99 | } |
| 100 | } |
| 101 | |
| 102 | // playCamera copies one incoming video track into the camera device until the |
| 103 | // session ends. Without a device it just drains the track. |
| 104 | func (h *host) playCamera(ctx context.Context, pc *webrtc.PeerConnection, track *webrtc.TrackRemote) { |
| 105 | if h.camera == "" { |
| 106 | drain(track) |
| 107 | return |
| 108 | } |
| 109 | mime := track.Codec().MimeType |
| 110 | log.Printf("camera: client camera track (%s)", mime) |
| 111 | for ctx.Err() == nil { |
| 112 | // wait for the camera to start (or start again) |
| 113 | track.SetReadDeadline(time.Time{}) |
| 114 | first, _, err := track.ReadRTP() |
| 115 | if err != nil { |
| 116 | log.Printf("camera: track ended (%v)", err) |
| 117 | return |
| 118 | } |
| 119 | if len(first.Payload) == 0 { |
| 120 | continue |
| 121 | } |
| 122 | err = h.cameraRun(ctx, pc, track, mime, first) |
| 123 | if errors.Is(err, errPreempted) { |
| 124 | // another device's camera has the webcam: ignore this one until it |
| 125 | // stops sending, or it would take the device straight back |
| 126 | log.Printf("camera: another device's camera took over") |
| 127 | for { |
| 128 | track.SetReadDeadline(time.Now().Add(camIdle)) |
| 129 | if _, _, err := track.ReadRTP(); err != nil { |
| 130 | if isTimeout(err) { |
| 131 | break |
| 132 | } |
| 133 | return |
| 134 | } |
| 135 | } |
| 136 | continue |
| 137 | } |
| 138 | if err != nil { |
| 139 | log.Printf("camera: %v", err) |
| 140 | if strings.Contains(err.Error(), "unsupported") { |
| 141 | drain(track) |
| 142 | return |
| 143 | } |
| 144 | time.Sleep(time.Second) |
| 145 | } |
| 146 | } |
| 147 | } |
| 148 | |
| 149 | var errPreempted = errors.New("preempted") |
| 150 | |
| 151 | func isTimeout(err error) bool { |
| 152 | var t interface{ Timeout() bool } |
| 153 | return errors.As(err, &t) && t.Timeout() |
| 154 | } |
| 155 | |
| 156 | // cameraRun runs ffmpeg for one stretch of frames: until they stop for |
| 157 | // camIdle, the session ends, or a newer camera takes over. |
| 158 | func (h *host) cameraRun(ctx context.Context, pc *webrtc.PeerConnection, track *webrtc.TrackRemote, mime string, first *rtp.Packet) error { |
| 159 | rctx, cancel := context.WithCancel(ctx) |
| 160 | defer cancel() |
| 161 | id := h.cam.take(cancel) |
| 162 | defer h.cam.release(id) |
| 163 | |
| 164 | // provisioning sets it too; without it this v4l2loopback refuses every |
| 165 | // writer after the first one closes |
| 166 | exec.Command("v4l2-ctl", "-d", h.camera, "-c", "keep_format=1").Run() |
| 167 | format := camFormat(mime) |
| 168 | if format == "" { |
| 169 | return errors.New("unsupported camera codec " + mime) |
| 170 | } |
| 171 | // ffmpeg first: the IVF writer writes its header into the pipe at once, |
| 172 | // which blocks until someone reads it |
| 173 | pr, pw := io.Pipe() |
| 174 | args := camArgs(format, h.camera) |
| 175 | cmd := exec.CommandContext(rctx, "ffmpeg", args...) |
| 176 | cmd.Stdin = pr |
| 177 | cmd.Stderr = os.Stderr |
| 178 | log.Printf("ffmpeg(camera) %s", strings.Join(args, " ")) |
| 179 | if err := cmd.Start(); err != nil { |
| 180 | return err |
| 181 | } |
| 182 | w, err := newCamWriter(mime, pw) |
| 183 | if err != nil { |
| 184 | pw.Close() |
| 185 | cmd.Wait() |
| 186 | return err |
| 187 | } |
| 188 | log.Printf("camera: on -> %s (%s)", h.camera, mime) |
| 189 | done := make(chan struct{}) |
| 190 | go func() { cmd.Wait(); pr.CloseWithError(io.EOF); close(done) }() |
| 191 | |
| 192 | // a keyframe now, and again a few times while ffmpeg starts: the camera |
| 193 | // may have begun mid-stream and nothing decodes before one |
| 194 | pli := func() { |
| 195 | pc.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(track.SSRC())}}) |
| 196 | } |
| 197 | pli() |
| 198 | go func() { |
| 199 | for i := 0; i < 3; i++ { |
| 200 | select { |
| 201 | case <-time.After(time.Second): |
| 202 | pli() |
| 203 | case <-rctx.Done(): |
| 204 | return |
| 205 | } |
| 206 | } |
| 207 | }() |
| 208 | |
| 209 | frames := 0 |
| 210 | pkt := first |
| 211 | for { |
| 212 | if len(pkt.Payload) > 0 { |
| 213 | if err := w.WriteRTP(pkt); err != nil { |
| 214 | break // ffmpeg is gone |
| 215 | } |
| 216 | if pkt.Marker { |
| 217 | frames++ |
| 218 | } |
| 219 | } |
| 220 | if rctx.Err() != nil { |
| 221 | break |
| 222 | } |
| 223 | track.SetReadDeadline(time.Now().Add(camIdle)) |
| 224 | if pkt, _, err = track.ReadRTP(); err != nil { |
| 225 | break // stopped sending, or the session closed |
| 226 | } |
| 227 | } |
| 228 | w.Close() |
| 229 | pw.Close() |
| 230 | cancel() |
| 231 | <-done |
| 232 | log.Printf("camera: off after %d frames", frames) |
| 233 | if ctx.Err() == nil && h.cam.current() != id { |
| 234 | return errPreempted |
| 235 | } |
| 236 | return nil |
| 237 | } |