askrobots / network-computer
Public
Your phone is the terminal. Stream a Linux/Mac desktop to an iPhone over WebRTC through NAT, with audio and input. Open source, single binaries.
Dan Quellhorst
Firefox policy: no WebRTC mDNS (a page using WebRTC made Firefox listen on UDP 5353)
a1d92f3
1 month, 3 weeks ago
History
capture.go · 7.1 KB · 253 lines
| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "bufio" |
| 5 | "bytes" |
| 6 | "context" |
| 7 | "fmt" |
| 8 | "io" |
| 9 | "log" |
| 10 | "os" |
| 11 | "os/exec" |
| 12 | "runtime" |
| 13 | "strconv" |
| 14 | "strings" |
| 15 | "time" |
| 16 | |
| 17 | "github.com/pion/webrtc/v4/pkg/media" |
| 18 | ) |
| 19 | |
| 20 | // captureOpts describes one ffmpeg screen pipeline. |
| 21 | type captureOpts struct { |
| 22 | Display int // index of the screen to capture |
| 23 | FPS int |
| 24 | Width int // 0 = native |
| 25 | Height int |
| 26 | Bitrate string // e.g. "8M" |
| 27 | Encoder string // ffmpeg encoder name, "" = platform default |
| 28 | Extra string // raw extra ffmpeg args inserted before the output |
| 29 | Custom string // full custom ffmpeg argument string; overrides everything |
| 30 | } |
| 31 | |
| 32 | // ffmpegArgs builds a low-latency H.264 Annex-B pipeline for the current OS. |
| 33 | // Every branch ends with "-f h264 pipe:1" so the reader is identical. |
| 34 | func ffmpegArgs(o captureOpts) []string { |
| 35 | if o.Custom != "" { |
| 36 | return strings.Fields(o.Custom) |
| 37 | } |
| 38 | fps := strconv.Itoa(o.FPS) |
| 39 | var in []string |
| 40 | enc := o.Encoder |
| 41 | switch runtime.GOOS { |
| 42 | case "darwin": |
| 43 | // avfoundation screen devices come after the cameras; the index is |
| 44 | // looked up by name at startup in resolveDarwinScreen. |
| 45 | in = []string{"-f", "avfoundation", "-capture_cursor", "1", "-capture_mouse_clicks", "0", |
| 46 | "-framerate", fps, "-pixel_format", "uyvy422", "-i", fmt.Sprintf("%d:none", o.Display)} |
| 47 | if enc == "" { |
| 48 | enc = "h264_videotoolbox" |
| 49 | } |
| 50 | case "linux": |
| 51 | disp := os.Getenv("DISPLAY") |
| 52 | if disp == "" { |
| 53 | disp = ":0" |
| 54 | } |
| 55 | in = []string{"-f", "x11grab", "-framerate", fps, "-draw_mouse", "1", "-i", fmt.Sprintf("%s.%d", disp, o.Display)} |
| 56 | if enc == "" { |
| 57 | enc = "libx264" |
| 58 | } |
| 59 | case "windows": |
| 60 | in = []string{"-f", "gdigrab", "-framerate", fps, "-draw_mouse", "1", "-i", "desktop"} |
| 61 | if enc == "" { |
| 62 | enc = "libx264" |
| 63 | } |
| 64 | default: |
| 65 | log.Fatalf("no default capture for %s; use -ffmpeg-args", runtime.GOOS) |
| 66 | } |
| 67 | |
| 68 | args := []string{"-hide_banner", "-loglevel", "warning", "-fflags", "nobuffer", "-flags", "low_delay"} |
| 69 | args = append(args, in...) |
| 70 | // avfoundation reports a bogus nominal rate and ffmpeg would duplicate frames |
| 71 | // to match it; passthrough lets the capture device set the pace. x11grab and |
| 72 | // gdigrab timestamp correctly and warn about passthrough, so darwin only. |
| 73 | if runtime.GOOS == "darwin" { |
| 74 | args = append(args, "-fps_mode", "passthrough") |
| 75 | } |
| 76 | if o.Width > 0 && o.Height > 0 { |
| 77 | args = append(args, "-vf", fmt.Sprintf("scale=%d:%d", o.Width, o.Height)) |
| 78 | } |
| 79 | args = append(args, "-c:v", enc, "-b:v", o.Bitrate, "-maxrate", o.Bitrate, |
| 80 | "-g", strconv.Itoa(o.FPS*2), "-bf", "0", "-pix_fmt", "yuv420p") |
| 81 | switch enc { |
| 82 | case "h264_videotoolbox": |
| 83 | args = append(args, "-realtime", "1", "-prio_speed", "1", "-profile:v", "main", "-allow_sw", "1") |
| 84 | case "libx264": |
| 85 | args = append(args, "-preset", "ultrafast", "-tune", "zerolatency", "-profile:v", "baseline", "-x264-params", "repeat-headers=1") |
| 86 | case "h264_vaapi", "h264_nvenc", "h264_qsv", "h264_amf": |
| 87 | // caller usually needs -Extra for hwupload / device selection |
| 88 | } |
| 89 | if o.Extra != "" { |
| 90 | args = append(args, strings.Fields(o.Extra)...) |
| 91 | } |
| 92 | return append(args, "-f", "h264", "pipe:1") |
| 93 | } |
| 94 | |
| 95 | // resolveDarwinScreen maps "screen N" to the avfoundation device index by |
| 96 | // parsing ffmpeg's device list, since cameras shift the numbering. |
| 97 | func resolveDarwinScreen(n int) (int, error) { |
| 98 | out, _ := exec.Command("ffmpeg", "-hide_banner", "-f", "avfoundation", "-list_devices", "true", "-i", "").CombinedOutput() |
| 99 | for _, line := range strings.Split(string(out), "\n") { |
| 100 | if strings.Contains(line, fmt.Sprintf("Capture screen %d", n)) { |
| 101 | i := strings.Index(line, "] [") |
| 102 | if i < 0 { |
| 103 | continue |
| 104 | } |
| 105 | rest := line[i+3:] |
| 106 | j := strings.Index(rest, "]") |
| 107 | return strconv.Atoi(rest[:j]) |
| 108 | } |
| 109 | } |
| 110 | return 0, fmt.Errorf("screen %d not found in avfoundation device list", n) |
| 111 | } |
| 112 | |
| 113 | // streamVideo runs ffmpeg and pushes one access unit per WriteSample until |
| 114 | // ctx is cancelled. It restarts ffmpeg if it dies. |
| 115 | func streamVideo(ctx context.Context, o captureOpts, track sampleWriter, onKeyframe func()) { |
| 116 | for ctx.Err() == nil { |
| 117 | if err := runOnce(ctx, o, track, onKeyframe); err != nil && ctx.Err() == nil { |
| 118 | log.Printf("ffmpeg: %v (restarting in 1s)", err) |
| 119 | time.Sleep(time.Second) |
| 120 | } |
| 121 | } |
| 122 | } |
| 123 | |
| 124 | type sampleWriter interface { |
| 125 | WriteSample(media.Sample) error |
| 126 | } |
| 127 | |
| 128 | func runOnce(ctx context.Context, o captureOpts, track sampleWriter, onKeyframe func()) error { |
| 129 | args := ffmpegArgs(o) |
| 130 | log.Printf("ffmpeg %s", strings.Join(args, " ")) |
| 131 | cmd := exec.CommandContext(ctx, "ffmpeg", args...) |
| 132 | cmd.Stderr = os.Stderr |
| 133 | stdout, err := cmd.StdoutPipe() |
| 134 | if err != nil { |
| 135 | return err |
| 136 | } |
| 137 | if err := cmd.Start(); err != nil { |
| 138 | return err |
| 139 | } |
| 140 | defer cmd.Wait() |
| 141 | |
| 142 | r := bufio.NewReaderSize(stdout, 4<<20) |
| 143 | var au []byte // access unit being assembled (Annex-B, may hold SPS/PPS/SEI + slice) |
| 144 | auHasSlice := false |
| 145 | last := time.Now() |
| 146 | frames, bytesOut := 0, 0 |
| 147 | tick := time.NewTicker(5 * time.Second) |
| 148 | defer tick.Stop() |
| 149 | |
| 150 | flush := func() { |
| 151 | if len(au) == 0 { |
| 152 | return |
| 153 | } |
| 154 | now := time.Now() |
| 155 | d := now.Sub(last) |
| 156 | last = now |
| 157 | if d > time.Second { // first frame or a stall: don't blow up the RTP clock |
| 158 | d = time.Second / time.Duration(o.FPS) |
| 159 | } |
| 160 | if err := track.WriteSample(media.Sample{Data: au, Duration: d}); err != nil { |
| 161 | log.Printf("write sample: %v", err) |
| 162 | } |
| 163 | frames++ |
| 164 | bytesOut += len(au) |
| 165 | au = nil |
| 166 | auHasSlice = false |
| 167 | } |
| 168 | |
| 169 | for nal := range nalUnits(r) { |
| 170 | if len(nal) == 0 { |
| 171 | continue |
| 172 | } |
| 173 | typ := nal[0] & 0x1f |
| 174 | isSlice := typ == 1 || typ == 5 |
| 175 | // A new slice after we already hold one means a new frame started. |
| 176 | if isSlice && auHasSlice && firstMB(nal) { |
| 177 | flush() |
| 178 | } |
| 179 | if typ == 5 && onKeyframe != nil { |
| 180 | onKeyframe() |
| 181 | } |
| 182 | au = append(au, 0, 0, 0, 1) |
| 183 | au = append(au, nal...) |
| 184 | if isSlice { |
| 185 | auHasSlice = true |
| 186 | } |
| 187 | select { |
| 188 | case <-tick.C: |
| 189 | log.Printf("video: %.0f fps, %.1f Mbit/s", float64(frames)/5, float64(bytesOut)*8/5/1e6) |
| 190 | frames, bytesOut = 0, 0 |
| 191 | default: |
| 192 | } |
| 193 | } |
| 194 | flush() |
| 195 | return fmt.Errorf("ffmpeg stdout closed") |
| 196 | } |
| 197 | |
| 198 | // firstMB reports whether a slice NAL has first_mb_in_slice == 0, i.e. it is |
| 199 | // the first slice of a picture. ue(v) == 0 is encoded as a leading 1 bit. |
| 200 | func firstMB(nal []byte) bool { |
| 201 | return len(nal) > 1 && nal[1]&0x80 != 0 |
| 202 | } |
| 203 | |
| 204 | // nalUnits yields NAL units (without start codes) from an Annex-B stream. |
| 205 | func nalUnits(r *bufio.Reader) func(yield func([]byte) bool) { |
| 206 | return func(yield func([]byte) bool) { |
| 207 | var buf []byte |
| 208 | tmp := make([]byte, 64<<10) |
| 209 | start := -1 // index in buf where the current NAL payload starts |
| 210 | for { |
| 211 | n, err := r.Read(tmp) |
| 212 | if n > 0 { |
| 213 | buf = append(buf, tmp[:n]...) |
| 214 | for { |
| 215 | // look for the next start code after 'start' |
| 216 | from := 0 |
| 217 | if start >= 0 { |
| 218 | from = start |
| 219 | } |
| 220 | idx := bytes.Index(buf[from:], []byte{0, 0, 1}) |
| 221 | if idx < 0 { |
| 222 | break |
| 223 | } |
| 224 | idx += from |
| 225 | if start >= 0 { |
| 226 | end := idx |
| 227 | if end > 0 && buf[end-1] == 0 { // 4-byte start code |
| 228 | end-- |
| 229 | } |
| 230 | if !yield(buf[start:end]) { |
| 231 | return |
| 232 | } |
| 233 | } |
| 234 | start = idx + 3 |
| 235 | // drop consumed bytes to keep buf small |
| 236 | if start > 1<<20 { |
| 237 | buf = append([]byte(nil), buf[start:]...) |
| 238 | start = 0 |
| 239 | } |
| 240 | } |
| 241 | } |
| 242 | if err != nil { |
| 243 | if start >= 0 && start < len(buf) { |
| 244 | yield(buf[start:]) |
| 245 | } |
| 246 | if err != io.EOF { |
| 247 | log.Printf("read ffmpeg: %v", err) |
| 248 | } |
| 249 | return |
| 250 | } |
| 251 | } |
| 252 | } |
| 253 | } |