Your phone is the terminal. Stream a Linux/Mac desktop to an iPhone over WebRTC through NAT, with audio and input. Open source, single binaries.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/network-computer.git
HTTPS
https://gitstable.com/git/askrobots/network-computer.git
SSH
ssh://git@gitstable.com:2222/askrobots/network-computer.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

Dan Quellhorst Firefox policy: no WebRTC mDNS (a page using WebRTC made Firefox listen on UDP 5353) a1d92f3 1 month, 3 weeks ago History
capture.go · 7.1 KB · 253 lines
1package main
2
3import (
4 "bufio"
5 "bytes"
6 "context"
7 "fmt"
8 "io"
9 "log"
10 "os"
11 "os/exec"
12 "runtime"
13 "strconv"
14 "strings"
15 "time"
16
17 "github.com/pion/webrtc/v4/pkg/media"
18)
19
20// captureOpts describes one ffmpeg screen pipeline.
21type captureOpts struct {
22 Display int // index of the screen to capture
23 FPS int
24 Width int // 0 = native
25 Height int
26 Bitrate string // e.g. "8M"
27 Encoder string // ffmpeg encoder name, "" = platform default
28 Extra string // raw extra ffmpeg args inserted before the output
29 Custom string // full custom ffmpeg argument string; overrides everything
30}
31
32// ffmpegArgs builds a low-latency H.264 Annex-B pipeline for the current OS.
33// Every branch ends with "-f h264 pipe:1" so the reader is identical.
34func ffmpegArgs(o captureOpts) []string {
35 if o.Custom != "" {
36 return strings.Fields(o.Custom)
37 }
38 fps := strconv.Itoa(o.FPS)
39 var in []string
40 enc := o.Encoder
41 switch runtime.GOOS {
42 case "darwin":
43 // avfoundation screen devices come after the cameras; the index is
44 // looked up by name at startup in resolveDarwinScreen.
45 in = []string{"-f", "avfoundation", "-capture_cursor", "1", "-capture_mouse_clicks", "0",
46 "-framerate", fps, "-pixel_format", "uyvy422", "-i", fmt.Sprintf("%d:none", o.Display)}
47 if enc == "" {
48 enc = "h264_videotoolbox"
49 }
50 case "linux":
51 disp := os.Getenv("DISPLAY")
52 if disp == "" {
53 disp = ":0"
54 }
55 in = []string{"-f", "x11grab", "-framerate", fps, "-draw_mouse", "1", "-i", fmt.Sprintf("%s.%d", disp, o.Display)}
56 if enc == "" {
57 enc = "libx264"
58 }
59 case "windows":
60 in = []string{"-f", "gdigrab", "-framerate", fps, "-draw_mouse", "1", "-i", "desktop"}
61 if enc == "" {
62 enc = "libx264"
63 }
64 default:
65 log.Fatalf("no default capture for %s; use -ffmpeg-args", runtime.GOOS)
66 }
67
68 args := []string{"-hide_banner", "-loglevel", "warning", "-fflags", "nobuffer", "-flags", "low_delay"}
69 args = append(args, in...)
70 // avfoundation reports a bogus nominal rate and ffmpeg would duplicate frames
71 // to match it; passthrough lets the capture device set the pace. x11grab and
72 // gdigrab timestamp correctly and warn about passthrough, so darwin only.
73 if runtime.GOOS == "darwin" {
74 args = append(args, "-fps_mode", "passthrough")
75 }
76 if o.Width > 0 && o.Height > 0 {
77 args = append(args, "-vf", fmt.Sprintf("scale=%d:%d", o.Width, o.Height))
78 }
79 args = append(args, "-c:v", enc, "-b:v", o.Bitrate, "-maxrate", o.Bitrate,
80 "-g", strconv.Itoa(o.FPS*2), "-bf", "0", "-pix_fmt", "yuv420p")
81 switch enc {
82 case "h264_videotoolbox":
83 args = append(args, "-realtime", "1", "-prio_speed", "1", "-profile:v", "main", "-allow_sw", "1")
84 case "libx264":
85 args = append(args, "-preset", "ultrafast", "-tune", "zerolatency", "-profile:v", "baseline", "-x264-params", "repeat-headers=1")
86 case "h264_vaapi", "h264_nvenc", "h264_qsv", "h264_amf":
87 // caller usually needs -Extra for hwupload / device selection
88 }
89 if o.Extra != "" {
90 args = append(args, strings.Fields(o.Extra)...)
91 }
92 return append(args, "-f", "h264", "pipe:1")
93}
94
95// resolveDarwinScreen maps "screen N" to the avfoundation device index by
96// parsing ffmpeg's device list, since cameras shift the numbering.
97func resolveDarwinScreen(n int) (int, error) {
98 out, _ := exec.Command("ffmpeg", "-hide_banner", "-f", "avfoundation", "-list_devices", "true", "-i", "").CombinedOutput()
99 for _, line := range strings.Split(string(out), "\n") {
100 if strings.Contains(line, fmt.Sprintf("Capture screen %d", n)) {
101 i := strings.Index(line, "] [")
102 if i < 0 {
103 continue
104 }
105 rest := line[i+3:]
106 j := strings.Index(rest, "]")
107 return strconv.Atoi(rest[:j])
108 }
109 }
110 return 0, fmt.Errorf("screen %d not found in avfoundation device list", n)
111}
112
113// streamVideo runs ffmpeg and pushes one access unit per WriteSample until
114// ctx is cancelled. It restarts ffmpeg if it dies.
115func streamVideo(ctx context.Context, o captureOpts, track sampleWriter, onKeyframe func()) {
116 for ctx.Err() == nil {
117 if err := runOnce(ctx, o, track, onKeyframe); err != nil && ctx.Err() == nil {
118 log.Printf("ffmpeg: %v (restarting in 1s)", err)
119 time.Sleep(time.Second)
120 }
121 }
122}
123
124type sampleWriter interface {
125 WriteSample(media.Sample) error
126}
127
128func runOnce(ctx context.Context, o captureOpts, track sampleWriter, onKeyframe func()) error {
129 args := ffmpegArgs(o)
130 log.Printf("ffmpeg %s", strings.Join(args, " "))
131 cmd := exec.CommandContext(ctx, "ffmpeg", args...)
132 cmd.Stderr = os.Stderr
133 stdout, err := cmd.StdoutPipe()
134 if err != nil {
135 return err
136 }
137 if err := cmd.Start(); err != nil {
138 return err
139 }
140 defer cmd.Wait()
141
142 r := bufio.NewReaderSize(stdout, 4<<20)
143 var au []byte // access unit being assembled (Annex-B, may hold SPS/PPS/SEI + slice)
144 auHasSlice := false
145 last := time.Now()
146 frames, bytesOut := 0, 0
147 tick := time.NewTicker(5 * time.Second)
148 defer tick.Stop()
149
150 flush := func() {
151 if len(au) == 0 {
152 return
153 }
154 now := time.Now()
155 d := now.Sub(last)
156 last = now
157 if d > time.Second { // first frame or a stall: don't blow up the RTP clock
158 d = time.Second / time.Duration(o.FPS)
159 }
160 if err := track.WriteSample(media.Sample{Data: au, Duration: d}); err != nil {
161 log.Printf("write sample: %v", err)
162 }
163 frames++
164 bytesOut += len(au)
165 au = nil
166 auHasSlice = false
167 }
168
169 for nal := range nalUnits(r) {
170 if len(nal) == 0 {
171 continue
172 }
173 typ := nal[0] & 0x1f
174 isSlice := typ == 1 || typ == 5
175 // A new slice after we already hold one means a new frame started.
176 if isSlice && auHasSlice && firstMB(nal) {
177 flush()
178 }
179 if typ == 5 && onKeyframe != nil {
180 onKeyframe()
181 }
182 au = append(au, 0, 0, 0, 1)
183 au = append(au, nal...)
184 if isSlice {
185 auHasSlice = true
186 }
187 select {
188 case <-tick.C:
189 log.Printf("video: %.0f fps, %.1f Mbit/s", float64(frames)/5, float64(bytesOut)*8/5/1e6)
190 frames, bytesOut = 0, 0
191 default:
192 }
193 }
194 flush()
195 return fmt.Errorf("ffmpeg stdout closed")
196}
197
198// firstMB reports whether a slice NAL has first_mb_in_slice == 0, i.e. it is
199// the first slice of a picture. ue(v) == 0 is encoded as a leading 1 bit.
200func firstMB(nal []byte) bool {
201 return len(nal) > 1 && nal[1]&0x80 != 0
202}
203
204// nalUnits yields NAL units (without start codes) from an Annex-B stream.
205func nalUnits(r *bufio.Reader) func(yield func([]byte) bool) {
206 return func(yield func([]byte) bool) {
207 var buf []byte
208 tmp := make([]byte, 64<<10)
209 start := -1 // index in buf where the current NAL payload starts
210 for {
211 n, err := r.Read(tmp)
212 if n > 0 {
213 buf = append(buf, tmp[:n]...)
214 for {
215 // look for the next start code after 'start'
216 from := 0
217 if start >= 0 {
218 from = start
219 }
220 idx := bytes.Index(buf[from:], []byte{0, 0, 1})
221 if idx < 0 {
222 break
223 }
224 idx += from
225 if start >= 0 {
226 end := idx
227 if end > 0 && buf[end-1] == 0 { // 4-byte start code
228 end--
229 }
230 if !yield(buf[start:end]) {
231 return
232 }
233 }
234 start = idx + 3
235 // drop consumed bytes to keep buf small
236 if start > 1<<20 {
237 buf = append([]byte(nil), buf[start:]...)
238 start = 0
239 }
240 }
241 }
242 if err != nil {
243 if start >= 0 && start < len(buf) {
244 yield(buf[start:])
245 }
246 if err != io.EOF {
247 log.Printf("read ffmpeg: %v", err)
248 }
249 return
250 }
251 }
252 }
253}