Your phone is the terminal. Stream a Linux/Mac desktop to an iPhone over WebRTC through NAT, with audio and input. Open source, single binaries.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/network-computer.git
HTTPS
https://gitstable.com/git/askrobots/network-computer.git
SSH
ssh://git@gitstable.com:2222/askrobots/network-computer.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

Dan Quellhorst Firefox policy: no WebRTC mDNS (a page using WebRTC made Firefox listen on UDP 5353) a1d92f3 1 month, 3 weeks ago History
nc-desk · 14.3 KB · 398 lines
1#!/usr/bin/env python3
2"""nc-desk: control the desktop from a script, a voice, or an AI agent.
3
4Every command prints one JSON object, {"ok": true, ...} or {"ok": false,
5"error": "..."}, and exits 0 or 1. Runs as the desk user in the desktop
6session (from root, or without a display, it re-runs itself through nc-as-user).
7
8Windows
9 nc-desk windows open windows: id, title, class, x, y, w, h
10 nc-desk arrange WINDOW left|right|top|bottom|max|restore|center|min|close
11 nc-desk place WINDOW X Y W H exact position and size
12 nc-desk focus WINDOW
13Mouse and keyboard (screen pixels; `nc-desk screen` gives the size)
14 nc-desk click X Y [left|right|middle] [double]
15 nc-desk drag X1 Y1 X2 Y2
16 nc-desk scroll X Y up|down [STEPS]
17 nc-desk type TEXT
18 nc-desk key COMBO... xdotool names: ctrl+s, Return, alt+Tab
19Screen
20 nc-desk screen size and work area (without the panel)
21 nc-desk screenshot FILE [MAXWIDTH] JPEG; prints the scale from image to screen
22Files (only inside the home folder; never overwrites; Trash can be undone)
23 nc-desk ls [PATH] [GLOB]
24 nc-desk mkdir PATH
25 nc-desk move SRC... DEST DEST: a folder, or a new name for one SRC
26 nc-desk copy SRC... DEST
27 nc-desk trash PATH...
28Apps (the dbbasic apps, driven by name through their control socket)
29 nc-desk apps running dbbasic apps: their actions and state
30 nc-desk app NAME[@PID] ACTION [JSON-ARGS] nc-desk app writer insert_text '{"text": "Hi"}'
31
32WINDOW is part of a window's title or class, case-insensitive ("firefox",
33"Mousepad"), or an id from `nc-desk windows`.
34"""
35import fnmatch
36import json
37import os
38import re
39import shutil
40import subprocess
41import sys
42
43HOME = os.path.expanduser("~")
44
45
46class Fail(Exception):
47 pass
48
49
50def run(*cmd, check=True):
51 r = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
52 if check and r.returncode != 0:
53 raise Fail(f"{cmd[0]}: {(r.stderr or r.stdout).strip() or 'failed'}")
54 return r.stdout
55
56
57# ---------- windows ----------
58
59def windows():
60 out = []
61 geo = {}
62 for line in run("wmctrl", "-lG").splitlines():
63 p = line.split(None, 7)
64 if len(p) == 8:
65 geo[p[0]] = [int(v) for v in p[2:6]]
66 for line in run("wmctrl", "-lx").splitlines():
67 p = line.split(None, 4)
68 if len(p) < 5 or p[1] == "-1": # -1: sticky (panel, desktop)
69 continue
70 wid, cls, title = p[0], p[2], p[4]
71 x, y, w, h = geo.get(wid, [0, 0, 0, 0])
72 out.append({"id": wid, "title": title, "class": cls, "x": x, "y": y, "w": w, "h": h})
73 return out
74
75
76def find_window(spec):
77 wins = windows()
78 for w in wins:
79 if w["id"] == spec:
80 return w
81 s = spec.lower()
82 hits = [w for w in wins if s in w["title"].lower() or s in w["class"].lower()]
83 if not hits:
84 raise Fail(f"no window like {spec!r}")
85 return hits[0]
86
87
88def work_area():
89 """The usable screen: without the panel."""
90 out = run("xprop", "-root", "_NET_WORKAREA", check=False)
91 m = re.search(r"=\s*(\d+),\s*(\d+),\s*(\d+),\s*(\d+)", out)
92 if m:
93 return [int(v) for v in m.groups()]
94 w, h = screen_size()
95 return [0, 0, w, h]
96
97
98def screen_size():
99 m = re.search(r"dimensions:\s+(\d+)x(\d+)", run("xdpyinfo"))
100 return int(m.group(1)), int(m.group(2))
101
102
103def unmax(wid):
104 run("wmctrl", "-i", "-r", wid, "-b", "remove,maximized_vert,maximized_horz", check=False)
105
106
107def place(wid, x, y, w, h):
108 unmax(wid)
109 run("wmctrl", "-i", "-r", wid, "-e", f"0,{x},{y},{w},{h}")
110 run("wmctrl", "-i", "-a", wid)
111
112
113def arrange(spec, how):
114 win = find_window(spec)
115 wid = win["id"]
116 X, Y, W, H = work_area()
117 if how == "max":
118 run("wmctrl", "-i", "-r", wid, "-b", "add,maximized_vert,maximized_horz")
119 run("wmctrl", "-i", "-a", wid)
120 elif how == "restore":
121 unmax(wid)
122 elif how == "min":
123 run("xdotool", "windowminimize", str(int(wid, 16)))
124 elif how == "close":
125 run("wmctrl", "-i", "-c", wid)
126 elif how == "left":
127 place(wid, X, Y, W // 2, H)
128 elif how == "right":
129 place(wid, X + W // 2, Y, W - W // 2, H)
130 elif how == "top":
131 place(wid, X, Y, W, H // 2)
132 elif how == "bottom":
133 place(wid, X, Y + H // 2, W, H - H // 2)
134 elif how == "center":
135 w, h = min(win["w"] or W * 2 // 3, W), min(win["h"] or H * 2 // 3, H)
136 place(wid, X + (W - w) // 2, Y + (H - h) // 2, w, h)
137 else:
138 raise Fail(f"arrange how? {how!r}")
139 return {"window": win["title"], "arranged": how}
140
141
142# ---------- mouse and keyboard ----------
143
144BUTTONS = {"left": "1", "middle": "2", "right": "3"}
145
146
147def click(x, y, button="left", double=""):
148 b = BUTTONS.get(button)
149 if not b:
150 raise Fail(f"button? {button!r}")
151 cmd = ["xdotool", "mousemove", "--sync", str(int(x)), str(int(y)), "click"]
152 if double == "double":
153 cmd += ["--repeat", "2", "--delay", "80"]
154 run(*cmd, b)
155 return {"clicked": [int(x), int(y)], "button": button, "double": double == "double"}
156
157
158def drag(x1, y1, x2, y2):
159 run("xdotool", "mousemove", "--sync", str(int(x1)), str(int(y1)), "mousedown", "1",
160 "mousemove", "--sync", str(int(x2)), str(int(y2)), "sleep", "0.1", "mouseup", "1")
161 return {"dragged": [[int(x1), int(y1)], [int(x2), int(y2)]]}
162
163
164def scroll(x, y, direction, steps="3"):
165 b = {"up": "4", "down": "5"}.get(direction)
166 if not b:
167 raise Fail(f"scroll up or down, not {direction!r}")
168 run("xdotool", "mousemove", "--sync", str(int(x)), str(int(y)), "click", "--repeat", str(int(steps)), b)
169 return {"scrolled": direction, "steps": int(steps)}
170
171
172def keys(*combos):
173 for c in combos:
174 if not re.fullmatch(r"[A-Za-z0-9_+]{1,40}", c):
175 raise Fail(f"odd key {c!r}")
176 run("xdotool", "key", "--", *combos)
177 return {"pressed": list(combos)}
178
179
180def type_text(text):
181 run("xdotool", "type", "--delay", "8", "--", text)
182 return {"typed": len(text)}
183
184
185# ---------- screen ----------
186
187def screenshot(path, maxwidth="1280"):
188 w, h = screen_size()
189 mw = min(int(maxwidth), w)
190 run("ffmpeg", "-loglevel", "error", "-y", "-f", "x11grab", "-video_size", f"{w}x{h}", "-i", ":0",
191 "-frames:v", "1", "-vf", f"scale={mw}:-2", "-q:v", "4", "-f", "image2", "-c:v", "mjpeg", path)
192 return {"file": path, "screen": [w, h], "image": [mw, round(h * mw / w / 2) * 2], "scale": w / mw}
193
194
195# ---------- files ----------
196
197def inside_home(p, must_exist=True):
198 full = os.path.realpath(os.path.expanduser(p if p.startswith(("/", "~")) else os.path.join(HOME, p)))
199 homes = {os.path.realpath(HOME)} | {os.path.realpath(os.path.join(HOME, d))
200 for d in os.listdir(HOME) if os.path.islink(os.path.join(HOME, d))}
201 if not any(full == h or full.startswith(h + os.sep) for h in homes):
202 raise Fail(f"{p}: outside your home folder")
203 if must_exist and not os.path.lexists(full):
204 raise Fail(f"{p}: not found")
205 return full
206
207
208def show(p):
209 home = os.path.realpath(HOME)
210 for d in os.listdir(HOME): # show desk-linked folders by their home name
211 link = os.path.join(HOME, d)
212 if os.path.islink(link) and (p == os.path.realpath(link) or p.startswith(os.path.realpath(link) + os.sep)):
213 return "~/" + d + p[len(os.path.realpath(link)):]
214 return "~" + p[len(home):] if p.startswith(home) else p
215
216
217def ls(path="~", pattern="*"):
218 d = inside_home(path)
219 items = []
220 for name in sorted(os.listdir(d)):
221 if name.startswith(".") or not fnmatch.fnmatch(name.lower(), pattern.lower()):
222 continue
223 full = os.path.join(d, name)
224 items.append({"name": name, "dir": os.path.isdir(full),
225 "size": os.path.getsize(full) if os.path.isfile(full) else None})
226 return {"path": show(d), "items": items[:300], "more": max(0, len(items) - 300)}
227
228
229def not_hidden(full):
230 """File actions stay out of hidden files and folders (~/.ssh, ~/.config...)."""
231 rel = os.path.relpath(full, os.path.realpath(HOME))
232 for home_link in os.listdir(HOME): # desk-linked folders count as their home name
233 real = os.path.realpath(os.path.join(HOME, home_link))
234 if os.path.islink(os.path.join(HOME, home_link)) and (full == real or full.startswith(real + os.sep)):
235 rel = os.path.join(home_link, os.path.relpath(full, real))
236 if any(part.startswith(".") and part not in (".", "..") for part in rel.split(os.sep)):
237 raise Fail(f"{show(full)}: hidden files and folders are left alone")
238 return full
239
240
241def mkdir(path):
242 full = not_hidden(inside_home(path, must_exist=False))
243 os.makedirs(full, exist_ok=True)
244 return {"made": show(full)}
245
246
247def targets(srcs, dest):
248 """Where each source goes; refuses anything that would overwrite."""
249 srcs = [not_hidden(inside_home(s)) for s in srcs]
250 d = not_hidden(inside_home(dest, must_exist=False))
251 if os.path.isdir(d):
252 pairs = [(s, os.path.join(d, os.path.basename(s))) for s in srcs]
253 elif len(srcs) == 1:
254 if not os.path.isdir(os.path.dirname(d)):
255 raise Fail(f"{dest}: no such folder")
256 pairs = [(srcs[0], d)]
257 else:
258 raise Fail(f"{dest}: not a folder (make it first)")
259 for s, t in pairs:
260 if os.path.lexists(t):
261 raise Fail(f"{show(t)} already exists: not overwriting")
262 if os.path.realpath(t).startswith(os.path.realpath(s) + os.sep):
263 raise Fail(f"can't put {show(s)} inside itself")
264 return pairs
265
266
267def move(*args):
268 if len(args) < 2:
269 raise Fail("move SRC... DEST")
270 done = []
271 for s, t in targets(args[:-1], args[-1]):
272 shutil.move(s, t)
273 done.append([show(s), show(t)])
274 return {"moved": done}
275
276
277def copy(*args):
278 if len(args) < 2:
279 raise Fail("copy SRC... DEST")
280 done = []
281 for s, t in targets(args[:-1], args[-1]):
282 (shutil.copytree if os.path.isdir(s) else shutil.copy2)(s, t)
283 done.append([show(s), show(t)])
284 return {"copied": done}
285
286
287def trash(*paths):
288 done = []
289 for p in paths:
290 full = not_hidden(inside_home(p))
291 if full in {os.path.realpath(HOME)} | {os.path.realpath(os.path.join(HOME, d)) for d in os.listdir(HOME)}:
292 if os.path.dirname(full) == os.path.realpath(HOME) or os.path.islink(os.path.join(HOME, os.path.basename(full))):
293 raise Fail(f"{p}: won't trash a top-level folder")
294 run("gio", "trash", full)
295 done.append(show(full))
296 return {"trashed": done, "undo": "restore from the Trash"}
297
298
299# ---------- the dbbasic apps' control sockets (dbbasic_app_kit AppControl) ----------
300
301def app_sockets():
302 """name -> socket path, the newest process of each app first."""
303 uid = os.getuid()
304 dirs = [os.path.join(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{uid}"), "dbbasic"),
305 f"/run/user/{uid}/dbbasic", f"/tmp/dbbasic-{os.environ.get('USER', '')}"]
306 found = {}
307 for d in dict.fromkeys(dirs):
308 try:
309 names = os.listdir(d)
310 except OSError:
311 continue
312 for n in names:
313 m = re.fullmatch(r"([a-z][a-z0-9_-]*)-(\d+)\.sock", n)
314 if m and os.path.exists(f"/proc/{m.group(2)}"):
315 found.setdefault(m.group(1), []).append((int(m.group(2)), os.path.join(d, n)))
316 return {k: [p for _, p in sorted(v, reverse=True)] for k, v in found.items()}
317
318
319def app_request(path, action, args=None, timeout=20):
320 import socket
321 with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as c:
322 c.settimeout(timeout)
323 c.connect(path)
324 c.sendall((json.dumps({"action": action, "args": args or {}}) + "\n").encode())
325 data = b""
326 while not data.endswith(b"\n"):
327 chunk = c.recv(65536)
328 if not chunk:
329 break
330 data += chunk
331 return json.loads(data.decode() or "{}")
332
333
334def apps():
335 out = []
336 for name, paths in sorted(app_sockets().items()):
337 try:
338 r = app_request(paths[0], "describe", timeout=5)
339 if r.get("ok"):
340 out.append(r["result"])
341 except (OSError, ValueError):
342 pass
343 return {"apps": out}
344
345
346def app(name, action, args="{}"):
347 # NAME@PID: that instance exactly (a release test beside the person's own copy)
348 name, _, pid = name.partition("@")
349 paths = app_sockets().get(name.lower())
350 if paths and pid:
351 paths = [p for p in paths if p.endswith(f"-{pid}.sock")]
352 if not paths:
353 raise Fail(f"{name}{'@' + pid if pid else ''} is not running (or has no control); open it first")
354 try:
355 parsed = json.loads(args)
356 except ValueError:
357 raise Fail("JSON-ARGS must be a JSON object")
358 if not isinstance(parsed, dict):
359 raise Fail("JSON-ARGS must be a JSON object")
360 r = app_request(paths[0], action, parsed)
361 if not r.get("ok"):
362 raise Fail(r.get("error") or f"{name} refused {action}")
363 return {"app": name, "action": action, "result": r.get("result")}
364
365
366COMMANDS = {
367 "windows": lambda: {"windows": windows()},
368 "arrange": arrange,
369 "place": lambda spec, x, y, w, h: (place(find_window(spec)["id"], int(x), int(y), int(w), int(h)),
370 {"placed": spec})[1],
371 "focus": lambda spec: (run("wmctrl", "-i", "-a", find_window(spec)["id"]), {"focused": spec})[1],
372 "click": click, "drag": drag, "scroll": scroll, "type": type_text, "key": keys,
373 "screen": lambda: dict(zip(("w", "h"), screen_size()), work_area=work_area()),
374 "screenshot": screenshot,
375 "ls": ls, "mkdir": mkdir, "move": move, "copy": copy, "trash": trash,
376 "apps": apps, "app": app,
377}
378
379
380def main():
381 if (os.geteuid() == 0 or not os.environ.get("DISPLAY")) and os.environ.get("NC_DESK_NO_REEXEC") != "1":
382 os.execvp("nc-as-user", ["nc-as-user", "env", "NC_DESK_NO_REEXEC=1", sys.argv[0], *sys.argv[1:]])
383 if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS:
384 print(__doc__)
385 sys.exit(2)
386 try:
387 res = COMMANDS[sys.argv[1]](*sys.argv[2:])
388 print(json.dumps({"ok": True, **res}, ensure_ascii=False))
389 except (Fail, TypeError, ValueError, OSError, subprocess.TimeoutExpired) as e:
390 msg = str(e)
391 if isinstance(e, TypeError):
392 msg = f"wrong arguments for {sys.argv[1]}; see nc-desk --help"
393 print(json.dumps({"ok": False, "error": msg}, ensure_ascii=False))
394 sys.exit(1)
395
396
397if __name__ == "__main__":
398 main()