Puppet Stage: a browser puppet that lip-syncs to local text-to-speech and takes stage directions, live or from a screenplay.

0 downloads

dbbasic-face/server.js
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-face.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-face.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-face.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

Dan Quellhorst Converse: characters know they have bodies 52ca998 4 months, 1 week ago History
server.js · 51.7 KB · 1251 lines
1#!/usr/bin/env node
2/**
3 * Puppet stage server.
4 *
5 * - Renders TTS to WAV via macOS `say` or `espeak`, extracts a viseme
6 * timeline with Rhubarb Lip Sync, and serves both to the browser stage.
7 * - Broadcasts direction cues (speech, actions, movement) to all connected
8 * stages over Server-Sent Events.
9 * - Runs "screenplay" scripts: plain lines are spoken, [bracketed] lines are
10 * stage directions, (action) prefixes run concurrently with speech.
11 *
12 * Zero npm dependencies.
13 */
14'use strict';
15
16const http = require('http');
17const fs = require('fs');
18const path = require('path');
19const crypto = require('crypto');
20const { execFile } = require('child_process');
21
22const ROOT = __dirname;
23const CACHE = path.join(ROOT, 'cache');
24const PUBLIC = path.join(ROOT, 'public');
25const CHARACTERS = path.join(ROOT, 'characters');
26const ASSETS = path.join(ROOT, 'assets');
27const EXAMPLES = path.join(ROOT, 'examples');
28const RHUBARB = findRhubarb();
29const PORT = process.env.PORT || 3123;
30
31// Format-agnostic asset resolution: an asset id resolves to the first
32// existing file among a known extension set (SVG first, then raster), so
33// PNG/JPG/WebP/GIF drop into the same slots as SVG with no code change.
34const ASSET_EXTS = ['.svg', '.png', '.jpg', '.jpeg', '.webp', '.gif'];
35
36// music/sfx assets are JSON note patterns (see docs/design/actors-wearables-
37// music.md), not images, so they resolve against their own extension set.
38const AUDIO_KINDS = ['music', 'sfx'];
39const AUDIO_EXTS = ['.json'];
40
41function resolveAsset(kind, id) {
42 const exts = AUDIO_KINDS.includes(kind) ? AUDIO_EXTS : ASSET_EXTS;
43 for (const ext of exts) {
44 const file = path.join(ASSETS, kind, id + ext);
45 if (fs.existsSync(file)) return file;
46 }
47 return null;
48}
49
50function findRhubarb() {
51 if (process.env.RHUBARB_PATH) return process.env.RHUBARB_PATH;
52 const toolsDir = path.join(ROOT, 'tools');
53 try {
54 for (const entry of fs.readdirSync(toolsDir)) {
55 const candidate = path.join(toolsDir, entry, 'rhubarb');
56 if (fs.existsSync(candidate)) return candidate;
57 }
58 } catch { /* no tools dir yet */ }
59 return 'rhubarb'; // hope it's on PATH; probeRhubarb() handles absence
60}
61
62fs.mkdirSync(CACHE, { recursive: true });
63
64// ---------------------------------------------------------------- helpers
65
66function run(cmd, args, opts = {}) {
67 return new Promise((resolve, reject) => {
68 execFile(cmd, args, { maxBuffer: 16 * 1024 * 1024, ...opts }, (err, stdout, stderr) => {
69 if (err) reject(new Error(`${cmd} failed: ${stderr || err.message}`));
70 else resolve(stdout);
71 });
72 });
73}
74
75function sha1(s) {
76 return crypto.createHash('sha1').update(s).digest('hex').slice(0, 16);
77}
78
79function readBody(req) {
80 return new Promise((resolve, reject) => {
81 let data = '';
82 req.on('data', (c) => { data += c; if (data.length > 1e6) req.destroy(); });
83 req.on('end', () => {
84 try { resolve(data ? JSON.parse(data) : {}); } catch (e) { reject(e); }
85 });
86 req.on('error', reject);
87 });
88}
89
90function readRaw(req, limit = 32 * 1024 * 1024) {
91 return new Promise((resolve, reject) => {
92 const chunks = [];
93 let size = 0;
94 req.on('data', (c) => {
95 size += c.length;
96 if (size > limit) { req.destroy(); reject(new Error('body too large')); return; }
97 chunks.push(c);
98 });
99 req.on('end', () => resolve(Buffer.concat(chunks)));
100 req.on('error', reject);
101 });
102}
103
104function json(res, code, obj) {
105 const body = JSON.stringify(obj);
106 res.writeHead(code, { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) });
107 res.end(body);
108}
109
110// ------------------------------------------------------------ SSE clients
111
112const clients = new Set();
113
114function broadcast(cue) {
115 const msg = `data: ${JSON.stringify(cue)}\n\n`;
116 for (const res of clients) res.write(msg);
117}
118
119setInterval(() => { for (const res of clients) res.write(': ping\n\n'); }, 25000);
120
121// ------------------------------------------------------------- TTS + sync
122
123let rhubarbOk = null; // lazily probed
124let lastSlowpx = 0; // Date.now() at the start of the most recent /slowpx request (nav marker for screenshot timing)
125
126async function probeRhubarb() {
127 if (rhubarbOk !== null) return rhubarbOk;
128 try { await run(RHUBARB, ['--version']); rhubarbOk = true; }
129 catch { rhubarbOk = false; console.warn('rhubarb unavailable; falling back to amplitude lip-sync'); }
130 return rhubarbOk;
131}
132
133// Per-key in-flight render dedup: two concurrent callers for the same cache
134// key (e.g. a script re-performed while its render-ahead window is still
135// in flight, or /api/say racing that same render-ahead) must not both render
136// to the same raw temp path — the second render's ffmpeg step can lose the
137// race against the first's cleanup unlink ("ffmpeg failed"). Instead, the
138// second caller awaits the first's promise and both read the same result.
139const speechInFlight = new Map(); // key -> Promise<void>
140const externalAudioInFlight = new Map(); // key -> Promise<void>
141
142/**
143 * Render `text` with the given engine and produce {audio, timeline, duration}.
144 * Results are cached by content hash in cache/.
145 */
146async function prepareSpeech({ text, engine = 'say', voice = '', rate = 0 }) {
147 engine = engine === 'espeak' ? 'espeak' : 'say';
148 const key = sha1([engine, voice, rate, text].join('|'));
149 const wav = path.join(CACHE, `${key}.wav`);
150 const meta = path.join(CACHE, `${key}.meta.json`);
151
152 if (!fs.existsSync(wav) || !fs.existsSync(meta)) {
153 let p = speechInFlight.get(key);
154 if (!p) {
155 p = renderSpeech(key, wav, meta, { text, engine, voice, rate });
156 speechInFlight.set(key, p);
157 p.finally(() => {
158 if (speechInFlight.get(key) === p) speechInFlight.delete(key);
159 });
160 }
161 await p;
162 }
163
164 const { timeline, duration } = JSON.parse(fs.readFileSync(meta, 'utf8'));
165 return { audio: `/audio/${key}.wav`, timeline, duration, text };
166}
167
168// Does the actual say/espeak -> ffmpeg -> rhubarb render for prepareSpeech.
169// Split out so prepareSpeech can dedup concurrent callers on `key` instead
170// of each starting its own render into the same raw temp path.
171async function renderSpeech(key, wav, meta, { text, engine, voice, rate }) {
172 const raw = path.join(CACHE, `${key}.raw.${engine === 'say' ? 'aiff' : 'wav'}`);
173 if (engine === 'say') {
174 const args = ['-o', raw];
175 if (voice) args.push('-v', voice);
176 if (rate) args.push('-r', String(rate));
177 args.push(text);
178 await run('say', args);
179 } else {
180 const args = ['-w', raw];
181 if (voice) args.push('-v', voice);
182 if (rate) args.push('-s', String(rate));
183 args.push(text);
184 await run('espeak', args);
185 }
186 await run('ffmpeg', ['-y', '-loglevel', 'error', '-i', raw,
187 '-ar', '16000', '-ac', '1', '-c:a', 'pcm_s16le', wav]);
188 fs.unlinkSync(raw);
189
190 let timeline = null;
191 let duration = 0;
192 if (await probeRhubarb()) {
193 try {
194 const dialog = path.join(CACHE, `${key}.txt`);
195 fs.writeFileSync(dialog, text);
196 const out = await run(RHUBARB, ['-f', 'json', '--extendedShapes', 'GHX',
197 '--dialogFile', dialog, wav]);
198 const parsed = JSON.parse(out);
199 timeline = parsed.mouthCues;
200 duration = parsed.metadata.duration;
201 fs.unlinkSync(dialog);
202 } catch (e) {
203 console.warn('rhubarb failed for this clip:', e.message);
204 }
205 }
206 if (!duration) {
207 const probe = await run('ffprobe', ['-v', 'error', '-show_entries',
208 'format=duration', '-of', 'csv=p=0', wav]);
209 duration = parseFloat(probe) || 0;
210 }
211 fs.writeFileSync(meta, JSON.stringify({ timeline, duration }));
212}
213
214/**
215 * Lip-sync audio rendered elsewhere (e.g. an external TTS service):
216 * normalize it, run Rhubarb (using `text` as a dialog hint if given),
217 * and return the same {audio, timeline, duration} shape as prepareSpeech.
218 */
219async function prepareExternalAudio(bytes, text = '') {
220 const key = sha1('ext|' + text + '|' + crypto.createHash('sha1').update(bytes).digest('hex'));
221 const wav = path.join(CACHE, `${key}.wav`);
222 const meta = path.join(CACHE, `${key}.meta.json`);
223
224 if (!fs.existsSync(wav) || !fs.existsSync(meta)) {
225 let p = externalAudioInFlight.get(key);
226 if (!p) {
227 p = renderExternalAudio(key, wav, meta, bytes, text);
228 externalAudioInFlight.set(key, p);
229 p.finally(() => {
230 if (externalAudioInFlight.get(key) === p) externalAudioInFlight.delete(key);
231 });
232 }
233 await p;
234 }
235
236 const { timeline, duration } = JSON.parse(fs.readFileSync(meta, 'utf8'));
237 return { audio: `/audio/${key}.wav`, timeline, duration, text };
238}
239
240// Does the actual ffmpeg -> rhubarb render for prepareExternalAudio. Split
241// out so prepareExternalAudio can dedup concurrent callers on `key` instead
242// of each starting its own render into the same raw temp path.
243async function renderExternalAudio(key, wav, meta, bytes, text) {
244 const raw = path.join(CACHE, `${key}.raw`);
245 fs.writeFileSync(raw, bytes);
246 await run('ffmpeg', ['-y', '-loglevel', 'error', '-i', raw,
247 '-ar', '16000', '-ac', '1', '-c:a', 'pcm_s16le', wav]);
248 fs.unlinkSync(raw);
249
250 let timeline = null;
251 let duration = 0;
252 if (await probeRhubarb()) {
253 try {
254 const args = ['-f', 'json', '--extendedShapes', 'GHX'];
255 if (text) {
256 const dialog = path.join(CACHE, `${key}.txt`);
257 fs.writeFileSync(dialog, text);
258 args.push('--dialogFile', dialog);
259 }
260 const out = await run(RHUBARB, [...args, wav]);
261 const parsed = JSON.parse(out);
262 timeline = parsed.mouthCues;
263 duration = parsed.metadata.duration;
264 } catch (e) {
265 console.warn('rhubarb failed for external audio:', e.message);
266 }
267 }
268 if (!duration) {
269 const probe = await run('ffprobe', ['-v', 'error', '-show_entries',
270 'format=duration', '-of', 'csv=p=0', wav]);
271 duration = parseFloat(probe) || 0;
272 }
273 fs.writeFileSync(meta, JSON.stringify({ timeline, duration }));
274}
275
276// ---------------------------------------------------------- script runner
277
278const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
279
280// rough pacing estimates for sequencing; the browser animates the real thing
281const ACTION_SECONDS = { wave: 1.6, jump: 0.9, nod: 1.2, shake: 1.2, bow: 1.8, dance: 2.4 };
282
283// Layout presets: macros that clear existing frames and create the named
284// ones. Only the frame-id bookkeeping needed while parsing lives here — the
285// actual rect math is the client's job (public/stage.js).
286const LAYOUT_FRAMES = {
287 single: ['main'],
288 split: ['left', 'right'],
289 stack: ['top', 'bottom'],
290 thirds: ['third-l', 'third-c', 'third-r'],
291 'pip-tr': ['main', 'pip'], 'pip-tl': ['main', 'pip'],
292 'pip-br': ['main', 'pip'], 'pip-bl': ['main', 'pip'],
293};
294
295// Tokenize `key:value key:"quoted value"` pairs, e.g. `character:ava bg:desk`.
296function parseKV(str) {
297 const out = {};
298 const re = /([a-z][\w-]*):("([^"]*)"|\S+)/gi;
299 let m;
300 while ((m = re.exec(str))) out[m[1].toLowerCase()] = m[3] !== undefined ? m[3] : m[2];
301 return out;
302}
303
304// left/center|middle/right resolve to stage-percent positions (25/50/75,
305// case-insensitive); anything else parses as a literal percent number.
306// Used wherever a direction's x might be authored as a word instead of a
307// number: [place ... at <pos>], [walk to <pos>], [<id> walk <pos>],
308// [<id> move <pos>]. Does NOT apply to [enter/exit] left|right, which mean
309// off-stage sides, not on-stage positions.
310function parsePosWord(v) {
311 const w = String(v == null ? '' : v).toLowerCase();
312 if (w === 'left') return 25;
313 if (w === 'center' || w === 'middle') return 50;
314 if (w === 'right') return 75;
315 return parseFloat(v);
316}
317
318// Resolves a bare id/name to the frame or actor it refers to: a declared
319// frame id or actor id wins as-is; otherwise, if `id` names a character
320// who's been framed/placed (tracked in charLocs, see parseScript), resolve
321// through that mapping. Falls back to `id` itself (unresolved) so callers
322// keep their existing "unknown target" handling for genuinely bad input.
323function resolveTarget(id, frameIds, actorIds, charLocs) {
324 if (frameIds.has(id) || actorIds.has(id)) return id;
325 const loc = charLocs && charLocs.get(id);
326 return loc ? loc.id : id;
327}
328
329// Speaker-prefix resolution shared by plain "<id>: text" lines and the
330// bracket-line tolerance in parseScript: frame id -> actor id -> character
331// name (via charLocs). Returns {frame|actor, text} or null if the label
332// isn't a known target, in which case the string is left alone by the
333// caller (spoken/directed verbatim, prefix and all).
334function matchSpeaker(text, frameIds, actorIds, charLocs) {
335 const m = text.match(/^([A-Za-z][\w-]*):\s*(.*)$/);
336 if (!m) return null;
337 if (frameIds.has(m[1])) return { frame: m[1], text: m[2] };
338 if (actorIds.has(m[1])) return { actor: m[1], text: m[2] };
339 const loc = charLocs && charLocs.get(m[1]);
340 if (loc) return loc.kind === 'frame' ? { frame: loc.id, text: m[2] } : { actor: loc.id, text: m[2] };
341 return null;
342}
343
344/**
345 * Parse a screenplay into a cue list.
346 * [wave] → action cue
347 * [walk to 70] → movement (percent of stage width; also accepts
348 * left|center|middle|right as 25/50/75)
349 * [look left|right|front], [emote happy], [wait 1.5]
350 * [engine espeak], [voice Samantha], [rate 180] → speech settings
351 * [layout split] → frame layout preset
352 * [frame left character:ava bg:desk view:face] → create/update a frame
353 * [frame right clear] → remove a frame
354 * [scene desk] → set the active frame's background
355 * [show image:chart fit:contain] → content tile in the active frame
356 * [lower-third "Ava Reyes" "Host"] → overlay, auto-dismissed after a
357 * 6000ms hold by default; add a trailing
358 * `hold:<ms>` to override (`hold:0` persists —
359 * no auto-dismiss — matching the old behavior)
360 * [lower-third clear] → remove the current lower-third early
361 * [clear] → wipe every placed actor (and its worn props),
362 * every frame's content tile, and every overlay
363 * across the whole stage — the same cleanup a
364 * new script runs automatically on script-start.
365 * Music, captions on/off, backgrounds, and frame
366 * layout are left alone.
367 * [captions on|off] → toggle the caption bar (default on)
368 * [iris out|in 900], [fade out|in 900] → fullscreen transition
369 * (ms optional, default 700)
370 * left: Good evening. → speak cue targeting frame "left" (id must be
371 * a frame declared earlier in the script); once
372 * a character has been framed/placed, its NAME
373 * works the same way — `rex: Woof!` resolves to
374 * whichever frame/actor rex currently occupies
375 * (resolution order: frame id, actor id, name)
376 * [left: Good evening.] → same as the line above, tolerated: a bracketed
377 * line whose inner text is itself a speaker
378 * prefix is unwrapped to a spoken line rather
379 * than treated as an unknown direction
380 * [left wave] → direction targeting frame "left"; a placed/
381 * framed character's name also routes here,
382 * e.g. [rex sit] once rex occupies a frame/actor
383 * (wave) Hello! → action fired concurrently with the line
384 * Hello! → speech
385 *
386 * [place hoop at 70 scale:0.5] → place a prop/character actor
387 * ("at" required — a percent number or
388 * left|center|middle|right (25/50/75); "id:"
389 * defaults to the name, "scale:" defaults 1 for
390 * characters/0.4 for props, bare "behind" flag
391 * draws it behind the frame's primary character)
392 * [place rex at 15 scale:0.6 id:sidekick] → `what` names a folder in
393 * characters/ → a full rig; otherwise it
394 * resolves as assets/props/<what>
395 * [remove sidekick] → destroy a placed actor
396 * [<id> move 40], [<id> scale 0.8], [<id> spin], [<id> bounce] → prop
397 * actor directions (glide/resize/spin/hop)
398 * sidekick: Woof! → speak cue targeting an actor id (same
399 * resolution order as frame ids: frame first,
400 * then actor)
401 * [sidekick leap] → direction on a character actor — the full
402 * direction set applies (walk/emote/actions/
403 * speak), same grammar as a frame id
404 * [wear right tophat], [wear right tophat scale:1.2 anchor:head],
405 * [unwear right] → pin/remove a prop asset at a character's
406 * named anchor (`target` is a frame id — its
407 * primary character — an actor id, or (once
408 * framed/placed) a character name; anchor
409 * defaults to "head")
410 * [music circus] → cross-fade to assets/music/circus.json
411 * (a looping JSON note pattern), stage-global
412 * [music off] → fade out and stop the current music
413 * [sfx tada] → one-shot assets/sfx/tada.json, not looped,
414 * not ducked
415 */
416function parseScript(src) {
417 const cues = [];
418 const frameIds = new Set(['main']); // tracks declared frame ids as we go
419 const actorIds = new Set(); // tracks ids declared by [place ...] as we go
420 // character name -> {kind:'frame'|'actor', id} it currently occupies, so
421 // writers can address characters by name once they've been framed/placed
422 // (`[rex sit]`, `rex: Woof!`, `[wear bo tophat]`); later assignments
423 // override earlier ones. Populated by directionCue's frame/place handling.
424 const charLocs = new Map();
425 for (const rawLine of src.split('\n')) {
426 const line = rawLine.trim();
427 if (!line || line.startsWith('#')) continue;
428
429 if (/^(\[[^\]]+\]\s*)+$/.test(line)) {
430 for (const m of line.matchAll(/\[([^\]]+)\]/g)) {
431 const inner = m[1];
432 // Tolerance: `[left: some words]` / `[<name>: words]` is a spoken
433 // line wearing brackets by mistake (common in LLM-authored
434 // screenplays) — unwrap it instead of treating it as an unknown
435 // direction, but only when the label actually resolves.
436 const sp = matchSpeaker(inner, frameIds, actorIds, charLocs);
437 if (sp) {
438 const cue = { type: 'speak', text: sp.text, concurrent: null, source: line };
439 if (sp.frame) cue.frame = sp.frame;
440 if (sp.actor) cue.actor = sp.actor;
441 cues.push(cue);
442 } else {
443 cues.push(directionCue(inner, line, frameIds, actorIds, charLocs));
444 }
445 }
446 continue;
447 }
448
449 let text = line;
450 let frame = null;
451 let actor = null;
452 const sp = matchSpeaker(text, frameIds, actorIds, charLocs);
453 if (sp) { frame = sp.frame || null; actor = sp.actor || null; text = sp.text; }
454
455 let concurrent = null;
456 const inline = text.match(/^\(([^)]+)\)\s*(.*)$/);
457 if (inline) {
458 concurrent = directionCue(inline[1], line, frameIds, actorIds, charLocs);
459 text = inline[2];
460 if (frame) concurrent.frame = frame;
461 if (actor) concurrent.actor = actor;
462 }
463 const cue = { type: 'speak', text, concurrent, source: line };
464 if (frame) cue.frame = frame;
465 if (actor) cue.actor = actor;
466 cues.push(cue);
467 }
468 return cues;
469}
470
471function directionCue(body, source, frameIds, actorIds, charLocs) {
472 frameIds = frameIds || new Set(['main']);
473 actorIds = actorIds || new Set();
474 charLocs = charLocs || new Map();
475 const parts = body.trim().split(/\s+/);
476 const head = parts[0].toLowerCase();
477
478 if (head === 'layout') {
479 const preset = (parts[1] || 'single').toLowerCase();
480 const ids = LAYOUT_FRAMES[preset] || ['main'];
481 frameIds.clear();
482 for (const id of ids) frameIds.add(id);
483 return { type: 'layout', preset, source };
484 }
485
486 if (head === 'frame') {
487 const id = parts[1];
488 const restStr = parts.slice(2).join(' ');
489 if (/^clear\b/i.test(restStr)) {
490 frameIds.delete(id);
491 return { type: 'frame-clear', id, source };
492 }
493 const kv = parseKV(restStr);
494 const cue = { type: 'frame', id, source };
495 if (kv.slot) cue.slot = kv.slot;
496 if (kv.bg) cue.bg = kv.bg;
497 if (kv.character) cue.character = kv.character;
498 if (kv.view) cue.view = kv.view;
499 if (kv.facing !== undefined) cue.facing = parseInt(kv.facing, 10);
500 if (kv.rect) cue.rect = kv.rect.split(',').map(Number);
501 frameIds.add(id);
502 if (kv.character) charLocs.set(kv.character, { kind: 'frame', id });
503 return cue;
504 }
505
506 if (head === 'scene') return { type: 'scene', bg: parts.slice(1).join(' '), source };
507
508 if (head === 'show') {
509 const kv = parseKV(parts.slice(1).join(' '));
510 let kind = 'text', value = '';
511 for (const k of ['text', 'image', 'video']) {
512 if (kv[k] !== undefined) { kind = k; value = kv[k]; break; }
513 }
514 const cue = { type: 'content', kind, value, source };
515 if (kv.fit) cue.fit = kv.fit;
516 return cue;
517 }
518
519 if (head === 'lower-third') {
520 // stable id so a new lower-third replaces the current one instead of
521 // stacking; `[lower-third clear]` removes it.
522 if ((parts[1] || '').toLowerCase() === 'clear' && !body.includes('"')) {
523 return { type: 'overlay-clear', id: 'lower-third', source };
524 }
525 const strs = [...body.matchAll(/"([^"]*)"/g)].map((m) => m[1]);
526 const cue = {
527 type: 'overlay', template: 'lower-third', id: 'lower-third',
528 slots: { title: strs[0] || '', subtitle: strs[1] || '' }, source,
529 };
530 // Optional trailing `hold:<ms>` key: default 6000 (auto-dismiss after
531 // 6s) when omitted; `hold:0` means persist (omit hold entirely, same as
532 // the old un-timed behavior); any other value is that hold in ms.
533 const holdMatch = body.match(/\bhold:(\d+)\b/i);
534 const holdMs = holdMatch ? parseInt(holdMatch[1], 10) : 6000;
535 if (holdMs > 0) cue.hold = holdMs;
536 return cue;
537 }
538
539 if (head === 'walk' || head === 'enter' || head === 'exit') {
540 // [walk to 70] [walk to center] [enter from left] [exit right]
541 const arg = parts[parts.length - 1].toLowerCase();
542 // walk's left/right are on-stage position words (25/75); enter/exit's
543 // left/right stay off-stage sides — handled separately below.
544 let x = head === 'walk' ? parsePosWord(arg) : parseFloat(arg);
545 if (head === 'enter') x = 50;
546 else if (head === 'exit' && isNaN(x)) x = arg === 'left' ? -15 : 115;
547 const from = parts.includes('from') ? (parts[parts.indexOf('from') + 1] || '') : null;
548 return { type: 'walk', x, from, jump: head === 'enter' ? from : null, source };
549 }
550 if (head === 'wait') return { type: 'wait', seconds: parseFloat(parts[1]) || 1, source };
551 if (head === 'view' || head === 'zoom') {
552 return { type: 'view', mode: (parts[1] || 'body').toLowerCase(), source };
553 }
554 if (head === 'look') return { type: 'look', dir: parts[1] || 'front', source };
555 if (head === 'emote') return { type: 'action', name: parts[1] || 'neutral', source };
556 if (head === 'engine' || head === 'voice' || head === 'rate') {
557 return { type: 'setting', key: head, value: parts.slice(1).join(' '), source };
558 }
559 if (head === 'captions') {
560 const arg = (parts[1] || 'on').toLowerCase();
561 return { type: 'captions', on: arg !== 'off', source };
562 }
563 if (head === 'iris' || head === 'fade') {
564 const ms = parseInt(parts[2], 10);
565 return { type: 'transition', name: head, dir: (parts[1] || 'out').toLowerCase(), ms: isNaN(ms) ? 700 : ms, source };
566 }
567
568 if (head === 'place') {
569 // [place <what> at <x> [id:<id>] [scale:<n>] [behind]] — <x> accepts a
570 // percent number or left|center|middle|right (25/50/75).
571 const rest = parts.slice(1);
572 const what = rest[0];
573 const atIdx = rest.indexOf('at');
574 const x = atIdx >= 0 ? parsePosWord(rest[atIdx + 1]) : NaN;
575 const restStr = parts.slice(1).join(' ');
576 const kv = parseKV(restStr);
577 const id = kv.id || what;
578 const cue = { type: 'place', id, what, x, source };
579 if (kv.scale !== undefined) cue.scale = parseFloat(kv.scale);
580 if (/\bbehind\b/.test(restStr)) cue.behind = true;
581 actorIds.add(id);
582 // `what` names a character iff it's a folder in characters/ — track
583 // where it now lives so later `[<what> ...]`/`what: text` addresses it.
584 if (fs.existsSync(path.join(CHARACTERS, what))) charLocs.set(what, { kind: 'actor', id });
585 return cue;
586 }
587
588 if (head === 'remove') {
589 const id = parts[1];
590 actorIds.delete(id);
591 return { type: 'remove', id, source };
592 }
593
594 if (head === 'wear') {
595 // [wear <target> <prop> [scale:<n>] [anchor:<name>]] — target resolves
596 // frame id -> actor id -> character name (see resolveTarget).
597 const target = resolveTarget(parts[1], frameIds, actorIds, charLocs);
598 const prop = parts[2];
599 const kv = parseKV(parts.slice(3).join(' '));
600 const cue = { type: 'wear', target, prop, source };
601 if (kv.scale !== undefined) cue.scale = parseFloat(kv.scale);
602 if (kv.anchor !== undefined) cue.anchor = kv.anchor;
603 return cue;
604 }
605
606 if (head === 'unwear') {
607 const target = resolveTarget(parts[1], frameIds, actorIds, charLocs);
608 const cue = { type: 'unwear', target, source };
609 if (parts[2]) cue.anchor = parts[2];
610 return cue;
611 }
612
613 if (head === 'music') {
614 const arg = (parts[1] || '').toLowerCase();
615 if (arg === 'off') return { type: 'music', off: true, source };
616 return { type: 'music', id: parts[1], source };
617 }
618
619 if (head === 'sfx') return { type: 'sfx', id: parts[1], source };
620
621 // Prop-actor directions — only meaningful when the head token that routed
622 // here was an actor id (see the actorIds fallback below), e.g.
623 // [hoop1 move 40] recurses into directionCue("move 40", ...).
624 if (head === 'move') return { type: 'move', x: parsePosWord(parts[1]), source };
625 if (head === 'scale') return { type: 'scale', value: parseFloat(parts[1]), source };
626 if (head === 'spin') return { type: 'spin', source };
627 if (head === 'bounce') return { type: 'bounce', source };
628
629 // [<id> <direction...>] — first token is a known frame id: apply the rest
630 // of the direction to that frame.
631 if (frameIds.has(head) && parts.length > 1) {
632 const sub = directionCue(parts.slice(1).join(' '), source, frameIds, actorIds, charLocs);
633 sub.frame = head;
634 return sub;
635 }
636
637 // Same, for a placed actor id (checked after frame ids, per spec).
638 if (actorIds.has(head) && parts.length > 1) {
639 const sub = directionCue(parts.slice(1).join(' '), source, frameIds, actorIds, charLocs);
640 sub.actor = head;
641 return sub;
642 }
643
644 // Same, for a character NAME that's been framed/placed (checked after
645 // frame ids and actor ids, per spec) — e.g. [rex sit] once rex occupies a
646 // frame or actor slot; resolves to whichever one it currently occupies.
647 {
648 const loc = charLocs.get(head);
649 if (loc && parts.length > 1) {
650 const sub = directionCue(parts.slice(1).join(' '), source, frameIds, actorIds, charLocs);
651 if (loc.kind === 'frame') sub.frame = loc.id; else sub.actor = loc.id;
652 return sub;
653 }
654 }
655
656 // [clear] — wipe placed actors/worn props/content tiles/overlays across
657 // every frame, on demand mid-script (same cleanup `script-start` runs
658 // automatically). Music, captions on/off, backgrounds, and frame layout
659 // are untouched.
660 if (head === 'clear') return { type: 'clear', source };
661
662 return { type: 'action', name: head, source };
663}
664
665let scriptToken = 0;
666
667// Per-character declared voice, e.g. characters/ava/manifest.json's
668// `"voice": {"engine":"say","voice":"Samantha"}`. Cached by character name
669// so screenplays with many lines don't re-read manifests per cue.
670const characterVoiceCache = {};
671function characterVoice(name) {
672 if (!name) return null;
673 if (Object.prototype.hasOwnProperty.call(characterVoiceCache, name)) return characterVoiceCache[name];
674 let voice = null;
675 try {
676 const manifest = JSON.parse(fs.readFileSync(path.join(CHARACTERS, name, 'manifest.json'), 'utf8'));
677 if (manifest && manifest.voice) voice = manifest.voice;
678 } catch { /* missing/unreadable manifest; fall back to default voice */ }
679 characterVoiceCache[name] = voice;
680 return voice;
681}
682
683// Single ordered pass over a parsed cue list that replays the same
684// settings/frameChar/override-flag state the (formerly inline) speak handler
685// used, and attaches the fully-resolved TTS params to every speak cue as
686// `cue.speech`. Pulling this out lets runScript render speech ahead of the
687// playback loop instead of only when a line is about to play.
688function resolveSpeech(cues, mainCharacter) {
689 const settings = { engine: 'say', voice: '', rate: 0 };
690 let engineOverridden = false;
691 let voiceOverridden = false;
692 // frame id -> character name, so each spoken line can resolve the voice of
693 // whoever is actually in that frame. Seeded with the client's notion of
694 // who's in the main frame, so solo screenplays pick up that character's
695 // declared voice too.
696 const frameChar = {};
697 if (mainCharacter) frameChar.main = mainCharacter;
698 // actor id -> character name, for actor-targeted speak cues (mirrors
699 // frameChar but seeded from [place] cues instead of [frame]/[character]).
700 const actorChar = {};
701
702 for (const cue of cues) {
703 if (cue.type === 'setting') {
704 settings[cue.key] = cue.value;
705 if (cue.key === 'engine') engineOverridden = true;
706 if (cue.key === 'voice') voiceOverridden = true;
707 continue;
708 }
709 if (cue.type === 'frame' && cue.character) frameChar[cue.id] = cue.character;
710 if (cue.type === 'character') frameChar[cue.frame || 'main'] = cue.name;
711 if (cue.type === 'place') {
712 // `what` names a character iff it's a folder in characters/; a prop
713 // actor placed over a stale id clears any earlier character mapping.
714 if (fs.existsSync(path.join(CHARACTERS, cue.what))) actorChar[cue.id] = cue.what;
715 else delete actorChar[cue.id];
716 }
717 if (cue.type === 'remove') delete actorChar[cue.id];
718
719 if (cue.type === 'speak') {
720 // Resolution: an explicit [engine]/[voice] direction always wins;
721 // otherwise use the declared voice of the character in this line's
722 // actor (if actor-targeted) or frame; otherwise fall back to the
723 // (default) settings.
724 const charName = cue.actor ? actorChar[cue.actor] : frameChar[cue.frame || 'main'];
725 const cv = characterVoice(charName);
726 const engine = engineOverridden ? settings.engine : (cv && cv.engine) || settings.engine;
727 const voice = voiceOverridden ? settings.voice : (cv && cv.voice !== undefined ? cv.voice : settings.voice);
728 cue.speech = { text: cue.text, engine, voice, rate: settings.rate };
729 }
730 }
731}
732
733const RENDER_WINDOW = 4;
734
735async function runScript(src, mainCharacter) {
736 const token = ++scriptToken;
737 const cues = parseScript(src);
738 resolveSpeech(cues, mainCharacter);
739 const speakCues = cues.filter((c) => c.type === 'speak');
740
741 // Bounded render-ahead: kick off prepareSpeech for the k-th speak cue at
742 // most once, and stash the promise on the cue itself. Calling this
743 // repeatedly (e.g. re-priming a window that's already full) is a no-op.
744 function ensureRender(k) {
745 if (k < 0 || k >= speakCues.length) return;
746 const c = speakCues[k];
747 if (!c._renderP) {
748 c._renderP = prepareSpeech(c.speech);
749 c._renderP.catch(() => {}); // avoid unhandled-rejection noise; real errors surface at the await below
750 }
751 }
752
753 for (let k = 0; k < RENDER_WINDOW; k++) ensureRender(k);
754 let speakIndex = 0;
755
756 if (speakCues.length > 0) {
757 try {
758 await speakCues[0]._renderP;
759 } catch (e) {
760 // ignore here; the playback loop's per-line error handling will surface
761 // this once it reaches line 1
762 }
763 if (token !== scriptToken) return; // replaced while pre-rendering; don't start
764 }
765
766 broadcast({ type: 'script-start', lines: cues.map((c) => c.source) });
767
768 for (let i = 0; i < cues.length; i++) {
769 if (token !== scriptToken) break; // stopped or replaced
770 const cue = cues[i];
771 broadcast({ type: 'script-line', index: i });
772
773 if (cue.type === 'setting' || cue.type === 'wait') {
774 if (cue.type === 'wait') await sleep(cue.seconds * 1000);
775 continue;
776 }
777
778 if (cue.type === 'speak') {
779 const k = speakIndex++;
780 ensureRender(k); // in case it wasn't primed (e.g. window > remaining lines)
781 let prepared;
782 try {
783 prepared = await cue._renderP;
784 } catch (e) {
785 broadcast({ type: 'error', message: e.message });
786 ensureRender(k + RENDER_WINDOW);
787 continue;
788 }
789 if (token !== scriptToken) break;
790 if (cue.concurrent) broadcast(cue.concurrent);
791 const speakCue = { type: 'speak', ...prepared };
792 if (cue.frame) speakCue.frame = cue.frame;
793 if (cue.actor) speakCue.actor = cue.actor;
794 broadcast(speakCue);
795 await sleep(prepared.duration * 1000 + 300);
796 ensureRender(k + RENDER_WINDOW);
797 continue;
798 }
799
800 broadcast(cue);
801 if (cue.type === 'action') await sleep((ACTION_SECONDS[cue.name] || 1) * 1000);
802 else if (cue.type === 'walk') await sleep(1200);
803 else if (cue.type === 'transition') await sleep(cue.ms + 100);
804 else await sleep(400);
805 }
806
807 if (token === scriptToken) broadcast({ type: 'script-end' });
808}
809
810// ----------------------------------------------------------------- voices
811
812async function listVoices(engine) {
813 if (engine === 'espeak') {
814 const out = await run('espeak', ['--voices']);
815 return out.split('\n').slice(1).map((l) => l.trim().split(/\s+/)[3]).filter(Boolean);
816 }
817 const out = await run('say', ['-v', '?']);
818 return out.split('\n')
819 .map((l) => l.match(/^(.+?)\s{2,}([a-z]{2}[_-]\w+)/))
820 .filter((m) => m && m[2].startsWith('en'))
821 .map((m) => m[1].trim());
822}
823
824// ------------------------------------------------------ conversation mode
825//
826// CONVERSATION MODE: push-to-talk speech -> OpenAI transcription -> an
827// OpenAI chat completion replying in character -> spoken back through the
828// existing prepareSpeech/broadcast pipeline. Key loading mirrors the
829// integrations/dbbasic/puppet-stage director.py convention (OPENAI_API_KEY
830// env, else an OPENAI_KEY_FILE .env-style file) so both surfaces agree on
831// how a deployment supplies the key. The key lives only in this
832// module-level variable — never logged, never echoed back in a response.
833
834let openaiKey = '';
835(function loadOpenAIKey() {
836 const env = (process.env.OPENAI_API_KEY || '').trim();
837 if (env) { openaiKey = env; return; }
838 const file = (process.env.OPENAI_KEY_FILE || '').trim();
839 if (!file) return;
840 try {
841 const text = fs.readFileSync(file, 'utf8');
842 for (const line of text.split('\n')) {
843 const l = line.trim();
844 if (l.startsWith('OPENAI_API_KEY=')) {
845 openaiKey = l.slice('OPENAI_API_KEY='.length).trim().replace(/^["']|["']$/g, '');
846 break;
847 }
848 }
849 } catch { /* missing/unreadable key file; converse endpoints report not-configured */ }
850})();
851
852const NO_KEY_ERROR = 'no OpenAI key configured (set OPENAI_API_KEY or OPENAI_KEY_FILE)';
853const TRANSCRIBE_MODEL = process.env.TRANSCRIBE_MODEL || 'gpt-4o-mini-transcribe';
854const CONVERSE_MODEL = process.env.CONVERSE_MODEL || 'gpt-5.4-mini';
855
856// Content-Type -> filename extension, so the multipart upload's filename
857// matches the container the browser actually recorded (webm/ogg/mp4/wav).
858function audioExtFromContentType(ct) {
859 ct = (ct || '').toLowerCase();
860 if (ct.includes('ogg')) return 'ogg';
861 if (ct.includes('mp4') || ct.includes('m4a') || ct.includes('aac')) return 'mp4';
862 if (ct.includes('wav')) return 'wav';
863 return 'webm';
864}
865
866// Zero-dependency multipart/form-data POST to OpenAI's transcription
867// endpoint: two fields (model, file) built by hand via Buffer.concat.
868async function openaiTranscribe(bytes, contentType) {
869 if (!openaiKey) throw new Error(NO_KEY_ERROR);
870 const boundary = `----puppetstage${crypto.randomBytes(16).toString('hex')}`;
871 const ext = audioExtFromContentType(contentType);
872 const head = Buffer.from(
873 `--${boundary}\r\nContent-Disposition: form-data; name="model"\r\n\r\n${TRANSCRIBE_MODEL}\r\n` +
874 `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="audio.${ext}"\r\n` +
875 `Content-Type: ${contentType || 'application/octet-stream'}\r\n\r\n`);
876 const tail = Buffer.from(`\r\n--${boundary}--\r\n`);
877 const body = Buffer.concat([head, bytes, tail]);
878
879 const controller = new AbortController();
880 const timer = setTimeout(() => controller.abort(), 60000);
881 let resp, text;
882 try {
883 resp = await fetch('https://api.openai.com/v1/audio/transcriptions', {
884 method: 'POST',
885 headers: { Authorization: `Bearer ${openaiKey}`, 'Content-Type': `multipart/form-data; boundary=${boundary}` },
886 body,
887 signal: controller.signal,
888 });
889 text = await resp.text();
890 } finally {
891 clearTimeout(timer);
892 }
893 if (!resp.ok) throw new Error(`OpenAI ${resp.status}: ${text.slice(0, 300)}`);
894 let parsed;
895 try { parsed = JSON.parse(text); } catch { throw new Error('unexpected OpenAI response (transcribe)'); }
896 return parsed.text || '';
897}
898
899// Zero-dependency chat completion call for the in-character reply.
900async function openaiChat(messages) {
901 if (!openaiKey) throw new Error(NO_KEY_ERROR);
902 const controller = new AbortController();
903 const timer = setTimeout(() => controller.abort(), 60000);
904 let resp, text;
905 try {
906 resp = await fetch('https://api.openai.com/v1/chat/completions', {
907 method: 'POST',
908 headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${openaiKey}` },
909 body: JSON.stringify({ model: CONVERSE_MODEL, messages, temperature: 0.8 }),
910 signal: controller.signal,
911 });
912 text = await resp.text();
913 } finally {
914 clearTimeout(timer);
915 }
916 if (!resp.ok) throw new Error(`OpenAI ${resp.status}: ${text.slice(0, 300)}`);
917 let parsed;
918 try { parsed = JSON.parse(text); } catch { throw new Error('unexpected OpenAI response (chat)'); }
919 const content = parsed.choices && parsed.choices[0] && parsed.choices[0].message && parsed.choices[0].message.content;
920 if (!content) throw new Error('unexpected OpenAI response shape (chat)');
921 return content.trim();
922}
923
924// Per-session rolling chat history: session id -> {messages:[{role,content}],
925// lastActive}. Capped at the last 20 messages; idle sessions (30 min) are
926// pruned opportunistically whenever /api/converse is next called.
927const CONVERSE_HISTORY_CAP = 20;
928const CONVERSE_IDLE_MS = 30 * 60 * 1000;
929const conversations = new Map();
930
931function conversationFor(id) {
932 const now = Date.now();
933 for (const [k, v] of conversations) {
934 if (now - v.lastActive > CONVERSE_IDLE_MS) conversations.delete(k);
935 }
936 let s = conversations.get(id);
937 if (!s) { s = { messages: [], lastActive: now }; conversations.set(id, s); }
938 s.lastActive = now;
939 return s;
940}
941
942function pushCapped(list, entry) {
943 list.push(entry);
944 if (list.length > CONVERSE_HISTORY_CAP) list.splice(0, list.length - CONVERSE_HISTORY_CAP);
945}
946
947// Builds the system prompt for a character: manifest.persona if the
948// manifest declares one (another agent is adding that field in parallel —
949// this only ever READS it), else a generic fallback. Also returns the
950// character's action names so the reply can be parsed against them.
951function buildConverseContext(character) {
952 let persona = null;
953 let actionNames = [];
954 try {
955 const manifest = JSON.parse(fs.readFileSync(path.join(CHARACTERS, character, 'manifest.json'), 'utf8'));
956 if (manifest && manifest.persona) persona = manifest.persona;
957 actionNames = Object.keys((manifest && manifest.actions) || {});
958 } catch { /* missing/unreadable manifest; fall back below */ }
959 const displayName = character ? character.charAt(0).toUpperCase() + character.slice(1) : 'the puppet';
960 const base = persona || `You are ${displayName}, a puppet character on a small stage.`;
961 const actionsList = actionNames.length ? actionNames.join(', ') : 'none';
962 const prompt = `${base}\n` +
963 'You are a live animated puppet on a real stage, and you have a body: you can ' +
964 `physically perform these actions right now: ${actionsList}. ` +
965 'To perform one, write its name in parentheses inline with your words, e.g. "(nod) Absolutely." ' +
966 'Use one when it fits what you are saying — agree with a (nod), refuse with a (shake), ' +
967 'delight with a (happy). If someone asks whether you can do something on that list, ' +
968 'say yes and DO it in the same reply. At most two actions per reply; only names from ' +
969 'the list; nothing else in parentheses.\n' +
970 'Reply in character, conversationally, 1-3 short sentences (spoken aloud). No emoji, no markdown.';
971 return { prompt, actionNames };
972}
973
974// Extracts (action) tokens that match the character's known action names out
975// of a reply, in order, and strips them (and the whitespace they leave
976// behind) from the spoken text. A parenthetical that ISN'T a known action
977// name is left in place rather than silently eaten — the persona prompt
978// asks the model not to use parens for anything else, but a stray one
979// shouldn't corrupt the spoken line.
980function extractActions(text, actionNames) {
981 const known = new Set(actionNames);
982 const actions = [];
983 const stripped = text.replace(/\(([a-zA-Z][\w-]*)\)/g, (whole, name) => {
984 if (known.has(name)) { actions.push(name); return ' '; }
985 return whole;
986 });
987 return { text: stripped.replace(/\s+/g, ' ').trim(), actions };
988}
989
990// ----------------------------------------------------------------- static
991
992const MIME = {
993 '.html': 'text/html', '.js': 'text/javascript', '.css': 'text/css',
994 '.json': 'application/json', '.svg': 'image/svg+xml', '.wav': 'audio/wav',
995 '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
996 '.webp': 'image/webp', '.gif': 'image/gif',
997 '.mp4': 'video/mp4', '.webm': 'video/webm',
998};
999
1000function serveFile(res, filePath) {
1001 fs.readFile(filePath, (err, data) => {
1002 if (err) { res.writeHead(404); res.end('not found'); return; }
1003 res.writeHead(200, { 'Content-Type': MIME[path.extname(filePath)] || 'application/octet-stream' });
1004 res.end(data);
1005 });
1006}
1007
1008// ----------------------------------------------------------------- server
1009
1010const server = http.createServer(async (req, res) => {
1011 const url = new URL(req.url, `http://${req.headers.host}`);
1012 const p = url.pathname;
1013
1014 try {
1015 if (p === '/api/events') {
1016 res.writeHead(200, {
1017 'Content-Type': 'text/event-stream',
1018 'Cache-Control': 'no-cache',
1019 Connection: 'keep-alive',
1020 });
1021 res.write(': connected\n\n');
1022 clients.add(res);
1023 req.on('close', () => clients.delete(res));
1024 return;
1025 }
1026
1027 if (p === '/api/say' && req.method === 'POST') {
1028 const body = await readBody(req);
1029 if (!body.text) return json(res, 400, { error: 'text required' });
1030 const prepared = await prepareSpeech(body);
1031 const cue = { type: 'speak', ...prepared };
1032 if (body.frame) cue.frame = body.frame;
1033 if (body.actor) cue.actor = body.actor;
1034 broadcast(cue);
1035 return json(res, 200, prepared);
1036 }
1037
1038 // external-TTS entry point: POST raw audio bytes (any ffmpeg-readable
1039 // format), optional ?text= transcript improves lip-sync accuracy
1040 if (p === '/api/speak-audio' && req.method === 'POST') {
1041 const bytes = await readRaw(req);
1042 if (!bytes.length) return json(res, 400, { error: 'audio body required' });
1043 const prepared = await prepareExternalAudio(bytes, url.searchParams.get('text') || '');
1044 broadcast({ type: 'speak', ...prepared });
1045 return json(res, 200, prepared);
1046 }
1047
1048 // CONVERSATION MODE: readiness probe — the UI hides the Converse panel
1049 // when this reports not-ready (no key configured).
1050 if (p === '/api/converse' && req.method === 'GET') {
1051 return json(res, 200, { ready: !!openaiKey });
1052 }
1053
1054 // raw audio bytes body (webm/ogg/mp4/wav — whatever the browser's
1055 // MediaRecorder produced; Content-Type says which), forwarded to
1056 // OpenAI's transcription endpoint.
1057 if (p === '/api/transcribe' && req.method === 'POST') {
1058 if (!openaiKey) return json(res, 400, { error: NO_KEY_ERROR });
1059 const bytes = await readRaw(req);
1060 if (!bytes.length) return json(res, 400, { error: 'audio body required' });
1061 try {
1062 const text = await openaiTranscribe(bytes, req.headers['content-type']);
1063 return json(res, 200, { text });
1064 } catch (e) {
1065 return json(res, 502, { error: e.message });
1066 }
1067 }
1068
1069 // {text, character, session, frame?} -> an in-character OpenAI reply,
1070 // spoken on stage. Broadcasts (in order) a user-said caption cue, an
1071 // optional concurrent action cue (the reply's first inline action, like
1072 // a screenplay's (action) prefix), the speak cue itself, then any
1073 // further inline actions as trailing action cues.
1074 if (p === '/api/converse' && req.method === 'POST') {
1075 if (!openaiKey) return json(res, 400, { error: NO_KEY_ERROR });
1076 const body = await readBody(req);
1077 const userText = (body.text || '').trim();
1078 const character = body.character;
1079 if (!userText) return json(res, 400, { error: 'text required' });
1080 if (!character) return json(res, 400, { error: 'character required' });
1081
1082 const session = conversationFor(body.session || 'default');
1083 const { prompt, actionNames } = buildConverseContext(character);
1084 pushCapped(session.messages, { role: 'user', content: userText });
1085
1086 let reply;
1087 try {
1088 reply = await openaiChat([{ role: 'system', content: prompt }, ...session.messages]);
1089 } catch (e) {
1090 session.messages.pop(); // don't keep a user turn that never got a reply
1091 return json(res, 502, { error: e.message });
1092 }
1093 pushCapped(session.messages, { role: 'assistant', content: reply });
1094
1095 const { text: spokenText, actions } = extractActions(reply, actionNames);
1096
1097 broadcast({ type: 'user-said', text: userText });
1098 if (actions[0]) {
1099 const actionCue = { type: 'action', name: actions[0] };
1100 if (body.frame) actionCue.frame = body.frame;
1101 broadcast(actionCue);
1102 }
1103
1104 let prepared;
1105 try {
1106 const voice = characterVoice(character) || {};
1107 prepared = await prepareSpeech({ text: spokenText || reply, engine: voice.engine, voice: voice.voice, rate: voice.rate });
1108 } catch (e) {
1109 return json(res, 502, { error: e.message });
1110 }
1111 const speakCue = { type: 'speak', ...prepared };
1112 if (body.frame) speakCue.frame = body.frame;
1113 broadcast(speakCue);
1114
1115 for (const name of actions.slice(1)) {
1116 const cue = { type: 'action', name };
1117 if (body.frame) cue.frame = body.frame;
1118 broadcast(cue);
1119 }
1120
1121 return json(res, 200, { reply: spokenText, actions, performed: true });
1122 }
1123
1124 if (p === '/api/cue' && req.method === 'POST') {
1125 const body = await readBody(req);
1126 if (!body.type) return json(res, 400, { error: 'type required' });
1127 broadcast(body);
1128 return json(res, 200, { ok: true });
1129 }
1130
1131 if (p === '/api/script' && req.method === 'POST') {
1132 const body = await readBody(req);
1133 if (!body.script) return json(res, 400, { error: 'script required' });
1134 runScript(body.script, body.mainCharacter); // runs in background
1135 return json(res, 200, { ok: true });
1136 }
1137
1138 if (p === '/api/script/stop' && req.method === 'POST') {
1139 scriptToken++;
1140 broadcast({ type: 'script-end' });
1141 return json(res, 200, { ok: true });
1142 }
1143
1144 if (p === '/api/voices') {
1145 return json(res, 200, { voices: await listVoices(url.searchParams.get('engine')) });
1146 }
1147
1148 if (p === '/api/characters') {
1149 const names = fs.readdirSync(CHARACTERS, { withFileTypes: true })
1150 .filter((d) => d.isDirectory()).map((d) => d.name);
1151 return json(res, 200, { characters: names });
1152 }
1153
1154 // GET /api/examples → basenames (no .txt) of examples/*.txt, sorted.
1155 if (p === '/api/examples') {
1156 let names = [];
1157 try {
1158 names = fs.readdirSync(EXAMPLES)
1159 .filter((f) => f.endsWith('.txt'))
1160 .map((f) => f.slice(0, -'.txt'.length))
1161 .sort();
1162 } catch { /* no examples dir yet */ }
1163 return json(res, 200, { examples: names });
1164 }
1165
1166 // GET /api/examples/<name> → {name, script} of examples/<name>.txt.
1167 if (p.startsWith('/api/examples/')) {
1168 const rel = path.normalize(p.slice('/api/examples/'.length));
1169 if (rel.startsWith('..')) { res.writeHead(403); return res.end(); }
1170 const file = path.join(EXAMPLES, rel + '.txt');
1171 if (!fs.existsSync(file)) return json(res, 404, { error: 'example not found' });
1172 return json(res, 200, { name: rel, script: fs.readFileSync(file, 'utf8') });
1173 }
1174
1175 // GET /api/assets → {backgrounds, props, overlays, music, sfx}: ids
1176 // (filenames minus extension) available in assets/<kind>/, deduped
1177 // across the relevant extension set (svg/raster for most kinds, .json
1178 // for the audio kinds).
1179 if (p === '/api/assets') {
1180 const listKind = (kind) => {
1181 let files = [];
1182 try { files = fs.readdirSync(path.join(ASSETS, kind)); } catch { return []; }
1183 const exts = AUDIO_KINDS.includes(kind) ? AUDIO_EXTS : ASSET_EXTS;
1184 const ids = new Set();
1185 for (const f of files) {
1186 const ext = path.extname(f);
1187 if (exts.includes(ext)) ids.add(f.slice(0, -ext.length));
1188 }
1189 return [...ids].sort();
1190 };
1191 return json(res, 200, {
1192 backgrounds: listKind('backgrounds'),
1193 props: listKind('props'),
1194 overlays: listKind('overlays'),
1195 music: listKind('music'),
1196 sfx: listKind('sfx'),
1197 });
1198 }
1199
1200 // 1x1 gif served after a delay — lets headless screenshots wait for async boot
1201 if (p === '/slowpx') {
1202 lastSlowpx = Date.now(); // marks navigation: the page requests this immediately on load
1203 const gif = Buffer.from('R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7', 'base64');
1204 setTimeout(() => { res.writeHead(200, { 'Content-Type': 'image/gif' }); res.end(gif); },
1205 parseInt(url.searchParams.get('d'), 10) || 2500);
1206 return;
1207 }
1208
1209 // GET /api/last-shot → {t}: Date.now() (server clock) at the start of the
1210 // most recent /slowpx request, i.e. the navigation timestamp of the most
1211 // recent headless screenshot page load. Used by the screenshot harness to
1212 // anchor action cues to actual page navigation instead of a fixed sleep
1213 // after spawning the browser process (browser startup jitter otherwise
1214 // made cue timing nondeterministic). 0 if no /slowpx request has landed yet.
1215 if (p === '/api/last-shot') {
1216 return json(res, 200, { t: lastSlowpx || 0 });
1217 }
1218
1219 // GET /api/asset/<kind>/<id> → the resolved file (first existing
1220 // extension among svg/png/jpg/jpeg/webp/gif), so the client can learn
1221 // which format exists without guessing.
1222 if (p.startsWith('/api/asset/')) {
1223 const [kind, id] = p.slice('/api/asset/'.length).split('/');
1224 const file = kind && id ? resolveAsset(kind, id) : null;
1225 if (!file) return json(res, 404, { error: 'asset not found' });
1226 return serveFile(res, file);
1227 }
1228
1229 // static routes
1230 if (p.startsWith('/audio/')) return serveFile(res, path.join(CACHE, path.basename(p)));
1231 if (p.startsWith('/characters/')) {
1232 const rel = path.normalize(p.slice('/characters/'.length));
1233 if (rel.startsWith('..')) { res.writeHead(403); return res.end(); }
1234 return serveFile(res, path.join(CHARACTERS, rel));
1235 }
1236 if (p.startsWith('/assets/')) {
1237 const rel = path.normalize(p.slice('/assets/'.length));
1238 if (rel.startsWith('..')) { res.writeHead(403); return res.end(); }
1239 return serveFile(res, path.join(ASSETS, rel));
1240 }
1241 const file = p === '/' ? 'index.html' : path.normalize(p).replace(/^\/+/, '');
1242 if (file.startsWith('..')) { res.writeHead(403); return res.end(); }
1243 return serveFile(res, path.join(PUBLIC, file));
1244 } catch (e) {
1245 return json(res, 500, { error: e.message });
1246 }
1247});
1248
1249server.listen(PORT, () => {
1250 console.log(`Puppet stage: http://localhost:${PORT}`);
1251});