Puppet Stage: a browser puppet that lip-syncs to local text-to-speech and takes stage directions, live or from a screenplay.

0 downloads

dbbasic-face/public/stage.js
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-face.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-face.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-face.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

Dan Quellhorst Converse: characters know they have bodies 52ca998 4 months, 1 week ago History
stage.js · 82.2 KB · 2260 lines
1'use strict';
2
3// --------------------------------------------------------------- helpers
4
5function looksLikeUrl(v) {
6 return /^([a-z]+:)?\/\//i.test(v) || v.startsWith('/') || /\.[a-z0-9]{2,5}(\?.*)?$/i.test(v);
7}
8
9function assetUrl(kind, id) { return `/api/asset/${kind}/${id}`; }
10
11// Design-height reference for prop actors, matching the same 720-design
12// scaling rule characters use (Puppet.applyScale): a full-height character
13// reads around 600px tall in the 720-tall design, so a prop `scale: 0.3`
14// (against this same reference) lands knee-high.
15const PROP_DESIGN_HEIGHT = 600;
16
17// Named slots resolve to stage-percent rects [x, y, w, h]. `rect:` on a
18// frame cue overrides `slot:` entirely.
19const SLOTS = {
20 full: [0, 0, 100, 100],
21 left: [0, 0, 50, 100],
22 right: [50, 0, 50, 100],
23 top: [0, 0, 100, 50],
24 bottom: [0, 50, 100, 50],
25 'third-l': [0, 0, 33.34, 100],
26 'third-c': [33.33, 0, 33.34, 100],
27 'third-r': [66.66, 0, 33.34, 100],
28 'pip-tr': [66, 4, 30, 30],
29 'pip-tl': [4, 4, 30, 30],
30 'pip-br': [66, 66, 30, 30],
31 'pip-bl': [4, 66, 30, 30],
32};
33
34// Layout presets: macros that clear existing frames and create the named ones.
35const LAYOUT_PRESETS = {
36 single: [{ id: 'main', slot: 'full' }],
37 split: [{ id: 'left', slot: 'left' }, { id: 'right', slot: 'right' }],
38 stack: [{ id: 'top', slot: 'top' }, { id: 'bottom', slot: 'bottom' }],
39 thirds: [
40 { id: 'third-l', slot: 'third-l' },
41 { id: 'third-c', slot: 'third-c' },
42 { id: 'third-r', slot: 'third-r' },
43 ],
44 'pip-tr': [{ id: 'main', slot: 'full' }, { id: 'pip', slot: 'pip-tr' }],
45 'pip-tl': [{ id: 'main', slot: 'full' }, { id: 'pip', slot: 'pip-tl' }],
46 'pip-br': [{ id: 'main', slot: 'full' }, { id: 'pip', slot: 'pip-br' }],
47 'pip-bl': [{ id: 'main', slot: 'full' }, { id: 'pip', slot: 'pip-bl' }],
48};
49
50// ---------------------------------------------------------------- Puppet
51//
52// A Puppet is bound to one frame's DOM subtree
53// (.frame-camera > .frame-pos > .frame-flip) rather than fixed page ids, so
54// several can exist side by side, one per frame.
55
56class Puppet {
57 constructor(frameEl) {
58 this.stage = frameEl; // the frame's own rect (for width/camera math)
59 this.camera = frameEl.querySelector('.frame-camera');
60 this.pos = frameEl.querySelector('.frame-pos');
61 this.flip = frameEl.querySelector('.frame-flip');
62 this.manifest = null;
63 this.characterName = null;
64 this.mouthEls = {};
65 this.currentMouth = null;
66 this.heldAnims = [];
67 this.walkAnim = null;
68 this.bobAnim = null;
69 this.x = 50; // percent of frame width
70 this.facing = 1; // 1 = right, -1 = left
71 this.source = null; // current AudioBufferSourceNode
72 this.audioCtx = null;
73 this.analyser = null;
74 this.lipLoop = null;
75 this.blinkTimer = null;
76 this.view = 'body';
77 this.cam = { tx: 0, ty: 0, s: 1 };
78 this.breathAnim = null;
79 this.gazeTimer = null;
80 this.actorScale = 1; // extra multiplier for actor instances (place cue's scale:)
81 this.wornProps = new Map(); // anchor name -> the <g class="worn"> element pinned there
82 this.ducking = false; // true while this puppet holds a musicEngine.duck() (paired with unduck() in stopSpeaking)
83 }
84
85 async load(name) {
86 this.characterName = name;
87 const base = `/characters/${name}/`;
88 this.manifest = await (await fetch(base + 'manifest.json')).json();
89 const svgText = await (await fetch(base + this.manifest.svg)).text();
90 this.flip.innerHTML = svgText;
91 const svg = this.flip.querySelector('svg');
92 this.svg = svg;
93 svg.style.width = 'auto';
94 svg.style.overflow = 'visible';
95 this.applyScale();
96
97 this.mouthEls = {};
98 for (const [shape, sel] of Object.entries(this.manifest.mouths || {})) {
99 this.mouthEls[shape] = this.flip.querySelector(sel);
100 }
101 this.currentMouth = null; // fresh SVG: force the hide-all pass even if the shape name matches
102 this.setMouth(this.manifest.restMouth || 'X');
103 this.setX(this.x, 0);
104 this.startBlinking();
105 this.startIdle();
106 requestAnimationFrame(() => this.setView(this.view, true));
107 }
108
109 // manifest.height is a DESIGN height authored against a 720px-tall stage;
110 // rendering it at that literal pixel size made a character overflow (or
111 // shrink to nothing in) its frame whenever the frame wasn't 720px tall —
112 // e.g. any browser resize, or a split/thirds layout. Scaling it by the
113 // frame's actual height keeps the character the same PROPORTION of its
114 // frame at any size.
115 applyScale() {
116 if (!this.svg) return;
117 const designH = (this.manifest && this.manifest.height) || 300;
118 const frameH = this.stage.clientHeight || 720;
119 this.svg.style.height = Math.max(60, designH * this.actorScale * (frameH / 720)) + 'px';
120 }
121
122 $(sel) { return this.flip.querySelector(sel); }
123
124 setMouth(shape) {
125 if (shape === this.currentMouth) return;
126 for (const el of Object.values(this.mouthEls)) if (el) el.style.display = 'none';
127 const el = this.mouthEls[shape] || this.mouthEls[this.manifest.restMouth || 'X'];
128 if (el) el.style.display = '';
129 this.currentMouth = shape;
130 }
131
132 // ------------------------------------------------------------- speech
133 //
134 // Playback goes through Web Audio buffers rather than an <audio> element:
135 // speak cues arrive over SSE (outside any click), and Safari blocks
136 // HTMLMediaElement.play() there. An AudioContext, once resumed by any user
137 // gesture, can start buffers at any time.
138
139 ensureCtx() {
140 if (!this.audioCtx) this.audioCtx = new (window.AudioContext || window.webkitAudioContext)();
141 return this.audioCtx;
142 }
143
144 async speak(cue) {
145 this.stopSpeaking();
146 const ctx = this.ensureCtx();
147 if (ctx.state === 'suspended') await ctx.resume().catch(() => {});
148
149 let buffer;
150 try {
151 const bytes = await (await fetch(cue.audio)).arrayBuffer();
152 buffer = await ctx.decodeAudioData(bytes);
153 } catch (e) {
154 toast('audio failed: ' + e.message);
155 return;
156 }
157
158 if (ctx.state !== 'running') {
159 toast('🔇 Click anywhere once to enable sound');
160 await new Promise((r) => window.addEventListener('pointerdown', r, { once: true }));
161 await ctx.resume().catch(() => {});
162 if (ctx.state !== 'running') { toast('could not start audio'); return; }
163 }
164
165 const src = ctx.createBufferSource();
166 src.buffer = buffer;
167 this.analyser = ctx.createAnalyser();
168 this.analyser.fftSize = 512;
169 src.connect(this.analyser);
170 this.analyser.connect(ctx.destination);
171 this.source = src;
172
173 const timeline = cue.timeline;
174 const data = new Uint8Array(this.analyser.fftSize);
175 const startAt = ctx.currentTime;
176 const tick = () => {
177 if (this.source !== src) return;
178 if (timeline && timeline.length) {
179 const t = ctx.currentTime - startAt;
180 let shape = 'X';
181 for (const c of timeline) { if (t >= c.start && t < c.end) { shape = c.value; break; } }
182 this.setMouth(shape);
183 } else {
184 this.analyser.getByteTimeDomainData(data);
185 let sum = 0;
186 for (const v of data) { const d = (v - 128) / 128; sum += d * d; }
187 const rms = Math.sqrt(sum / data.length);
188 this.setMouth(rms > 0.14 ? 'D' : rms > 0.07 ? 'C' : rms > 0.025 ? 'B' : 'X');
189 }
190 this.lipLoop = requestAnimationFrame(tick);
191 };
192
193 await new Promise((resolve) => {
194 src.onended = resolve;
195 src.start();
196 this.ducking = true;
197 if (musicEngine) musicEngine.duck();
198 this.lipLoop = requestAnimationFrame(tick);
199 });
200 if (this.source === src) this.stopSpeaking();
201 }
202
203 stopSpeaking() {
204 if (this.lipLoop) cancelAnimationFrame(this.lipLoop);
205 this.lipLoop = null;
206 if (this.source) { try { this.source.stop(); } catch { /* already stopped */ } this.source = null; }
207 if (this.ducking) { this.ducking = false; if (musicEngine) musicEngine.unduck(); }
208 if (this.manifest) this.setMouth(this.manifest.restMouth || 'X');
209 }
210
211 // ------------------------------------------------------------ actions
212
213 act(name) {
214 const def = (this.manifest.actions || {})[name];
215 if (!def) return Promise.resolve();
216 if (def.reset) {
217 for (const a of this.heldAnims) a.cancel();
218 this.heldAnims = [];
219 return Promise.resolve();
220 }
221 const done = [];
222 for (const track of def.tracks) {
223 const el = this.$(track.target);
224 if (!el) continue;
225 if (track.origin) {
226 el.style.transformBox = 'fill-box';
227 el.style.transformOrigin = track.origin;
228 }
229 const anim = el.animate(track.keyframes, {
230 duration: track.duration || 1000,
231 easing: track.easing || 'ease-in-out',
232 iterations: track.iterations || 1,
233 fill: track.fill || 'none',
234 });
235 if (track.fill === 'forwards') this.heldAnims.push(anim);
236 done.push(anim.finished.catch(() => {}));
237 }
238 return Promise.all(done);
239 }
240
241 // ----------------------------------------------------------- movement
242
243 setX(pct, ms) {
244 const el = this.pos;
245 if (this.walkAnim) { this.walkAnim.cancel(); this.walkAnim = null; }
246 el.style.transition = ms ? `left ${ms}ms linear` : 'none';
247 el.style.left = pct + '%';
248 this.x = pct;
249 }
250
251 face(dir) {
252 this.facing = dir;
253 this.flip.style.transform = dir < 0 ? 'scaleX(-1)' : '';
254 }
255
256 async walkTo(pct, fromSide) {
257 if (fromSide === 'left') this.setX(-15, 0);
258 if (fromSide === 'right') this.setX(115, 0);
259 await new Promise((r) => requestAnimationFrame(r)); // flush the teleport
260 const stageW = this.stage.clientWidth;
261 if (pct >= 0 && pct <= 100) { // on-stage target: keep the whole character inside the frame
262 const charW = this.flip.getBoundingClientRect().width;
263 const halfPct = (charW / stageW) * 100 / 2;
264 pct = halfPct > 50 ? 50 : Math.min(Math.max(pct, halfPct), 100 - halfPct);
265 } // else: deliberate off-stage target (enter/exit) — leave as-is
266 const distPx = Math.abs(pct - this.x) / 100 * stageW;
267 const speed = (this.manifest.walk && this.manifest.walk.speed) || 220;
268 const ms = Math.max(200, distPx / speed * 1000);
269 this.face(pct >= this.x ? 1 : -1);
270
271 let bob = null;
272 if (!this.manifest.walk || this.manifest.walk.bob !== false) {
273 const root = this.$('#puppet') || this.flip.querySelector('svg');
274 bob = root.animate(
275 [{ transform: 'translateY(0)' }, { transform: 'translateY(-7px)' }, { transform: 'translateY(0)' }],
276 { duration: 260, iterations: Math.ceil(ms / 260) });
277 }
278 this.setX(pct, ms);
279 await new Promise((r) => setTimeout(r, ms));
280 if (bob) bob.cancel();
281 if (this.view === 'face') this.setView('face'); // keep the close-up framed
282 }
283
284 lookEls() {
285 const cfg = this.manifest.look;
286 return cfg ? [...this.flip.querySelectorAll(cfg.target)] : [];
287 }
288
289 setGaze(transform, ms) {
290 for (const el of this.lookEls()) {
291 el.style.transition = `transform ${ms}ms ease`;
292 el.style.transform = transform;
293 }
294 }
295
296 look(dir) {
297 const cfg = this.manifest.look;
298 if (!cfg) return;
299 const dx = cfg.dx || 5, dy = cfg.dy || 2;
300 const map = {
301 left: `translate(${-dx}px, 0)`, right: `translate(${dx}px, 0)`,
302 up: `translate(0, ${-dy * 2}px)`, down: `translate(0, ${dy * 2}px)`,
303 front: 'translate(0, 0)',
304 };
305 this.setGaze(map[dir] || map.front, 200);
306 }
307
308 // ------------------------------------------------------------- camera
309
310 /**
311 * 'body' shows the whole frame; 'face' zooms the camera onto the
312 * character's face (manifest views.face.focus, default #head).
313 */
314 setView(mode, instant = false) {
315 this.view = mode === 'face' ? 'face' : 'body';
316 this.camera.style.transition = instant ? 'none' : '';
317 if (this.view === 'body') {
318 this.cam = { tx: 0, ty: 0, s: 1 };
319 this.camera.style.transform = '';
320 return;
321 }
322 const cfg = (this.manifest.views && this.manifest.views.face) || {};
323 const el = this.$(cfg.focus || '#head');
324 if (!el) return;
325 const st = this.stage.getBoundingClientRect();
326 const r = el.getBoundingClientRect();
327 const c = this.cam;
328 // element rect in untransformed frame coordinates
329 const lx = (r.left - st.left - c.tx) / c.s;
330 const ly = (r.top - st.top - c.ty) / c.s;
331 const lw = r.width / c.s;
332 const lh = r.height / c.s;
333 const s = st.height * (cfg.fill || 0.6) / lh;
334 const tx = st.width / 2 - (lx + lw / 2) * s;
335 const ty = st.height * (cfg.centerY || 0.48) - (ly + lh / 2) * s;
336 this.cam = { tx, ty, s };
337 this.camera.style.transform = `translate(${tx}px, ${ty}px) scale(${s})`;
338 }
339
340 // ---------------------------------------------------------- idle life
341
342 startIdle() {
343 this.stopIdle();
344 const idle = this.manifest.idle || {};
345 if (idle.breathe) {
346 const el = this.$(idle.breathe.target);
347 if (el) {
348 const amt = idle.breathe.amount || 1.4;
349 this.breathAnim = el.animate(
350 [{ transform: 'translateY(0) scaleY(1)' },
351 { transform: `translateY(${-amt}px) scaleY(1.012)` },
352 { transform: 'translateY(0) scaleY(1)' }],
353 { duration: idle.breathe.period || 4200, iterations: Infinity, easing: 'ease-in-out' });
354 }
355 }
356 if (idle.gaze && this.manifest.look) {
357 const drift = () => {
358 if (!this.source) { // hold eye contact while speaking
359 const cfg = this.manifest.look;
360 const dx = (Math.random() * 2 - 1) * (cfg.dx || 5) * 0.7;
361 const dy = (Math.random() * 2 - 1) * (cfg.dy || 2) * 0.7;
362 this.setGaze(Math.random() < 0.4
363 ? 'translate(0,0)' : `translate(${dx}px, ${dy}px)`, 500);
364 }
365 this.gazeTimer = setTimeout(drift, 3000 + Math.random() * 5000);
366 };
367 this.gazeTimer = setTimeout(drift, 3500);
368 }
369 }
370
371 stopIdle() {
372 if (this.breathAnim) { this.breathAnim.cancel(); this.breathAnim = null; }
373 clearTimeout(this.gazeTimer);
374 }
375
376 startBlinking() {
377 clearTimeout(this.blinkTimer);
378 const cfg = this.manifest.blink;
379 if (!cfg) return;
380 const blink = () => {
381 const el = this.$(cfg.target);
382 if (el) {
383 el.animate(
384 [{ transform: 'scaleY(1)' }, { transform: 'scaleY(0.08)' }, { transform: 'scaleY(1)' }],
385 { duration: 140, easing: 'ease-in-out' });
386 }
387 this.blinkTimer = setTimeout(blink, 2500 + Math.random() * 3000);
388 };
389 this.blinkTimer = setTimeout(blink, 1500);
390 }
391
392 // ----------------------------------------------------------- wearables
393 //
394 // A worn prop is a <g class="worn"> injected as a following sibling of
395 // the anchor element, so it inherits that element's parent transforms
396 // (head bobs/nods) automatically. Placement: bottom-center of the prop
397 // sits at top-center of the anchor's bbox; width = anchor bbox width x
398 // the wear's scale (default 1); height keeps the prop's own aspect via
399 // its viewBox. Stored per anchor so a second wear on the same anchor
400 // replaces the first.
401
402 async wear(anchor, propId, scale = 1) {
403 const anchorSel = (this.manifest.anchors && this.manifest.anchors[anchor]) || '#head';
404 const anchorEl = this.$(anchorSel);
405 if (!anchorEl) { toast(`wear: no anchor "${anchor}" on ${this.characterName}`); return; }
406
407 let res;
408 try {
409 res = await fetch(assetUrl('props', propId));
410 if (!res.ok) throw new Error('missing asset');
411 } catch {
412 toast(`wear: missing prop "${propId}"`);
413 return;
414 }
415 const ctype = res.headers.get('content-type') || '';
416 if (!ctype.includes('svg')) {
417 toast(`wear: prop "${propId}" isn't vector art (wearables need SVG)`);
418 return;
419 }
420 const svgText = await res.text();
421 const doc = new DOMParser().parseFromString(svgText, 'image/svg+xml');
422 const propSvg = doc.documentElement;
423 if (!propSvg || propSvg.nodeName !== 'svg' || doc.querySelector('parsererror')) {
424 toast(`wear: invalid prop asset "${propId}"`);
425 return;
426 }
427
428 const vbParts = (propSvg.getAttribute('viewBox') || '').trim().split(/\s+/).map(Number);
429 const hasVb = vbParts.length === 4 && vbParts.every((n) => !isNaN(n));
430 const [vbX, vbY, vbW, vbH] = hasVb ? vbParts
431 : [0, 0, parseFloat(propSvg.getAttribute('width')) || 100, parseFloat(propSvg.getAttribute('height')) || 100];
432
433 const bbox = anchorEl.getBBox();
434 const width = bbox.width * scale;
435 const height = vbW ? width * (vbH / vbW) : width;
436 const px = bbox.x + bbox.width / 2 - width / 2; // bottom-center of prop = top-center of anchor
437 const py = bbox.y - height;
438 const k = vbW ? width / vbW : 1;
439
440 const ns = 'http://www.w3.org/2000/svg';
441 const g = document.createElementNS(ns, 'g');
442 g.setAttribute('class', 'worn');
443 g.dataset.anchor = anchor;
444 g.setAttribute('transform', `translate(${px}, ${py}) scale(${k}) translate(${-vbX}, ${-vbY})`);
445 for (const child of [...propSvg.children]) g.appendChild(document.importNode(child, true));
446
447 this.unwear(anchor); // second wear on the same anchor replaces
448 anchorEl.parentNode.insertBefore(g, anchorEl.nextSibling);
449 this.wornProps.set(anchor, g);
450 }
451
452 unwear(anchor) {
453 const g = this.wornProps.get(anchor);
454 if (g) { g.remove(); this.wornProps.delete(anchor); }
455 }
456}
457
458// ---------------------------------------------------------------- Stage
459//
460// The Stage owns a map of frames (id -> {el, bgEl, contentEl, puppet}),
461// each an independent rectangular sub-stage: background, optional content
462// tile, optional character (a Puppet bound to that frame's DOM). Cues route
463// by cue.frame, defaulting to the active frame (boots as "main", full-size),
464// so with no frame cue ever sent the stage behaves exactly as a single
465// puppet on one full-stage frame.
466
467class Stage {
468 constructor(framesEl, overlaysEl, captionsEl, takeEl) {
469 this.framesEl = framesEl;
470 this.overlaysEl = overlaysEl;
471 this.frames = new Map();
472 this.activeId = 'main';
473 this.overlays = new Map();
474 this.overlaySeq = 0;
475 this.overlayCache = new Map();
476 this.autoSlots = ['left', 'right'];
477 this.autoIdx = 0;
478
479 // captions: last-speaker-wins subtitle bar, off by default
480 this.captionsEl = captionsEl;
481 this.captionsOn = false;
482 this.captionsTimer = null;
483
484 // take: fullscreen transitions (iris/fade) share one layer + hole child
485 this.takeEl = takeEl;
486 this.holeEl = takeEl && takeEl.querySelector('.iris-hole');
487 this.takeAnim = null;
488 }
489
490 ensureFrame(id) { return this.frames.get(id) || this.createFrame(id); }
491
492 createFrame(id) {
493 const el = document.createElement('div');
494 el.className = 'frame';
495 el.dataset.id = id;
496 el.innerHTML =
497 '<div class="frame-bg"></div>' +
498 '<div class="frame-content"></div>' +
499 '<div class="frame-camera"><div class="frame-pos"><div class="frame-flip"></div></div></div>';
500 this.framesEl.appendChild(el);
501 const f = {
502 id, el,
503 bgEl: el.querySelector('.frame-bg'),
504 contentEl: el.querySelector('.frame-content'),
505 puppet: null,
506 actors: new Map(), // actor id -> { id, what, kind: 'character'|'prop', x, scale, behind, el, puppet }
507 };
508 this.frames.set(id, f);
509 this.setRect(f, 'full');
510 return f;
511 }
512
513 setRect(f, slot, rect) {
514 const r = rect || SLOTS[slot] || SLOTS.full;
515 f.el.style.left = r[0] + '%';
516 f.el.style.top = r[1] + '%';
517 f.el.style.width = r[2] + '%';
518 f.el.style.height = r[3] + '%';
519 this.rescaleFrame(f); // rect changed -> frame's pixel height likely changed too
520 }
521
522 // Recompute one frame's puppet scale (see Puppet.applyScale) and, if it's
523 // holding a face close-up, re-frame it so the shot doesn't drift once the
524 // underlying height has changed. Safe to call on a frame with no puppet.
525 rescaleFrame(f) {
526 if (f.puppet) {
527 f.puppet.applyScale();
528 if (f.puppet.view === 'face') f.puppet.setView('face', true);
529 }
530 for (const a of f.actors.values()) {
531 if (a.puppet) {
532 a.puppet.applyScale();
533 if (a.puppet.view === 'face') a.puppet.setView('face', true);
534 } else if (a.kind === 'prop') {
535 this.applyPropScale(f, a);
536 }
537 }
538 }
539
540 rescaleAll() {
541 for (const f of this.frames.values()) this.rescaleFrame(f);
542 }
543
544 autoSlot() {
545 const slot = this.autoSlots[this.autoIdx] || 'full';
546 this.autoIdx = Math.min(this.autoIdx + 1, this.autoSlots.length);
547 return slot;
548 }
549
550 async loadCharacterInto(f, name) {
551 if (!f.puppet) f.puppet = new Puppet(f.el);
552 await f.puppet.load(name);
553 }
554
555 // -------------------------------------------------------- frame cues
556
557 frame(cue) {
558 const existed = this.frames.has(cue.id);
559 const f = this.ensureFrame(cue.id);
560 if (cue.rect) this.setRect(f, null, cue.rect);
561 else if (cue.slot) this.setRect(f, cue.slot);
562 else if (!existed) this.setRect(f, this.autoSlot());
563 if (cue.bg !== undefined) this.setBg(f, cue.bg);
564 if (!cue.character) {
565 if (cue.view && f.puppet) f.puppet.setView(cue.view);
566 if (cue.facing !== undefined && f.puppet) f.puppet.face(cue.facing);
567 }
568 this.activeId = cue.id;
569 return f;
570 }
571
572 frameClear(cue) {
573 if (this.frames.size <= 1) return; // never remove the last frame
574 const f = this.frames.get(cue.id);
575 if (!f) return;
576 if (f.puppet) f.puppet.stopSpeaking();
577 for (const a of f.actors.values()) if (a.puppet) a.puppet.stopSpeaking();
578 f.el.remove();
579 this.frames.delete(cue.id);
580 if (this.activeId === cue.id) {
581 this.activeId = this.frames.has('main') ? 'main' : this.frames.keys().next().value;
582 }
583 }
584
585 layout(cue) {
586 const preset = cue.preset || 'single';
587 const specs = LAYOUT_PRESETS[preset] || LAYOUT_PRESETS.single;
588 const mainF = this.frames.get('main');
589 const keepChar = mainF && mainF.puppet ? mainF.puppet.characterName : null;
590
591 for (const f of this.frames.values()) {
592 if (f.puppet) f.puppet.stopSpeaking();
593 for (const a of f.actors.values()) if (a.puppet) a.puppet.stopSpeaking();
594 f.el.remove();
595 }
596 this.frames.clear();
597 this.autoIdx = 0;
598
599 let firstId = null;
600 for (const spec of specs) {
601 const f = this.createFrame(spec.id);
602 this.setRect(f, spec.slot);
603 if (!firstId) firstId = spec.id;
604 if (spec.id === 'main' && keepChar) this.loadCharacterInto(f, keepChar);
605 }
606 this.activeId = this.frames.has('main') ? 'main' : firstId;
607 }
608
609 // ------------------------------------------------------------ content
610
611 content(cue) {
612 const id = cue.frame || this.activeId;
613 if (cue.frame) this.activeId = id;
614 this.renderContent(this.ensureFrame(id), cue);
615 }
616
617 contentClear(cue) {
618 const f = this.frames.get(cue.frame || this.activeId);
619 if (f) f.contentEl.innerHTML = '';
620 }
621
622 renderContent(f, cue) {
623 const el = f.contentEl;
624 el.innerHTML = '';
625 if (cue.kind === 'image' || cue.kind === 'video') {
626 const src = looksLikeUrl(cue.value) ? cue.value : assetUrl('props', cue.value);
627 const tag = document.createElement(cue.kind === 'video' ? 'video' : 'img');
628 tag.src = src;
629 tag.style.maxWidth = '100%';
630 tag.style.maxHeight = '100%';
631 tag.style.objectFit = cue.fit || 'contain';
632 if (cue.kind === 'video') { tag.autoplay = true; tag.loop = true; tag.muted = true; tag.playsInline = true; }
633 el.appendChild(tag);
634 } else {
635 const div = document.createElement('div');
636 div.className = 'content-text';
637 div.textContent = cue.value;
638 el.appendChild(div);
639 }
640 }
641
642 // -------------------------------------------------------------- scene
643
644 scene(cue) {
645 const id = cue.frame || this.activeId;
646 if (cue.frame) this.activeId = id;
647 this.setBg(this.ensureFrame(id), cue.bg);
648 }
649
650 setBg(f, bg) {
651 if (bg === undefined) return;
652 f.bgEl.innerHTML = '';
653 if (!bg) { f.bgEl.style.background = ''; return; }
654 if (/^#[0-9a-f]{3,8}$/i.test(bg) || /^(linear|radial)-gradient\(/.test(bg) || looksLikeUrl(bg)) {
655 f.bgEl.style.background = /^(linear|radial)-gradient\(/.test(bg) || bg.startsWith('#')
656 ? bg : `center / cover no-repeat url("${bg}")`;
657 return;
658 }
659 this.loadBgAsset(f, bg); // asset id: resolve against assets/backgrounds/<id>.*
660 }
661
662 async loadBgAsset(f, id) {
663 try {
664 const res = await fetch(assetUrl('backgrounds', id));
665 if (!res.ok) throw new Error('missing asset');
666 const ctype = res.headers.get('content-type') || '';
667 if (ctype.includes('svg')) {
668 f.bgEl.innerHTML = await res.text();
669 f.bgEl.style.background = '';
670 } else {
671 f.bgEl.style.background = `center / cover no-repeat url("${res.url}")`;
672 }
673 } catch { /* asset pack may not have this id yet; leave background as-is */ }
674 }
675
676 // ------------------------------------------------------------- actors
677 //
678 // An actor is a placed entity in a frame in addition to its primary
679 // character: either a character instance (a full Puppet bound to its own
680 // DOM subtree, sibling to the primary's) or a prop instance (a plain
681 // positioned element holding the resolved asset). Both stand on the same
682 // floor line as the primary, positioned by x% (center-anchored), sized by
683 // scale. z-order: appended after the primary (in front) unless `behind`.
684
685 place(cue) {
686 const frameId = cue.frame || this.activeId;
687 if (cue.frame) this.activeId = frameId;
688 return this.placeActor(this.ensureFrame(frameId), cue);
689 }
690
691 async placeActor(f, cue) {
692 const id = cue.id || cue.what;
693 const existing = f.actors.get(id);
694 const x = cue.x !== undefined && !isNaN(cue.x) ? cue.x : undefined;
695
696 if (existing && existing.what === cue.what) {
697 // Same id + same `what`: update in place (x/scale glide via transition).
698 if (x !== undefined) {
699 existing.x = x;
700 if (existing.kind === 'character' && existing.puppet) existing.puppet.setX(x, 600);
701 else this.moveActor(existing, x);
702 }
703 if (cue.scale !== undefined) {
704 existing.scale = cue.scale;
705 if (existing.kind === 'character' && existing.puppet) {
706 const svg = existing.puppet.svg;
707 existing.puppet.actorScale = cue.scale;
708 if (svg) { svg.style.transition = 'height 600ms ease'; existing.puppet.applyScale(); }
709 } else {
710 this.scaleActor(existing, cue.scale);
711 }
712 }
713 if (cue.behind !== undefined && cue.behind !== existing.behind) {
714 existing.behind = cue.behind;
715 this.insertActorEl(f, existing);
716 }
717 return;
718 }
719 if (existing) this.destroyActor(f, existing); // different `what` at the same id: replace
720
721 const isCharacter = await this.probeCharacter(cue.what);
722 const actor = {
723 id, what: cue.what, kind: isCharacter ? 'character' : 'prop',
724 x: x !== undefined ? x : 50,
725 scale: cue.scale !== undefined ? cue.scale : (isCharacter ? 1 : 0.4),
726 behind: !!cue.behind,
727 frameId: f.id, el: null, puppet: null,
728 };
729 f.actors.set(id, actor);
730
731 if (actor.kind === 'character') {
732 const wrap = document.createElement('div');
733 wrap.className = 'frame-actor';
734 wrap.dataset.id = id;
735 wrap.innerHTML = '<div class="frame-camera"><div class="frame-pos"><div class="frame-flip"></div></div></div>';
736 actor.el = wrap;
737 this.insertActorEl(f, actor);
738 const puppet = new Puppet(wrap);
739 puppet.actorScale = actor.scale;
740 actor.puppet = puppet;
741 await puppet.load(cue.what);
742 puppet.setX(actor.x, 0);
743 } else {
744 const wrap = document.createElement('div');
745 wrap.className = 'prop-actor';
746 wrap.dataset.id = id;
747 actor.el = wrap;
748 this.insertActorEl(f, actor);
749 wrap.style.left = actor.x + '%';
750 this.applyPropScale(f, actor);
751 await this.loadPropAsset(actor, cue.what);
752 }
753 }
754
755 // Resolves whether `what` names a character (a rig, direction-capable) or
756 // a prop asset, the same way loading a character into a frame does —
757 // fetch its manifest and see if the server (or the dbbasic shim) has it.
758 async probeCharacter(what) {
759 try {
760 const r = await fetch(`/characters/${what}/manifest.json`);
761 return r.ok;
762 } catch { return false; }
763 }
764
765 insertActorEl(f, actor) {
766 const primaryCam = f.el.querySelector(':scope > .frame-camera');
767 if (actor.behind && primaryCam) f.el.insertBefore(actor.el, primaryCam);
768 else f.el.appendChild(actor.el);
769 }
770
771 applyPropScale(f, actor) {
772 const frameH = f.el.clientHeight || 720;
773 const h = Math.max(20, PROP_DESIGN_HEIGHT * actor.scale * (frameH / 720));
774 actor.el.style.height = h + 'px';
775 }
776
777 async loadPropAsset(actor, propId) {
778 try {
779 const res = await fetch(assetUrl('props', propId));
780 if (!res.ok) throw new Error('missing asset');
781 const ctype = res.headers.get('content-type') || '';
782 if (ctype.includes('svg')) {
783 actor.el.innerHTML = await res.text();
784 } else {
785 const img = document.createElement('img');
786 img.src = res.url;
787 actor.el.appendChild(img);
788 }
789 } catch {
790 toast(`place: missing prop asset "${propId}"`);
791 // Leave a visible placeholder so the id/scale/x is still legible on
792 // stage instead of silently rendering nothing.
793 actor.el.style.background = 'rgba(255,255,255,.12)';
794 actor.el.style.border = '1px dashed rgba(255,255,255,.4)';
795 actor.el.style.minWidth = '0.6em';
796 }
797 }
798
799 removeActor(cue) {
800 const target = cue.frame ? this.frames.get(cue.frame) : null;
801 let f = target, actor = f && f.actors.get(cue.id);
802 if (!actor) { // no explicit frame, or not found there: search every frame
803 for (const cand of this.frames.values()) {
804 if (cand.actors.has(cue.id)) { f = cand; actor = cand.actors.get(cue.id); break; }
805 }
806 }
807 if (!actor) return;
808 this.destroyActor(f, actor);
809 }
810
811 destroyActor(f, actor) {
812 if (actor.puppet) actor.puppet.stopSpeaking();
813 actor.el.remove();
814 f.actors.delete(actor.id);
815 }
816
817 // ------------------------------------------------------- full-stage reset
818 //
819 // Shared by the `script-start` cue (a new script is about to run) and the
820 // `[clear]` mid-script direction: wipes everything a script builds up on
821 // top of a frame — placed actors and their speech, worn props, content
822 // tiles, and overlays — so a re-run (or a mid-show reset) starts from a
823 // clean stage. Deliberately leaves music (it survives script-start by
824 // spec), captions on/off state, backgrounds, and the frame layout itself
825 // untouched — those aren't "things a script builds up", they're ambient
826 // stage state.
827 clearAll() {
828 for (const f of this.frames.values()) {
829 if (f.puppet) {
830 for (const anchor of [...f.puppet.wornProps.keys()]) f.puppet.unwear(anchor);
831 f.puppet.stopSpeaking();
832 }
833 for (const actor of [...f.actors.values()]) {
834 if (actor.puppet) for (const anchor of [...actor.puppet.wornProps.keys()]) actor.puppet.unwear(anchor);
835 this.destroyActor(f, actor); // stops actor speech + removes its DOM
836 }
837 f.contentEl.innerHTML = '';
838 }
839 this.overlayClear();
840 }
841
842 // Resolves the actor named by cue.actor, scoped to cue.frame (or the
843 // active frame) first; if no frame was explicitly given, falls back to a
844 // search across every frame (actor ids are typically unique stage-wide).
845 resolveActor(cue) {
846 const frameId = cue.frame || this.activeId;
847 if (cue.frame) this.activeId = frameId;
848 const f = this.frames.get(frameId);
849 let actor = f && f.actors.get(cue.actor);
850 if (!actor && !cue.frame) {
851 for (const cand of this.frames.values()) {
852 if (cand.actors.has(cue.actor)) { actor = cand.actors.get(cue.actor); break; }
853 }
854 }
855 return actor;
856 }
857
858 // ---------------------------------------------------- prop directions
859 //
860 // [<id> move N] / [<id> scale N] / [<id> spin] / [<id> bounce] — only
861 // meaningful for prop actors (character actors take the full walk/action
862 // direction set through their own Puppet instead).
863
864 moveActor(actor, x, instant) {
865 if (actor.kind !== 'prop') return;
866 actor.x = x;
867 actor.el.style.transition = instant ? 'none' : 'left 600ms ease';
868 actor.el.style.left = x + '%';
869 }
870
871 scaleActor(actor, scale, instant) {
872 if (actor.kind !== 'prop') return;
873 actor.scale = scale;
874 actor.el.style.transition = instant ? 'none' : 'height 600ms ease';
875 const f = this.frames.get(actor.frameId);
876 if (f) this.applyPropScale(f, actor);
877 }
878
879 spinActor(actor) {
880 if (actor.kind !== 'prop') return;
881 actor.el.style.transformOrigin = '50% 100%';
882 actor.el.animate(
883 [{ transform: 'translateX(-50%) rotate(0deg)' }, { transform: 'translateX(-50%) rotate(360deg)' }],
884 { duration: 700, easing: 'ease-in-out' });
885 }
886
887 bounceActor(actor) {
888 if (actor.kind !== 'prop') return;
889 actor.el.style.transformOrigin = '50% 100%';
890 actor.el.animate([
891 { transform: 'translateX(-50%) translateY(0) scale(1,1)' },
892 { transform: 'translateX(-50%) translateY(0) scale(1.15,0.85)', offset: 0.15 },
893 { transform: 'translateX(-50%) translateY(-30px) scale(0.9,1.15)', offset: 0.5 },
894 { transform: 'translateX(-50%) translateY(0) scale(1.15,0.85)', offset: 0.85 },
895 { transform: 'translateX(-50%) translateY(0) scale(1,1)' },
896 ], { duration: 500, easing: 'ease-in-out' });
897 }
898
899 // ----------------------------------------------------------- wearables
900
901 // `target` is a frame id (its primary character) or an actor id; frame
902 // ids resolve first, matching every other target-resolution rule here.
903 // Falls back to a character-name search (see findPuppetByCharacterName)
904 // as belt-and-braces — normally the parser has already resolved a
905 // character name to its frame/actor id by the time a cue gets here.
906 resolveWearTarget(targetId) {
907 const f = this.frames.get(targetId);
908 if (f && f.puppet) return f.puppet;
909 for (const cand of this.frames.values()) {
910 const a = cand.actors.get(targetId);
911 if (a && a.puppet) return a.puppet;
912 }
913 return this.findPuppetByCharacterName(targetId);
914 }
915
916 // Belt-and-braces: find a puppet whose loaded character matches `name`,
917 // checking each frame's primary character first, then its actors. Only
918 // reached when a cue's frame/actor/target id doesn't resolve as an id —
919 // normally the parser has already turned a character name into a real
920 // frame/actor id by the time a cue reaches the client. Kept cheap: two
921 // flat passes over already-loaded puppets, no network/DOM work.
922 findPuppetByCharacterName(name) {
923 if (!name) return null;
924 for (const f of this.frames.values()) {
925 if (f.puppet && f.puppet.characterName === name) return f.puppet;
926 }
927 for (const f of this.frames.values()) {
928 for (const a of f.actors.values()) {
929 if (a.puppet && a.puppet.characterName === name) return a.puppet;
930 }
931 }
932 return null;
933 }
934
935 wear(cue) {
936 const puppet = this.resolveWearTarget(cue.target);
937 if (!puppet) { toast(`wear: unknown target "${cue.target}"`); return; }
938 puppet.wear(cue.anchor || 'head', cue.prop, cue.scale !== undefined ? cue.scale : 1);
939 }
940
941 unwear(cue) {
942 const puppet = this.resolveWearTarget(cue.target);
943 if (!puppet) return;
944 puppet.unwear(cue.anchor || 'head');
945 }
946
947 // ----------------------------------------------------------- overlays
948
949 overlay(cue) {
950 const id = cue.id || `ov${++this.overlaySeq}`;
951 let ov = this.overlays.get(id);
952 if (!ov) {
953 ov = document.createElement('div');
954 ov.className = 'overlay';
955 this.overlaysEl.appendChild(ov);
956 this.overlays.set(id, ov);
957 }
958 clearTimeout(ov._holdTimer);
959 ov.dataset.enter = cue.enter || '';
960 ov.dataset.exit = cue.exit || '';
961 this.renderOverlay(ov, cue.template, cue.slots || {});
962 requestAnimationFrame(() => ov.classList.add('shown'));
963 if (cue.hold) ov._holdTimer = setTimeout(() => this.overlayClear({ id }), cue.hold);
964 }
965
966 overlayClear(cue) {
967 const removeOne = (id) => {
968 const ov = this.overlays.get(id);
969 if (!ov) return;
970 clearTimeout(ov._holdTimer);
971 this.overlays.delete(id);
972 ov.classList.remove('shown');
973 setTimeout(() => ov.remove(), 400);
974 };
975 if (cue && cue.id) { removeOne(cue.id); return; }
976 for (const id of [...this.overlays.keys()]) removeOne(id);
977 }
978
979 async renderOverlay(ov, template, slots) {
980 let html = this.overlayCache.get(template);
981 if (html === undefined) {
982 try { html = await (await fetch(assetUrl('overlays', template))).text(); }
983 catch { html = ''; }
984 this.overlayCache.set(template, html);
985 }
986 ov.innerHTML = html;
987 for (const [key, val] of Object.entries(slots)) {
988 const el = ov.querySelector(`[data-slot="${key}"]`);
989 if (el) el.textContent = val;
990 }
991 }
992
993 // ------------------------------------------------------------ captions
994
995 captions(cue) {
996 this.captionsOn = cue.on !== false;
997 if (!this.captionsOn) this.hideCaption();
998 }
999
1000 showCaption(cue) {
1001 if (!this.captionsOn) return;
1002 clearTimeout(this.captionsTimer);
1003 this.captionsEl.textContent = cue.text;
1004 this.captionsEl.classList.add('shown');
1005 this.captionsTimer = setTimeout(() => this.hideCaption(), (cue.duration || 0) * 1000 + 300);
1006 }
1007
1008 hideCaption() {
1009 clearTimeout(this.captionsTimer);
1010 this.captionsTimer = null;
1011 this.captionsEl.classList.remove('shown');
1012 this.captionsEl.textContent = '';
1013 }
1014
1015 // --------------------------------------------------------- transitions
1016 //
1017 // #take is one shared layer for both fade and iris: fade animates the
1018 // layer's own opacity (a flat black fill); iris animates a child "hole"
1019 // element's diameter, whose box-shadow paints black everywhere outside
1020 // it (clipped to #stage by #stage's overflow:hidden). Each call resets
1021 // whichever mode isn't in use so the two never fight over #take's state.
1022
1023 transition(cue) {
1024 if (this.takeAnim) { this.takeAnim.cancel(); this.takeAnim = null; }
1025 const ms = cue.ms || 700;
1026 if (cue.name === 'iris') this.irisTransition(cue.dir, ms);
1027 else this.fadeTransition(cue.dir, ms);
1028 }
1029
1030 fadeTransition(dir, ms) {
1031 const el = this.takeEl;
1032 if (this.holeEl) this.holeEl.style.display = 'none';
1033 el.style.display = 'block';
1034 el.style.background = '#000';
1035 const from = dir === 'in' ? 1 : 0;
1036 const to = dir === 'in' ? 0 : 1;
1037 el.style.opacity = from;
1038 const anim = el.animate([{ opacity: from }, { opacity: to }], { duration: ms, easing: 'linear', fill: 'forwards' });
1039 this.takeAnim = anim;
1040 anim.finished.then(() => {
1041 if (this.takeAnim !== anim) return;
1042 el.style.opacity = to;
1043 if (dir === 'in') el.style.display = 'none';
1044 }).catch(() => {});
1045 }
1046
1047 irisTransition(dir, ms) {
1048 const el = this.takeEl;
1049 const hole = this.holeEl;
1050 if (!hole) return;
1051 el.style.background = 'transparent';
1052 el.style.opacity = '1';
1053 el.style.display = 'block';
1054 hole.style.display = 'block';
1055 const full = Math.hypot(el.clientWidth, el.clientHeight) + 100; // fully clears the stage
1056 const from = dir === 'in' ? 0 : full;
1057 const to = dir === 'in' ? full : 0;
1058 hole.style.width = from + 'px';
1059 hole.style.height = from + 'px';
1060 const anim = hole.animate(
1061 [{ width: from + 'px', height: from + 'px' }, { width: to + 'px', height: to + 'px' }],
1062 { duration: ms, easing: 'linear', fill: 'forwards' });
1063 this.takeAnim = anim;
1064 anim.finished.then(() => {
1065 if (this.takeAnim !== anim) return;
1066 hole.style.width = to + 'px';
1067 hole.style.height = to + 'px';
1068 if (dir === 'in') { el.style.display = 'none'; hole.style.display = 'none'; }
1069 }).catch(() => {});
1070 }
1071
1072 // ----------------------------------------------- character direction
1073
1074 forFrame(cue) {
1075 const id = cue.frame || this.activeId;
1076 if (cue.frame) this.activeId = id;
1077 return this.ensureFrame(id).puppet;
1078 }
1079}
1080
1081// ------------------------------------------------------------- MusicEngine
1082//
1083// Stage-global background music + one-shot SFX, driven by small JSON note
1084// patterns (assets/music/*.json, assets/sfx/*.json — see docs/design/
1085// actors-wearables-music.md) rather than audio files. One instance for the
1086// whole stage (not per-frame): music survives frame/layout changes and
1087// script-start; only an explicit `[music off]` stops it. The instance is
1088// created lazily on the first `music`/`sfx` cue (see ensureMusicEngine
1089// below), so a script that never uses either never constructs an
1090// AudioContext or makes a sound — same backward-compatibility contract as
1091// every other additive feature here.
1092//
1093// Scheduling is the standard Web Audio "lookahead" pattern: a ~100ms timer
1094// looks ~300ms into ctx.currentTime and schedules any notes that fall due in
1095// that window with their exact absolute start time (rather than one
1096// setTimeout per note, which drifts). Each note is an OscillatorNode of the
1097// track's wave through its own short-envelope GainNode (~10ms linear attack,
1098// ~50ms linear release) so notes don't click. `loop:true` patterns are
1099// scheduled by wall-clock offset from an absolute pattern-start time modulo
1100// the pattern's total beat length, so they repeat seamlessly.
1101//
1102// Two gain stages, deliberately kept separate:
1103// - a per-player "layer" gain (crossfade control) — a new music cue ramps
1104// a fresh player's layer in from 0 while the old player's layer ramps
1105// out, both over ~300ms, then the old player is torn down.
1106// - the shared `musicMaster` gain (ducking control) that every music
1107// layer feeds into on its way to the destination, base ~0.5, ramped to
1108// 40% of base while any speech clip is playing anywhere on the stage
1109// (see Puppet.speak/stopSpeaking's musicEngine.duck()/unduck() calls)
1110// and back once nothing is speaking.
1111// SFX are one-shots: their own player connects straight to ctx.destination,
1112// bypassing musicMaster entirely, so they're never ducked and never
1113// crossfaded — they just play once over whatever else is happening.
1114
1115const MUSIC_LOOKAHEAD_MS = 100; // scheduler tick interval
1116const MUSIC_SCHEDULE_AHEAD = 0.3; // seconds of lookahead scheduled per tick
1117const MUSIC_BASE_GAIN = 0.5;
1118const MUSIC_DUCK_FACTOR = 0.4; // fraction of base gain while speech plays
1119const MUSIC_DUCK_MS = 150;
1120const MUSIC_CROSSFADE_MS = 300;
1121const NOTE_ATTACK = 0.01;
1122const NOTE_RELEASE = 0.05;
1123
1124function midiToFreq(m) { return 440 * Math.pow(2, (m - 69) / 12); }
1125
1126// One playing instance of a note pattern (either the current music track, an
1127// outgoing music track mid-crossfade, or a single SFX one-shot). Owns its own
1128// lookahead scheduler and layer GainNode; `output` is where that layer feeds
1129// (musicMaster for music, ctx.destination directly for sfx).
1130class PatternPlayer {
1131 constructor(ctx, pattern, output) {
1132 this.ctx = ctx;
1133 this.loop = !!pattern.loop;
1134 this.secPerBeat = 60 / (pattern.tempo || 120);
1135 this.notes = [];
1136 for (const track of pattern.tracks || []) {
1137 const wave = track.wave || 'sine';
1138 const gain = track.gain !== undefined ? track.gain : 0.2;
1139 for (const [beat, midi, len] of track.notes || []) {
1140 this.notes.push({ beat, midi, len, wave, gain });
1141 }
1142 }
1143 this.notes.sort((a, b) => a.beat - b.beat);
1144 this.patternBeats = this.notes.reduce((m, n) => Math.max(m, n.beat + n.len), 0) || 1;
1145
1146 this.layerGain = ctx.createGain();
1147 this.layerGain.gain.value = 1;
1148 this.layerGain.connect(output);
1149
1150 this.loopStart = null;
1151 this.nextIdx = 0;
1152 this.timer = null;
1153 this.stopped = false;
1154 }
1155
1156 start(fadeInMs) {
1157 const now = this.ctx.currentTime;
1158 this.loopStart = now + 0.05; // small safety margin so the first note never lands in the past
1159 this.nextIdx = 0;
1160 if (fadeInMs) {
1161 this.layerGain.gain.setValueAtTime(0, now);
1162 this.layerGain.gain.linearRampToValueAtTime(1, now + fadeInMs / 1000);
1163 } else {
1164 this.layerGain.gain.setValueAtTime(1, now);
1165 }
1166 this.schedule(); // cover the first window immediately; no gap before the timer's first tick
1167 this.timer = setInterval(() => this.schedule(), MUSIC_LOOKAHEAD_MS);
1168 }
1169
1170 schedule() {
1171 if (this.stopped) return;
1172 const aheadUntil = this.ctx.currentTime + MUSIC_SCHEDULE_AHEAD;
1173 const loopSeconds = this.patternBeats * this.secPerBeat;
1174 for (;;) {
1175 if (this.nextIdx >= this.notes.length) {
1176 if (!this.loop) {
1177 if (this.ctx.currentTime > this.loopStart + loopSeconds) this.stop();
1178 return;
1179 }
1180 this.loopStart += loopSeconds;
1181 this.nextIdx = 0;
1182 continue;
1183 }
1184 const note = this.notes[this.nextIdx];
1185 const t = this.loopStart + note.beat * this.secPerBeat;
1186 if (t > aheadUntil) return;
1187 this.playNote(note, t);
1188 this.nextIdx++;
1189 }
1190 }
1191
1192 playNote(note, t) {
1193 const ctx = this.ctx;
1194 const osc = ctx.createOscillator();
1195 osc.type = note.wave;
1196 osc.frequency.setValueAtTime(midiToFreq(note.midi), t);
1197 const g = ctx.createGain();
1198 const dur = Math.max(note.len * this.secPerBeat, NOTE_ATTACK + NOTE_RELEASE);
1199 const releaseStart = Math.max(t + NOTE_ATTACK, t + dur - NOTE_RELEASE);
1200 g.gain.setValueAtTime(0, t);
1201 g.gain.linearRampToValueAtTime(note.gain, t + NOTE_ATTACK);
1202 g.gain.setValueAtTime(note.gain, releaseStart);
1203 g.gain.linearRampToValueAtTime(0, releaseStart + NOTE_RELEASE);
1204 osc.connect(g);
1205 g.connect(this.layerGain);
1206 osc.start(t);
1207 osc.stop(releaseStart + NOTE_RELEASE + 0.02);
1208 }
1209
1210 fadeOutAndStop(ms) {
1211 const now = this.ctx.currentTime;
1212 this.layerGain.gain.cancelScheduledValues(now);
1213 this.layerGain.gain.setValueAtTime(this.layerGain.gain.value, now);
1214 this.layerGain.gain.linearRampToValueAtTime(0, now + ms / 1000);
1215 setTimeout(() => this.stop(), ms + 20);
1216 }
1217
1218 stop() {
1219 if (this.stopped) return;
1220 this.stopped = true;
1221 clearInterval(this.timer);
1222 try { this.layerGain.disconnect(); } catch { /* already disconnected */ }
1223 }
1224}
1225
1226class MusicEngine {
1227 constructor() {
1228 this.ctx = null;
1229 this.musicMaster = null;
1230 this.player = null; // currently-current music PatternPlayer (an outgoing one during crossfade keeps its own timer alive independently)
1231 this.currentId = null;
1232 this.cache = new Map(); // "kind/id" -> parsed JSON, or null for a confirmed-missing asset
1233 this.pending = new Map(); // "kind/id" -> in-flight fetch promise
1234 this.pendingMusicId = null; // desired track id requested before the AudioContext was running; started on unlock
1235 this.musicGen = 0; // bumped on every music()/stopMusic() call so a stale in-flight fetch can't clobber a newer request
1236 this.duckCount = 0;
1237 }
1238
1239 ensureCtx() {
1240 if (!this.ctx) {
1241 this.ctx = new (window.AudioContext || window.webkitAudioContext)();
1242 this.musicMaster = this.ctx.createGain();
1243 this.musicMaster.gain.value = MUSIC_BASE_GAIN;
1244 this.musicMaster.connect(this.ctx.destination);
1245 }
1246 return this.ctx;
1247 }
1248
1249 async fetchPattern(kind, id) {
1250 const key = `${kind}/${id}`;
1251 if (this.cache.has(key)) return this.cache.get(key);
1252 if (this.pending.has(key)) return this.pending.get(key);
1253 const p = (async () => {
1254 try {
1255 const res = await fetch(assetUrl(kind, id));
1256 if (!res.ok) throw new Error('missing asset');
1257 return await res.json();
1258 } catch {
1259 return null;
1260 }
1261 })();
1262 this.pending.set(key, p);
1263 const result = await p;
1264 this.pending.delete(key);
1265 this.cache.set(key, result);
1266 return result;
1267 }
1268
1269 // -------------------------------------------------------------- music
1270
1271 async music(cue) {
1272 const gen = ++this.musicGen;
1273 if (cue.off) { this.stopMusic(); return; }
1274 if (!cue.id) return;
1275 const ctx = this.ensureCtx();
1276 if (ctx.state !== 'running') {
1277 ctx.resume().catch(() => {});
1278 this.pendingMusicId = cue.id;
1279 toast('🔇 Click anywhere once to enable sound');
1280 return;
1281 }
1282 await this.startMusic(cue.id, gen);
1283 }
1284
1285 async startMusic(id, gen) {
1286 const pattern = await this.fetchPattern('music', id);
1287 if (gen !== this.musicGen) return; // superseded by a newer [music ...]/[music off] while fetching
1288 if (!pattern) { toast(`music: missing track "${id}"`); return; }
1289 const old = this.player;
1290 const fresh = new PatternPlayer(this.ctx, pattern, this.musicMaster);
1291 this.player = fresh;
1292 this.currentId = id;
1293 fresh.start(MUSIC_CROSSFADE_MS);
1294 if (old) old.fadeOutAndStop(MUSIC_CROSSFADE_MS);
1295 }
1296
1297 stopMusic() {
1298 this.musicGen++;
1299 this.pendingMusicId = null;
1300 if (this.player) { this.player.fadeOutAndStop(MUSIC_CROSSFADE_MS); this.player = null; }
1301 this.currentId = null;
1302 }
1303
1304 // ---------------------------------------------------------------- sfx
1305
1306 async sfx(cue) {
1307 if (!cue.id) return;
1308 const ctx = this.ensureCtx();
1309 if (ctx.state !== 'running') {
1310 await ctx.resume().catch(() => {});
1311 if (ctx.state !== 'running') { toast('🔇 Click anywhere once to enable sound'); return; }
1312 }
1313 const pattern = await this.fetchPattern('sfx', cue.id);
1314 if (!pattern) { toast(`sfx: missing "${cue.id}"`); return; }
1315 new PatternPlayer(ctx, { ...pattern, loop: false }, ctx.destination).start(0);
1316 }
1317
1318 // ------------------------------------------------------------- ducking
1319 //
1320 // A simple concurrent-speakers counter: Puppet.speak()/stopSpeaking() call
1321 // duck()/unduck() once per clip (across every frame/actor puppet), so
1322 // overlapping speech from more than one character still only ramps at the
1323 // 0->1 and 1->0 transitions.
1324
1325 duck() {
1326 this.duckCount++;
1327 if (this.duckCount === 1 && this.musicMaster) this.rampMaster(MUSIC_BASE_GAIN * MUSIC_DUCK_FACTOR);
1328 }
1329
1330 unduck() {
1331 this.duckCount = Math.max(0, this.duckCount - 1);
1332 if (this.duckCount === 0 && this.musicMaster) this.rampMaster(MUSIC_BASE_GAIN);
1333 }
1334
1335 rampMaster(target) {
1336 const now = this.ctx.currentTime;
1337 this.musicMaster.gain.cancelScheduledValues(now);
1338 this.musicMaster.gain.setValueAtTime(this.musicMaster.gain.value, now);
1339 this.musicMaster.gain.linearRampToValueAtTime(target, now + MUSIC_DUCK_MS / 1000);
1340 }
1341
1342 // Called from the shared gesture-unlock listener alongside every puppet's
1343 // own unlock (mirrors Puppet's suspended-context handling): if a [music]
1344 // cue arrived before any gesture, this is where its track actually starts.
1345 unlock() {
1346 if (!this.ctx) return;
1347 if (this.ctx.state !== 'running') this.ctx.resume().catch(() => {});
1348 if (this.pendingMusicId) {
1349 const id = this.pendingMusicId;
1350 this.pendingMusicId = null;
1351 this.startMusic(id, ++this.musicGen);
1352 }
1353 }
1354}
1355
1356// ------------------------------------------------------------- cue bus
1357
1358const $id = (i) => document.getElementById(i);
1359
1360const stage = new Stage($id('frames'), $id('overlays'), $id('captions'), $id('take'));
1361stage.ensureFrame('main'); // boots as one full-size frame + the default character
1362
1363// Stage-global MusicEngine, constructed lazily on the first music/sfx cue —
1364// a script that never uses either never allocates it (or its AudioContext).
1365let musicEngine = null;
1366function ensureMusicEngine() { return musicEngine || (musicEngine = new MusicEngine()); }
1367
1368function mainPuppet() {
1369 const f = stage.frames.get('main');
1370 return f && f.puppet;
1371}
1372
1373// Character direction (speak/action/walk/look/view) targets an actor's
1374// Puppet when the cue carries `actor`, else the frame's primary — same
1375// resolution order the parser applies (frame id, then actor id, then
1376// character name). Belt-and-braces: normally the parser has already
1377// resolved a character name to a real frame/actor id, but if a cue arrives
1378// with an id that doesn't resolve, fall back to matching it as a character
1379// name across every frame's primary and actors.
1380function puppetForCue(cue) {
1381 if (cue.actor) {
1382 const a = stage.resolveActor(cue);
1383 if (a) return a.puppet;
1384 return stage.findPuppetByCharacterName(cue.actor);
1385 }
1386 if (cue.frame && !stage.frames.has(cue.frame)) return stage.findPuppetByCharacterName(cue.frame);
1387 return stage.forFrame(cue);
1388}
1389
1390function handleCue(cue) {
1391 switch (cue.type) {
1392 case 'frame': {
1393 const f = stage.frame(cue);
1394 if (cue.character) loadCharacter(cue.character, f.id, { view: cue.view, facing: cue.facing });
1395 syncFrameTargets();
1396 break;
1397 }
1398 case 'frame-clear': stage.frameClear(cue); syncFrameTargets(); syncActorChips(); break;
1399 case 'layout': stage.layout(cue); syncFrameTargets(); syncActorChips(); break;
1400 case 'content': stage.content(cue); break;
1401 case 'content-clear': stage.contentClear(cue); break;
1402 case 'scene': stage.scene(cue); break;
1403 // place resolves whether `what` is a character or a prop asynchronously
1404 // (see Stage.placeActor), so the actor isn't in f.actors until that
1405 // settles — sync the chip row off the returned promise rather than
1406 // immediately, or a fast click would render the row one place behind.
1407 case 'place': stage.place(cue).then(syncActorChips); break;
1408 case 'remove': stage.removeActor(cue); syncActorChips(); break;
1409 case 'move': { const a = stage.resolveActor(cue); if (a) stage.moveActor(a, cue.x); break; }
1410 case 'scale': { const a = stage.resolveActor(cue); if (a) stage.scaleActor(a, cue.value); break; }
1411 case 'spin': { const a = stage.resolveActor(cue); if (a) stage.spinActor(a); break; }
1412 case 'bounce': { const a = stage.resolveActor(cue); if (a) stage.bounceActor(a); break; }
1413 case 'wear': stage.wear(cue); break;
1414 case 'unwear': stage.unwear(cue); break;
1415 case 'music': ensureMusicEngine().music(cue); break;
1416 case 'sfx': ensureMusicEngine().sfx(cue); break;
1417 case 'overlay': stage.overlay(cue); break;
1418 case 'overlay-clear': stage.overlayClear(cue); break;
1419 case 'captions': stage.captions(cue); break;
1420 case 'transition': stage.transition(cue); break;
1421 case 'speak': {
1422 const p = puppetForCue(cue);
1423 if (p) p.speak(cue);
1424 stage.showCaption(cue);
1425 // CONVERSATION MODE: if this speak cue is the reply we've been
1426 // waiting on (see sendConverse below), drop the "thinking" look pose
1427 // and flip the status line to speaking -> idle once the clip ends.
1428 if (awaitingConverseReply && p && p === converseThinkPuppet) {
1429 awaitingConverseReply = false;
1430 converseThinkPuppet = null;
1431 p.look('front');
1432 setConverseStatus('speaking…');
1433 clearTimeout(converseStatusTimer);
1434 converseStatusTimer = setTimeout(() => setConverseStatus('idle'), (cue.duration || 0) * 1000 + 300);
1435 }
1436 break;
1437 }
1438 // CONVERSATION MODE: the human's own transcribed/typed words, broadcast
1439 // by POST /api/converse purely for display — like a delayed CC bar.
1440 case 'user-said': showUserCaption(cue.text); break;
1441 case 'action': { const p = puppetForCue(cue); if (p) p.act(cue.name); break; }
1442 case 'walk': { const p = puppetForCue(cue); if (p) p.walkTo(cue.x, cue.jump || cue.from); break; }
1443 case 'look': { const p = puppetForCue(cue); if (p) p.look(cue.dir); break; }
1444 case 'view': { const p = puppetForCue(cue); if (p) p.setView(cue.mode); break; }
1445 case 'character': {
1446 const id = cue.frame || stage.activeId;
1447 if (cue.frame) stage.activeId = id;
1448 loadCharacter(cue.name, id);
1449 break;
1450 }
1451 case 'script-start':
1452 stage.clearAll();
1453 prompterStart(cue.lines);
1454 syncActorChips();
1455 break;
1456 case 'script-line': prompterHighlight(cue.index); break;
1457 case 'script-end': prompterEnd(); break;
1458 case 'clear': stage.clearAll(); syncActorChips(); break;
1459 case 'error': toast(cue.message); break;
1460 }
1461}
1462
1463const events = new EventSource('/api/events');
1464events.onmessage = (e) => handleCue(JSON.parse(e.data));
1465
1466// surface runtime errors on the stage itself — invaluable when a cue breaks
1467window.addEventListener('error', (e) => toast(`js error: ${e.message}`));
1468window.addEventListener('unhandledrejection', (e) =>
1469 toast(`js error: ${(e.reason && e.reason.message) || e.reason}`));
1470
1471// Any user gesture unlocks/keeps-alive every puppet's audio context, so
1472// speech cues arriving later over SSE are allowed to make sound (Safari
1473// autoplay policy). Safari additionally keeps a context inaudible until a
1474// source node is *started* inside a real gesture, so play one silent sample
1475// per context too.
1476const unlockedCtxs = new WeakSet();
1477function unlockAudio(p) {
1478 if (!p) return;
1479 const ctx = p.ensureCtx();
1480 if (ctx.state !== 'running') ctx.resume().catch(() => {});
1481 if (!unlockedCtxs.has(ctx)) {
1482 try {
1483 const src = ctx.createBufferSource();
1484 src.buffer = ctx.createBuffer(1, 1, 22050);
1485 src.connect(ctx.destination);
1486 src.start(0);
1487 unlockedCtxs.add(ctx);
1488 } catch { /* retry on next gesture */ }
1489 }
1490}
1491for (const ev of ['pointerdown', 'keydown']) {
1492 window.addEventListener(ev, () => {
1493 for (const f of stage.frames.values()) {
1494 if (f.puppet) unlockAudio(f.puppet);
1495 for (const a of f.actors.values()) if (a.puppet) unlockAudio(a.puppet);
1496 }
1497 if (musicEngine) musicEngine.unlock();
1498 }, true);
1499}
1500
1501// Puppet height is proportional to its frame's height (see
1502// Puppet.applyScale), so a browser resize can change every frame's pixel
1503// size at once even though no rect ever changed. Lightly debounced since
1504// 'resize' can fire in a burst while the window is being dragged.
1505let resizeTimer = null;
1506window.addEventListener('resize', () => {
1507 clearTimeout(resizeTimer);
1508 resizeTimer = setTimeout(() => stage.rescaleAll(), 100);
1509});
1510
1511// --------------------------------------------------------- teleprompter
1512
1513const prompter = document.getElementById('prompter');
1514
1515function prompterStart(lines) {
1516 prompter.innerHTML = '';
1517 for (const line of lines) {
1518 const div = document.createElement('div');
1519 div.className = 'p-line';
1520 div.textContent = line;
1521 prompter.appendChild(div);
1522 }
1523 prompter.classList.add('active');
1524}
1525
1526function prompterHighlight(i) {
1527 const lines = prompter.children;
1528 for (let j = 0; j < lines.length; j++) lines[j].classList.toggle('now', j === i);
1529 if (lines[i]) lines[i].scrollIntoView({ block: 'nearest', behavior: 'smooth' });
1530}
1531
1532function prompterEnd() {
1533 setTimeout(() => prompter.classList.remove('active'), 1500);
1534}
1535
1536function toast(msg) {
1537 const t = document.getElementById('toast');
1538 t.textContent = msg;
1539 t.classList.add('show');
1540 setTimeout(() => t.classList.remove('show'), 4000);
1541}
1542
1543// ------------------------------------------------------------- controls
1544//
1545// With one frame the panel targets it exactly as the single-puppet stage
1546// did before frames existed. Once a screenplay (or the layout buttons)
1547// splits the stage into more than one frame, a chip row (#frame-target)
1548// lets the user pick which frame every panel-originated cue targets;
1549// `targetFrame` is the source of truth and defaults to 'main'.
1550
1551let targetFrame = 'main';
1552
1553// Character folder names, fetched once at boot; reused by the Cast & Props
1554// panel to tell "characters" from "props" in the [place] optgroups without
1555// re-probing each asset the way Stage.probeCharacter does per placement.
1556let allCharacters = [];
1557
1558// -------------------------------------------------------------- record mode
1559//
1560// When active, every PANEL-initiated action (not incoming SSE cues, and
1561// never Run/Stop/Load) appends its screenplay-grammar equivalent to the
1562// #script textarea, so a user can drive the puppet by hand and end up with a
1563// runnable script. `chipsVisible()` mirrors the same single/multi-frame
1564// branch every other panel control already uses to decide whether to target
1565// `targetFrame` explicitly.
1566
1567let recording = false;
1568
1569// Auto-timing: while recording, a gap of more than 1.5s between one
1570// recorded line and the next gets a synthetic `[wait <gap>]` inserted ahead
1571// of it, so a hand-driven take reproduces roughly the same pacing when
1572// replayed. `lastRecordTs` is null right after recording starts (and after
1573// each toggle-on) so nothing is ever inserted before a session's first line.
1574let lastRecordTs = null;
1575const AUTO_WAIT_GAP_S = 1.5; // gaps shorter than this aren't worth a cue
1576const AUTO_WAIT_ROUND = 0.5; // rounded to the nearest half-second
1577const AUTO_WAIT_CAP_S = 5; // long pauses (e.g. stepping away) cap here
1578
1579function chipsVisible() {
1580 return $id('frame-target').style.display !== 'none';
1581}
1582
1583function appendScriptLine(line) {
1584 const ta = $id('script');
1585 ta.value = ta.value ? ta.value + '\n' + line : line;
1586 ta.scrollTop = ta.scrollHeight;
1587}
1588
1589function record(line) {
1590 if (!recording) return;
1591 const now = Date.now();
1592 if (lastRecordTs !== null) {
1593 const gapS = (now - lastRecordTs) / 1000;
1594 if (gapS > AUTO_WAIT_GAP_S) {
1595 const rounded = Math.min(AUTO_WAIT_CAP_S, Math.round(gapS / AUTO_WAIT_ROUND) * AUTO_WAIT_ROUND);
1596 appendScriptLine(`[wait ${rounded}]`);
1597 }
1598 }
1599 lastRecordTs = now;
1600 appendScriptLine(line);
1601}
1602
1603async function post(url, body) {
1604 const r = await fetch(url, {
1605 method: 'POST',
1606 headers: { 'Content-Type': 'application/json' },
1607 body: JSON.stringify(body),
1608 });
1609 if (!r.ok) toast((await r.json()).error || 'request failed');
1610}
1611
1612$id('say-form').addEventListener('submit', (e) => {
1613 e.preventDefault();
1614 const text = $id('say-text').value.trim();
1615 if (!text) return;
1616 post('/api/say', {
1617 text,
1618 engine: $id('engine').value,
1619 voice: $id('voice').value,
1620 frame: targetFrame,
1621 });
1622 record(chipsVisible() ? `${targetFrame}: ${text}` : text);
1623});
1624
1625$id('engine').addEventListener('change', (e) => {
1626 loadVoices();
1627 record(`[engine ${e.target.value}]`);
1628});
1629$id('voice').addEventListener('change', (e) => {
1630 record(`[voice ${e.target.value}]`);
1631});
1632
1633// audio diagnostics: state readout + a beep through the same output path
1634$id('beep').onclick = async () => {
1635 const ctx = mainPuppet().ensureCtx();
1636 await ctx.resume().catch(() => {});
1637 const osc = ctx.createOscillator();
1638 const gain = ctx.createGain();
1639 gain.gain.value = 0.25;
1640 osc.frequency.value = 440;
1641 osc.connect(gain);
1642 gain.connect(ctx.destination);
1643 osc.start();
1644 osc.stop(ctx.currentTime + 0.4);
1645};
1646setInterval(() => {
1647 const p = mainPuppet();
1648 const c = p && p.audioCtx;
1649 $id('audio-state').textContent = 'audio: ' + (c ? c.state : 'not started yet');
1650}, 500);
1651
1652async function loadVoices() {
1653 const engine = $id('engine').value;
1654 const sel = $id('voice');
1655 sel.innerHTML = '<option value="">default voice</option>';
1656 try {
1657 const { voices } = await (await fetch(`/api/voices?engine=${engine}`)).json();
1658 for (const v of voices) {
1659 const o = document.createElement('option');
1660 o.value = o.textContent = v;
1661 sel.appendChild(o);
1662 }
1663 } catch { /* voice list is a nicety */ }
1664}
1665
1666function targetPuppet() {
1667 const f = stage.frames.get(targetFrame);
1668 return f && f.puppet;
1669}
1670
1671function buildActionButtons() {
1672 const box = $id('actions');
1673 box.innerHTML = '';
1674 const p = targetPuppet();
1675 for (const name of Object.keys((p && p.manifest || {}).actions || {})) {
1676 const b = document.createElement('button');
1677 b.textContent = name;
1678 b.onclick = () => {
1679 post('/api/cue', { type: 'action', name, frame: targetFrame });
1680 record(chipsVisible() ? `[${targetFrame} ${name}]` : `[${name}]`);
1681 };
1682 box.appendChild(b);
1683 }
1684 for (const dir of ['left', 'front', 'right']) {
1685 const b = document.createElement('button');
1686 b.textContent = `look ${dir}`;
1687 b.onclick = () => {
1688 post('/api/cue', { type: 'look', dir, frame: targetFrame });
1689 record(chipsVisible() ? `[${targetFrame} look ${dir}]` : `[look ${dir}]`);
1690 };
1691 box.appendChild(b);
1692 }
1693 for (const mode of ['face', 'body']) {
1694 const b = document.createElement('button');
1695 b.textContent = `🎥 ${mode}`;
1696 b.onclick = () => {
1697 post('/api/cue', { type: 'view', mode, frame: targetFrame });
1698 record(chipsVisible() ? `[${targetFrame} view ${mode}]` : `[view ${mode}]`);
1699 };
1700 box.appendChild(b);
1701 }
1702}
1703
1704$id('walk').addEventListener('change', (e) => {
1705 const x = parseInt(e.target.value, 10);
1706 post('/api/cue', { type: 'walk', x, frame: targetFrame });
1707 record(chipsVisible() ? `[${targetFrame} walk ${x}]` : `[walk to ${x}]`);
1708});
1709
1710// -------------------------------------------------------------- scene
1711
1712for (const btn of document.querySelectorAll('#layout-buttons button')) {
1713 btn.onclick = () => {
1714 post('/api/cue', { type: 'layout', preset: btn.dataset.preset });
1715 record(`[layout ${btn.dataset.preset}]`);
1716 };
1717}
1718
1719$id('scene-bg').addEventListener('change', (e) => {
1720 const bg = e.target.value;
1721 if (!bg) return;
1722 post('/api/cue', { type: 'scene', bg }); // targets the active frame server-side
1723 record(chipsVisible() ? `[${targetFrame} scene ${bg}]` : `[scene ${bg}]`);
1724 e.target.value = ''; // picker, not a state display
1725});
1726
1727let captionsOn = false;
1728$id('captions-toggle').onclick = (e) => {
1729 captionsOn = !captionsOn;
1730 post('/api/cue', { type: 'captions', on: captionsOn });
1731 e.target.textContent = captionsOn ? 'CC on' : 'CC off';
1732 e.target.classList.toggle('on', captionsOn);
1733 record(captionsOn ? '[captions on]' : '[captions off]');
1734};
1735
1736for (const btn of document.querySelectorAll('[data-transition-name]')) {
1737 btn.onclick = () => {
1738 post('/api/cue', {
1739 type: 'transition', name: btn.dataset.transitionName, dir: btn.dataset.transitionDir,
1740 });
1741 record(`[${btn.dataset.transitionName} ${btn.dataset.transitionDir}]`);
1742 };
1743}
1744
1745async function loadAssets() {
1746 try {
1747 const { backgrounds, props, music, sfx } = await (await fetch('/api/assets')).json();
1748 const sel = $id('scene-bg');
1749 if (!backgrounds || !backgrounds.length) { sel.style.display = 'none'; }
1750 for (const id of backgrounds || []) {
1751 const o = document.createElement('option');
1752 o.value = o.textContent = id;
1753 sel.appendChild(o);
1754 }
1755
1756 buildCastPropsControls(props || []);
1757
1758 const musicSection = $id('music-section');
1759 if ((!music || !music.length) && (!sfx || !sfx.length)) { musicSection.style.display = 'none'; return; }
1760 musicSection.style.display = '';
1761
1762 const musicSel = $id('music-select');
1763 for (const id of music || []) {
1764 const o = document.createElement('option');
1765 o.value = o.textContent = id;
1766 musicSel.appendChild(o);
1767 }
1768
1769 const sfxBox = $id('sfx-buttons');
1770 for (const id of sfx || []) {
1771 const b = document.createElement('button');
1772 b.type = 'button';
1773 b.textContent = `🔔 ${id}`;
1774 b.onclick = () => {
1775 post('/api/cue', { type: 'sfx', id });
1776 record(`[sfx ${id}]`);
1777 };
1778 sfxBox.appendChild(b);
1779 }
1780 } catch { /* asset list is a nicety */ }
1781}
1782
1783// ------------------------------------------------------------ cast & props
1784//
1785// Mirrors dbbasic parity: the whole section stays hidden when there are no
1786// props to place/wear (an asset pack with no props leaves nothing useful to
1787// do here beyond placing a bare character, which isn't worth the UI). The
1788// "characters" optgroup reuses the boot-time character list (see init());
1789// the "props" optgroup and the Wear select are both the props id list from
1790// /api/assets, which loadAssets() also uses for the Music section.
1791
1792function buildCastPropsControls(props) {
1793 const section = $id('cast-section');
1794 if (!props.length) { section.style.display = 'none'; return; }
1795 section.style.display = '';
1796
1797 const placeSel = $id('place-what');
1798 placeSel.innerHTML = '';
1799 const charGroup = document.createElement('optgroup');
1800 charGroup.label = 'characters';
1801 for (const name of allCharacters) {
1802 const o = document.createElement('option');
1803 o.value = o.textContent = name;
1804 charGroup.appendChild(o);
1805 }
1806 placeSel.appendChild(charGroup);
1807 const propGroup = document.createElement('optgroup');
1808 propGroup.label = 'props';
1809 for (const id of props) {
1810 const o = document.createElement('option');
1811 o.value = o.textContent = id;
1812 propGroup.appendChild(o);
1813 }
1814 placeSel.appendChild(propGroup);
1815
1816 const wearSel = $id('wear-what');
1817 wearSel.innerHTML = '';
1818 for (const id of props) {
1819 const o = document.createElement('option');
1820 o.value = o.textContent = id;
1821 wearSel.appendChild(o);
1822 }
1823}
1824
1825$id('place-btn').onclick = () => {
1826 const what = $id('place-what').value;
1827 if (!what) return;
1828 const x = Math.min(99, Math.max(1, parseInt($id('place-x').value, 10) || 50));
1829 const isCharacter = allCharacters.includes(what);
1830 const defaultScale = isCharacter ? 1 : 0.4; // matches Stage.placeActor's own default
1831 post('/api/cue', { type: 'place', id: what, what, x, scale: defaultScale, frame: targetFrame });
1832 // this row never offers a non-default scale, so the scale: key is always omitted
1833 record(chipsVisible()
1834 ? `[${targetFrame} place ${what} at ${x}]`
1835 : `[place ${what} at ${x}]`);
1836};
1837
1838$id('wear-btn').onclick = () => {
1839 const prop = $id('wear-what').value;
1840 if (!prop) return;
1841 post('/api/cue', { type: 'wear', target: targetFrame, prop });
1842 record(`[wear ${targetFrame} ${prop}]`);
1843};
1844
1845$id('unwear-btn').onclick = () => {
1846 post('/api/cue', { type: 'unwear', target: targetFrame });
1847 record(`[unwear ${targetFrame}]`);
1848};
1849
1850// Placed-actor chips: every actor currently placed across every frame,
1851// rebuilt whenever a cue could have changed that set (see handleCue). Kept
1852// hidden when empty, same convention as #frame-target.
1853function syncActorChips() {
1854 const row = $id('actor-chips');
1855 const ids = [];
1856 for (const f of stage.frames.values()) for (const id of f.actors.keys()) ids.push(id);
1857 row.innerHTML = '';
1858 if (!ids.length) { row.style.display = 'none'; return; }
1859 row.style.display = '';
1860 for (const id of ids) {
1861 const b = document.createElement('button');
1862 b.type = 'button';
1863 b.textContent = `✕ ${id}`;
1864 b.onclick = () => {
1865 post('/api/cue', { type: 'remove', id });
1866 record(`[remove ${id}]`);
1867 };
1868 row.appendChild(b);
1869 }
1870}
1871
1872$id('music-select').addEventListener('change', (e) => {
1873 const id = e.target.value;
1874 if (!id) return;
1875 post('/api/cue', { type: 'music', id });
1876 record(`[music ${id}]`);
1877 e.target.value = ''; // picker, not a state display
1878});
1879
1880$id('music-off').onclick = () => {
1881 post('/api/cue', { type: 'music', off: true });
1882 record('[music off]');
1883};
1884
1885// ---------------------------------------------------------------- examples
1886
1887async function loadExamples() {
1888 try {
1889 const { examples } = await (await fetch('/api/examples')).json();
1890 const row = $id('example-row');
1891 if (!examples || !examples.length) { row.style.display = 'none'; return; }
1892 const sel = $id('example');
1893 for (const name of examples) {
1894 const o = document.createElement('option');
1895 o.value = o.textContent = name;
1896 sel.appendChild(o);
1897 }
1898 } catch { /* example list is a nicety */ }
1899}
1900
1901$id('load-example').onclick = async () => {
1902 const name = $id('example').value;
1903 if (!name) return;
1904 try {
1905 const { script } = await (await fetch(`/api/examples/${encodeURIComponent(name)}`)).json();
1906 $id('script').value = script;
1907 } catch { toast('could not load example'); }
1908};
1909
1910$id('run-script').onclick = () => {
1911 const p = mainPuppet();
1912 post('/api/script', { script: $id('script').value, mainCharacter: p && p.characterName });
1913};
1914$id('stop-script').onclick = () => post('/api/script/stop', {});
1915
1916$id('record-toggle').onclick = (e) => {
1917 recording = !recording;
1918 if (recording) lastRecordTs = null; // fresh session: never auto-wait before its first line
1919 e.target.textContent = recording ? '● Recording' : '● Record';
1920 e.target.classList.toggle('rec-on', recording);
1921 $id('wait-btn').disabled = !recording;
1922};
1923
1924$id('wait-btn').onclick = () => record('[wait 1]');
1925
1926$id('help-toggle').onclick = () => $id('help-panel').classList.toggle('active');
1927$id('help-close').onclick = () => $id('help-panel').classList.remove('active');
1928
1929// Applies a puppet's manifest-declared voice preference (engine + voice) to
1930// the engine/voice selects, refreshing the voice list in between so the
1931// preferred voice is actually a valid option by the time it's set. Shared
1932// between loadCharacter and the frame-target chip click handler.
1933async function applyVoicePref(puppet) {
1934 const pref = puppet && puppet.manifest && puppet.manifest.voice;
1935 if (!pref) return;
1936 if (pref.engine) $id('engine').value = pref.engine;
1937 await loadVoices();
1938 if (pref.voice) $id('voice').value = pref.voice;
1939}
1940
1941async function loadCharacter(name, frameId = 'main', extra = {}) {
1942 const f = stage.ensureFrame(frameId);
1943 await stage.loadCharacterInto(f, name);
1944 if (extra.view) f.puppet.setView(extra.view, true);
1945 if (extra.facing !== undefined) f.puppet.face(extra.facing);
1946 if (frameId !== targetFrame) return;
1947
1948 buildActionButtons();
1949 const sel = $id('character');
1950 if (sel.value !== name) sel.value = name;
1951 await applyVoicePref(f.puppet);
1952 updateConverseNote();
1953}
1954
1955$id('character').addEventListener('change', (e) => {
1956 post('/api/cue', { type: 'character', name: e.target.value, frame: targetFrame });
1957 record(`[frame ${targetFrame} character:${e.target.value}]`);
1958});
1959
1960// -------------------------------------------------------- frame targeting
1961//
1962// Only shown once a screenplay (or the layout buttons) creates more than
1963// one frame; with a single frame it stays hidden and targetFrame tracks
1964// that lone frame's id, exactly like the pre-frames single-puppet stage.
1965
1966function syncFrameTargets() {
1967 const row = $id('frame-target');
1968 const ids = [...stage.frames.keys()];
1969 if (!stage.frames.has(targetFrame)) {
1970 targetFrame = stage.frames.has(stage.activeId) ? stage.activeId : (ids[0] || 'main');
1971 }
1972 if (ids.length <= 1) {
1973 targetFrame = ids[0] || 'main';
1974 row.style.display = 'none';
1975 row.innerHTML = '';
1976 return;
1977 }
1978 row.style.display = '';
1979 row.innerHTML = '';
1980 for (const id of ids) {
1981 const b = document.createElement('button');
1982 b.type = 'button';
1983 b.textContent = id;
1984 if (id === targetFrame) b.classList.add('sel');
1985 b.onclick = () => selectTargetFrame(id);
1986 row.appendChild(b);
1987 }
1988}
1989
1990async function selectTargetFrame(id) {
1991 targetFrame = id;
1992 syncFrameTargets();
1993 const f = stage.frames.get(id);
1994 const p = f && f.puppet;
1995 const sel = $id('character');
1996 sel.value = (p && p.characterName) || ''; // silent: no change event dispatched
1997 buildActionButtons();
1998 await applyVoicePref(p);
1999 updateConverseNote();
2000}
2001
2002// ------------------------------------------------------ conversation mode
2003//
2004// Push-to-talk (pointer-held mic button, held Space, or a typed fallback) ->
2005// POST /api/transcribe (mic path only) -> POST /api/converse. The server
2006// does all the cue broadcasting itself (user-said caption, optional action,
2007// the spoken reply) over the existing SSE bus — see the 'user-said'/'speak'
2008// handling in handleCue above. A random per-tab session id threads turns
2009// into one conversation; "new conversation" mints a fresh one. The panel
2010// hides entirely when GET /api/converse reports no key configured; the mic
2011// button alone (not the whole section) dims out if getUserMedia is
2012// unavailable/denied, since the typed fallback still works with no
2013// microphone at all.
2014
2015let converseSessionId = null;
2016let converseMicOk = true;
2017let pttStream = null;
2018let pttRecorder = null;
2019let pttChunks = [];
2020let pttActive = false;
2021let awaitingConverseReply = false; // true from POST /api/converse until its matching speak cue arrives
2022let converseThinkPuppet = null; // puppet holding the "look up and think" idle pose while we wait
2023let converseStatusTimer = null;
2024let userCaptionTimer = null;
2025
2026function newConverseSession() {
2027 converseSessionId = Math.random().toString(36).slice(2) + Date.now().toString(36);
2028}
2029
2030function setConverseStatus(text) {
2031 const el = $id('converse-status');
2032 if (el) el.textContent = text;
2033}
2034
2035function updateConverseNote() {
2036 const el = $id('converse-note');
2037 if (!el) return;
2038 const p = targetPuppet();
2039 el.textContent = p && p.characterName ? `talking with ${p.characterName}` : 'no character loaded';
2040}
2041
2042// Delayed-CC display for the human's own words (see #user-caption in
2043// index.html) — visually distinct from the character's #captions bar
2044// (dimmer, top-center, "you:" prefix). Auto-hides after max(2.5s,
2045// words*0.35s), mirroring the pacing #captions gets from cue.duration.
2046function showUserCaption(text) {
2047 const el = $id('user-caption');
2048 if (!el) return;
2049 clearTimeout(userCaptionTimer);
2050 el.innerHTML = '';
2051 const prefix = document.createElement('span');
2052 prefix.className = 'you-prefix';
2053 prefix.textContent = 'you:';
2054 el.appendChild(prefix);
2055 el.appendChild(document.createTextNode(' ' + text));
2056 el.classList.add('shown');
2057 const words = text.trim().split(/\s+/).filter(Boolean).length;
2058 const ms = Math.max(2500, words * 350);
2059 userCaptionTimer = setTimeout(() => el.classList.remove('shown'), ms);
2060}
2061
2062function pickRecorderMime() {
2063 if (!window.MediaRecorder || !MediaRecorder.isTypeSupported) return '';
2064 // Opus/webm first (Chrome/Firefox default); mp4/m4a (AAC) is Safari's
2065 // supported container — MediaRecorder on Safari doesn't do webm at all.
2066 for (const c of ['audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', 'audio/mp4;codecs=mp4a.40.2', 'audio/aac']) {
2067 if (MediaRecorder.isTypeSupported(c)) return c;
2068 }
2069 return '';
2070}
2071
2072async function startPTT() {
2073 if (pttActive || !converseMicOk) return;
2074 if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
2075 converseMicOk = false;
2076 $id('ptt-btn').disabled = true;
2077 toast('microphone not available — use the text box instead');
2078 return;
2079 }
2080 try {
2081 pttStream = await navigator.mediaDevices.getUserMedia({ audio: true });
2082 } catch (e) {
2083 converseMicOk = false;
2084 $id('ptt-btn').disabled = true;
2085 toast('mic permission denied — use the text box instead');
2086 return;
2087 }
2088 const mime = pickRecorderMime();
2089 try {
2090 pttRecorder = mime ? new MediaRecorder(pttStream, { mimeType: mime }) : new MediaRecorder(pttStream);
2091 } catch (e) {
2092 toast('could not start recorder: ' + e.message);
2093 pttStream.getTracks().forEach((t) => t.stop());
2094 pttStream = null;
2095 return;
2096 }
2097 pttChunks = [];
2098 pttRecorder.ondataavailable = (e) => { if (e.data && e.data.size) pttChunks.push(e.data); };
2099 pttRecorder.start();
2100 pttActive = true;
2101 $id('ptt-btn').classList.add('listening');
2102 setConverseStatus('listening…');
2103}
2104
2105function stopPTT() {
2106 if (!pttActive || !pttRecorder) return;
2107 pttActive = false;
2108 $id('ptt-btn').classList.remove('listening');
2109 const recorder = pttRecorder;
2110 pttRecorder = null;
2111 const mimeType = recorder.mimeType || 'audio/webm';
2112 recorder.onstop = async () => {
2113 if (pttStream) { pttStream.getTracks().forEach((t) => t.stop()); pttStream = null; }
2114 const blob = new Blob(pttChunks, { type: mimeType });
2115 pttChunks = [];
2116 if (!blob.size) { setConverseStatus('idle'); return; }
2117 setConverseStatus('thinking…');
2118 let text;
2119 try {
2120 const r = await fetch('/api/transcribe', { method: 'POST', headers: { 'Content-Type': mimeType }, body: blob });
2121 const data = await r.json();
2122 if (!r.ok) throw new Error(data.error || 'transcribe failed');
2123 text = (data.text || '').trim();
2124 } catch (e) {
2125 toast('transcribe failed: ' + e.message);
2126 setConverseStatus('idle');
2127 return;
2128 }
2129 if (!text) { toast('heard nothing'); setConverseStatus('idle'); return; }
2130 sendConverse(text);
2131 };
2132 try { recorder.stop(); } catch { setConverseStatus('idle'); }
2133}
2134
2135// Shared by both input paths (a mic transcript or the typed fallback): posts
2136// one turn to /api/converse. The server broadcasts the user-said/action/
2137// speak cues itself over SSE; this only drives local "thinking" presence
2138// (look up while waiting; see handleCue's 'speak' case for the reply-arrival
2139// half) and the status line.
2140async function sendConverse(text) {
2141 const puppet = targetPuppet();
2142 if (!converseSessionId) newConverseSession();
2143 setConverseStatus('thinking…');
2144 converseThinkPuppet = puppet;
2145 if (puppet) puppet.look('up');
2146 awaitingConverseReply = true;
2147 try {
2148 const r = await fetch('/api/converse', {
2149 method: 'POST',
2150 headers: { 'Content-Type': 'application/json' },
2151 body: JSON.stringify({
2152 text, character: puppet && puppet.characterName,
2153 session: converseSessionId, frame: targetFrame,
2154 }),
2155 });
2156 const data = await r.json();
2157 if (!r.ok) throw new Error(data.error || 'converse failed');
2158 } catch (e) {
2159 awaitingConverseReply = false;
2160 converseThinkPuppet = null;
2161 if (puppet) puppet.look('front');
2162 toast('converse failed: ' + e.message);
2163 setConverseStatus('idle');
2164 }
2165}
2166
2167const pttBtn = $id('ptt-btn');
2168pttBtn.addEventListener('pointerdown', (e) => { e.preventDefault(); startPTT(); });
2169pttBtn.addEventListener('pointerup', stopPTT);
2170pttBtn.addEventListener('pointerleave', stopPTT);
2171pttBtn.addEventListener('pointercancel', stopPTT);
2172
2173// Spacebar doubles as push-to-talk, but only outside text fields (typing a
2174// space in the Say box or the screenplay editor must never trigger it), and
2175// never on an auto-repeat keydown (holding Space would otherwise re-fire
2176// startPTT on every OS repeat interval).
2177function inTextField() {
2178 const el = document.activeElement;
2179 if (!el) return false;
2180 const tag = el.tagName;
2181 return tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT' || el.isContentEditable;
2182}
2183window.addEventListener('keydown', (e) => {
2184 if (e.code !== 'Space' || e.repeat || inTextField()) return;
2185 e.preventDefault();
2186 startPTT();
2187});
2188window.addEventListener('keyup', (e) => {
2189 if (e.code !== 'Space') return;
2190 stopPTT();
2191});
2192
2193// Typed fallback: skips /api/transcribe entirely, so conversation mode
2194// works with no microphone (or no permission) at all.
2195function submitTypedConverse() {
2196 const input = $id('converse-text');
2197 const text = input.value.trim();
2198 if (!text) return;
2199 input.value = '';
2200 sendConverse(text);
2201}
2202$id('converse-send').onclick = submitTypedConverse;
2203$id('converse-text').addEventListener('keydown', (e) => {
2204 if (e.key === 'Enter') { e.preventDefault(); submitTypedConverse(); }
2205});
2206
2207$id('converse-reset').onclick = () => {
2208 newConverseSession();
2209 toast('new conversation');
2210};
2211
2212// Hides the whole section when the server has no OpenAI key configured
2213// (GET /api/converse -> {ready:false}), including when talking to the
2214// dbbasic package's shim (its fetch bridge passes unknown /api/* through to
2215// the object server, which 404s -> caught below -> hidden, same as a
2216// same-origin fetch failure).
2217async function initConverse() {
2218 const section = $id('converse-section');
2219 try {
2220 const { ready } = await (await fetch('/api/converse')).json();
2221 if (!ready) { section.style.display = 'none'; return; }
2222 } catch {
2223 section.style.display = 'none';
2224 return;
2225 }
2226 section.style.display = '';
2227 updateConverseNote();
2228}
2229
2230// ---------------------------------------------------------------- boot
2231
2232(async function init() {
2233 syncFrameTargets(); // boots as one frame ('main'): chip row starts hidden
2234 const { characters } = await (await fetch('/api/characters')).json();
2235 allCharacters = characters;
2236 const sel = $id('character');
2237 for (const c of characters) {
2238 const o = document.createElement('option');
2239 o.value = o.textContent = c;
2240 sel.appendChild(o);
2241 }
2242 await loadCharacter(characters[0], 'main');
2243 if (!mainPuppet().manifest.voice) loadVoices();
2244 loadAssets();
2245 loadExamples();
2246 initConverse();
2247 $id('script').value = [
2248 '# Try an example from the picker above, or run this:',
2249 '[captions on]',
2250 '[fade in 800]',
2251 'Hello! Welcome to the puppet stage.',
2252 '(wave) Every line here is a cue.',
2253 '[view face]',
2254 'The camera can come in close…',
2255 '[view body] [walk to 65]',
2256 '…and I can move around the stage.',
2257 '(bow) Load an example above to see a real show.',
2258 '[iris out 1000]',
2259 ].join('\n');
2260})();