diff --git a/docs/README.md b/docs/README.md index 277d6b3..f4d37ba 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,3 +8,4 @@ that don't show. Start with the [root README](../README.md) for the template itself. +- [workers.md](workers.md): run your Python in Web Workers when the main thread has to stay free (Web MIDI, live audio) diff --git a/docs/workers.md b/docs/workers.md new file mode 100644 index 0000000..62585a2 --- /dev/null +++ b/docs/workers.md @@ -0,0 +1,78 @@ +# Python in Web Workers + +By default the template runs your Python on the page's main thread, which +is simple and fine for most apps. When your app redraws a lot *and* has to +react on time, to Web MIDI, live audio or fast input, a long redraw +on the main thread delays everything else. The fix is to run your Python in +Web Workers and keep the page thin. `workers/` is a small kit for that: + +| File | Runs on | Does | +|---|---|---| +| `workers/host.js` | the page | starts workers, hands a canvas to one of them and relays its input, relays Web MIDI, owns the `AudioContext` | +| `workers/python_worker.js` | each worker | loads Pyodide, installs your wheels, copies your files in, runs your module | +| `workers/audio_worklet.js` | the audio thread | plays the audio a worker sends, with a queue that adapts to the machine | + +## A page that runs its app in a worker + +```html + + +``` + +In `main.py`, use the worker board config and everything else works as on +the page, including LVGL's `display_driver`: + +```python +import board_config # pydevices' board_configs/workerdisplay, copied in as board_config.py +import appdev +app = appdev.App(board_config) +``` + +`files` maps a URL (relative to the site root) to a path in the worker's +file system; that's how `board_config.py` and `board_peripherals.py` from +`board_configs/workerdisplay` get in. Pointer, wheel and key events on the +canvas reach `appdev` exactly as `PSDisplay` delivers them. + +## Sound + +```js +import { createAudio, resumeOnGesture } from "./workers/host.js"; +const audio = await createAudio({ channels: 1 }); +startWorker({ ..., ports: { audio: audio.workerPort } }); +resumeOnGesture(audio.context); // browsers start audio on the first click or key +``` + +In the worker, `from js import ports` and hand `ports.audio` to +`audiodev.worklet_audio` (pydevices): set `WorkletOutput(ports.audio).source` +to a function that renders blocks, or use its `pcm_out(port=ports.audio)` like +any other audio output. The worklet keeps about 32 ms queued at first and +works down to about 16 ms when the machine keeps up, backing off after any +dropout. + +## Web MIDI + +```js +import { createMidiRelay } from "./workers/host.js"; +const midi = createMidiRelay(); +startWorker({ ..., ports: { midi: midi.port() } }); +``` + +In the worker, `midiports.webmidi.relay_ports(ports.midi)` gives +`adafruit_midi.MIDI` its ports. Every input is relayed with its timestamp, +and what you write goes to the page's outputs. Call `midi.port()` once per +worker that should hear MIDI. + +## Serving it + +Workers load as modules and Pyodide fetches its files, so serve the site +over HTTP (`python -m http.server`), as for the rest of the template. The +service worker caches the kit like any other file once it is listed. diff --git a/workers/audio_worklet.js b/workers/audio_worklet.js new file mode 100644 index 0000000..c3aa491 --- /dev/null +++ b/workers/audio_worklet.js @@ -0,0 +1,158 @@ +// The AudioWorklet end of a Python worker's audio (audiodev.worklet_audio). +// +// It plays 256-frame blocks of int16 PCM the worker sends and asks for more +// over a MessagePort, keeping a queue of `target` frames: +// +// worklet -> worker {need: blocks, asked: frame} send that many blocks +// worker -> worklet {pcm: Int16Array, asked} (transferred, interleaved) +// worklet -> worker {offset} rendered frame k plays at +// context frame k + offset +// worklet -> worker {underrun: count} underruns since the last one +// +// The queue adapts: it starts at `startFrames`, steps down 256 frames at a +// time towards `minFrames` while its low point stays above 512 frames for two +// seconds, and steps up 512 frames (to at most `maxFrames`) on any underrun, +// then holds for ten seconds. The browser's audio thread asks for audio about +// every 10 ms, so a queue much under 768 frames can't keep up. +// +// The node's own port (to the page) answers {type: "stats"} and, when a test +// asks with {type: "tap", on: true}, posts a copy of every frame played. + +const BLOCK = 256; + +class PythonWorkerOutput extends AudioWorkletProcessor { + constructor(options) { + super(); + const o = options.processorOptions || {}; + this.channels = o.channels || 1; + this.minFrames = o.minFrames || 768; + this.maxFrames = o.maxFrames || 4096; + this.target = o.startFrames || 1536; + this.queue = []; + this.head = 0; // sample index into queue[0] + this.queued = 0; // frames queued + this.requested = 0; // frames asked for, not yet here + this.played = 0; // queued frames played so far + this.offset = null; + this.worker = null; + this.started = false; + this.underruns = 0; + this.windowLow = Infinity; + this.windowStart = 0; + this.holdUntil = 0; + this.low = Infinity; // low point since the last stats request + this.tap = null; + this.tapAt = 0; + this.tapStart = 0; + this.port.onmessage = (e) => this.onPage(e); + } + + onPage(e) { + const m = e.data; + if (m.type === "worker") { + this.worker = e.ports[0]; + this.worker.onmessage = (ev) => { + const pcm = ev.data.pcm; + if (!pcm) return; + const frames = pcm.length / this.channels; + this.queue.push(pcm); + this.queued += frames; + this.requested = Math.max(0, this.requested - frames); + }; + this.ask(); + } else if (m.type === "stats") { + this.port.postMessage({ type: "stats", underruns: this.underruns, target: this.target, + queued: this.queued, low: this.low, frame: currentFrame }); + this.low = Infinity; + } else if (m.type === "tap") { + this.tap = m.on ? new Float32Array(Math.floor(sampleRate / 2) * this.channels) : null; + this.tapAt = 0; + } + } + + ask() { + const short = this.target - this.queued - this.requested; + if (short > 0 && this.worker) { + const blocks = Math.ceil(short / BLOCK); + this.requested += blocks * BLOCK; + this.worker.postMessage({ need: blocks, asked: currentFrame }); + } + } + + adapt(underran) { + const now = currentFrame; + if (underran) { + this.target = Math.min(this.maxFrames, this.target + 512); + this.holdUntil = now + 10 * sampleRate; + this.windowStart = now; + this.windowLow = Infinity; + return; + } + if (this.queued < this.windowLow) this.windowLow = this.queued; + if (now - this.windowStart >= 2 * sampleRate) { + if (now >= this.holdUntil && this.windowLow > 512 && this.target > this.minFrames) { + this.target = Math.max(this.minFrames, this.target - BLOCK); + } + this.windowStart = now; + this.windowLow = Infinity; + } + } + + process(inputs, outputs) { + const out = outputs[0]; + const ch = this.channels; + const frames = out[0].length; + let i = 0; + while (i < frames && this.queue.length) { + const blk = this.queue[0]; + const n = Math.min(frames - i, (blk.length - this.head) / ch); + for (let k = 0; k < n; k++) { + for (let c = 0; c < ch; c++) out[c][i + k] = blk[this.head + k * ch + c] / 32768; + } + i += n; + this.head += n * ch; + this.queued -= n; + if (this.head >= blk.length) { this.queue.shift(); this.head = 0; } + } + let underran = false; + if (i < frames) { + for (let c = 0; c < out.length; c++) out[c].fill(0, i); + if (this.started) { + underran = true; + this.underruns++; + if (this.worker) this.worker.postMessage({ underrun: 1 }); + } + } else { + this.started = true; + } + this.played += i; + if (this.started) { + const offset = currentFrame + frames - this.played; + if (offset !== this.offset && this.worker) { + this.offset = offset; + this.worker.postMessage({ offset }); + } + if (this.queued < this.low) this.low = this.queued; + this.adapt(underran); + } + if (this.tap) this.record(out, frames); + this.ask(); + return true; + } + + record(out, frames) { + if (this.tapAt === 0) this.tapStart = currentFrame; + const ch = this.channels; + for (let k = 0; k < frames; k++) { + for (let c = 0; c < ch; c++) this.tap[(this.tapAt + k) * ch + c] = out[c][k]; + } + this.tapAt += frames; + if ((this.tapAt + frames) * ch > this.tap.length) { + const chunk = this.tap.slice(0, this.tapAt * ch); + this.port.postMessage({ type: "tap", start: this.tapStart, channels: ch, pcm: chunk }, [chunk.buffer]); + this.tapAt = 0; + } + } +} + +registerProcessor("python-worker-output", PythonWorkerOutput); diff --git a/workers/host.js b/workers/host.js new file mode 100644 index 0000000..f58c8f9 --- /dev/null +++ b/workers/host.js @@ -0,0 +1,173 @@ +// Page side of a Python app that runs in Web Workers. +// +// The page keeps only what must stay on the main thread: the canvas element +// and its input events, Web MIDI, and the AudioContext. Everything Python +// runs in workers (python_worker.js), so a long redraw or a garbage +// collection never delays a MIDI message or the audio. +// +// import { createAudio, createMidiRelay, startWorker, resumeOnGesture } from "./workers/host.js"; +// +// const audio = await createAudio({ channels: 1 }); // suspended until a gesture +// const midi = createMidiRelay(); +// const ui = startWorker({ +// name: "ui", main: "ui_main", canvas: document.getElementById("display_canvas"), +// packages: ["vendor/wheels/pydevices-....whl", "vendor/wheels/pydevices_lvgl-....whl"], +// files: { "app/ui_main.py": "ui_main.py" }, +// ports: { midi: midi.port() }, +// }); +// const engine = startWorker({ +// name: "engine", main: "engine_main", +// packages: [...], files: { "app/engine_main.py": "engine_main.py" }, +// ports: { audio: audio.workerPort, midi: midi.port() }, +// }); +// await Promise.all([ui.ready, engine.ready]); +// resumeOnGesture(audio.context); +// +// In Python: `from js import ports` (ports.audio for audiodev.worklet_audio, +// ports.midi for midiports.webmidi.relay_ports), and board_configs/workerdisplay +// for the canvas. + +const BASE = new URL("../", import.meta.url).href; + +// -- Python workers ----------------------------------------------------------- + +export function startWorker({ name = "python", main, packages = [], files = {}, env = {}, canvas = null, ports = {} }) { + const worker = new Worker(new URL("./python_worker.js", import.meta.url), { type: "module", name }); + const portNames = Object.keys(ports); + const transfer = portNames.map((k) => ports[k]); + let offscreen = null; + if (canvas) { + const rect = canvas.getBoundingClientRect(); + if (!canvas.width || !canvas.height) { + canvas.width = Math.round(rect.width); + canvas.height = Math.round(rect.height); + } + offscreen = canvas.transferControlToOffscreen(); + transfer.push(offscreen); + } + const handle = { worker, name, loaded: null, ready: null, error: null }; + handle.loaded = new Promise((resolveLoaded) => { + handle.ready = new Promise((resolve, reject) => { + worker.onmessage = (e) => { + const m = e.data; + if (m.type === "loaded") resolveLoaded(m); + else if (m.type === "ready") resolve(m); + else if (m.type === "error") { handle.error = m.error; console.error(`[${name}] ${m.error}`); reject(new Error(m.error)); } + else if (handle.onmessage) handle.onmessage(m); + }; + worker.onerror = (e) => { handle.error = e.message; reject(new Error(e.message || "worker failed to load")); }; + }); + }); + worker.postMessage({ type: "init", name, base: BASE, main, packages, files, env, portNames, canvas: offscreen }, transfer); + if (canvas) relayInput(canvas, worker); + return handle; +} + +// Pointer, wheel and key events on the canvas go to the worker, in the +// format displaydev.workerdisplay reads. The page prevents the browser +// defaults a worker can't (scrolling, the context menu). +const SCROLL_KEYS = new Set([" ", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight", "PageUp", "PageDown", "Home", "End", "Tab"]); + +export function relayInput(canvas, worker) { + canvas.tabIndex = 0; + canvas.style.touchAction = "none"; + const send = (e) => { + worker.postMessage({ type: "input", event: { + type: e.type, x: e.offsetX, y: e.offsetY, movementX: e.movementX || 0, movementY: e.movementY || 0, + button: e.button || 0, buttons: e.buttons || 0, pointerType: e.pointerType || "mouse", pointerId: e.pointerId || 0, + deltaX: e.deltaX || 0, deltaY: e.deltaY || 0, key: e.key || "", location: e.location || 0, repeat: !!e.repeat, + ctrlKey: !!e.ctrlKey, shiftKey: !!e.shiftKey, altKey: !!e.altKey, metaKey: !!e.metaKey, + } }); + }; + canvas.addEventListener("pointerdown", (e) => { canvas.focus(); canvas.setPointerCapture(e.pointerId); send(e); }); + for (const t of ["pointerup", "pointercancel", "pointermove"]) canvas.addEventListener(t, send); + canvas.addEventListener("wheel", (e) => { e.preventDefault(); send(e); }, { passive: false }); + canvas.addEventListener("contextmenu", (e) => e.preventDefault()); + for (const t of ["keydown", "keyup"]) { + canvas.addEventListener(t, (e) => { if (SCROLL_KEYS.has(e.key)) e.preventDefault(); send(e); }); + } + const resized = () => { + const r = canvas.getBoundingClientRect(); + worker.postMessage({ type: "resize", width: r.width, height: r.height }); + }; + new ResizeObserver(resized).observe(canvas); + resized(); + canvas.focus(); +} + +// -- Audio ---------------------------------------------------------------------- + +// An AudioContext and the worklet that plays a worker's blocks. Hand +// `workerPort` to the audio worker (ports: {audio: ...}). +export async function createAudio({ channels = 1, sampleRate = 48000, minFrames = 768, startFrames = 1536, maxFrames = 4096 } = {}) { + const context = new AudioContext({ latencyHint: "interactive", sampleRate }); + await context.audioWorklet.addModule(new URL("./audio_worklet.js", import.meta.url)); + const node = new AudioWorkletNode(context, "python-worker-output", { + numberOfInputs: 0, outputChannelCount: [channels], + processorOptions: { channels, minFrames, startFrames, maxFrames }, + }); + node.connect(context.destination); + const channel = new MessageChannel(); + node.port.postMessage({ type: "worker" }, [channel.port2]); + let statsWaiter = null; + const tapChunks = []; + node.port.onmessage = (e) => { + if (e.data.type === "stats" && statsWaiter) { statsWaiter(e.data); statsWaiter = null; } + else if (e.data.type === "tap") tapChunks.push(e.data); + }; + return { + context, node, workerPort: channel.port1, tapChunks, + stats: () => new Promise((r) => { statsWaiter = r; node.port.postMessage({ type: "stats" }); }), + tap: (on) => node.port.postMessage({ type: "tap", on }), + }; +} + +// Browsers start audio only after the user touches the page. +export function resumeOnGesture(context, target = window) { + if (context.state === "running") return; + const resume = () => context.resume(); + for (const t of ["pointerdown", "keydown"]) target.addEventListener(t, resume, { once: true, capture: true }); +} + +// -- Web MIDI ------------------------------------------------------------------- + +// Web MIDI exists only on the page. Each call to port() makes a MessagePort +// for one worker: every input's messages go to every port as +// {data: Uint8Array, timeStamp}, and {data} a worker sends goes to the +// outputs (midiports.webmidi.relay_ports reads and writes this format). +export function createMidiRelay({ sysex = false } = {}) { + const ports = []; + const relay = { access: null, inputs: [], outputs: [], error: null, onchange: null }; + const forward = (e) => { + const msg = { data: e.data, timeStamp: e.timeStamp }; + for (const p of ports) p.postMessage(msg); + }; + const attach = () => { + relay.inputs = []; + relay.outputs = []; + for (const input of relay.access.inputs.values()) { + input.onmidimessage = forward; + relay.inputs.push(input.name); + } + for (const output of relay.access.outputs.values()) relay.outputs.push(output); + if (relay.onchange) relay.onchange(relay); + }; + relay.ready = navigator.requestMIDIAccess + ? navigator.requestMIDIAccess({ sysex }).then((access) => { + relay.access = access; + access.onstatechange = attach; + attach(); + return relay; + }).catch((err) => { relay.error = String(err); return relay; }) + : Promise.resolve(Object.assign(relay, { error: "Web MIDI isn't available in this browser" })); + relay.port = () => { + const ch = new MessageChannel(); + ch.port1.onmessage = (e) => { + const data = e.data && e.data.data; + if (data) for (const out of relay.outputs) out.send(data); + }; + ports.push(ch.port1); + return ch.port2; + }; + return relay; +} diff --git a/workers/python_worker.js b/workers/python_worker.js new file mode 100644 index 0000000..37f319e --- /dev/null +++ b/workers/python_worker.js @@ -0,0 +1,74 @@ +// A Web Worker that runs Python (Pyodide) for a page built with host.js. +// +// The page sends one "init" message: what to install, which files to put in +// the worker's file system, which module to run, and any MessagePorts and an +// OffscreenCanvas to hand over. Python sees them as `js.ports.` and +// `js.display_canvas` (board_configs/workerdisplay reads the latter). +// Pointer, wheel and key events the page relays go to +// displaydev.workerdisplay once the app has imported it. + +import { loadPyodide } from "../vendor/pyscript/pyodide/pyodide.mjs"; + +self.ports = {}; +let py = null; +let display = null; // displaydev.workerdisplay, once imported +let modules = null; // sys.modules, to notice that import +const early = []; // input that arrived before the display existed + +function relayInput(msg) { + if (!display && modules && modules.has("displaydev.workerdisplay")) { + display = modules.get("displaydev.workerdisplay"); + } + if (!display) { + if (early.length < 64) early.push(msg); + return; + } + while (early.length) send(early.shift()); + send(msg); +} + +function send(msg) { + if (msg.type === "input") display.feed(py.toPy(msg.event)); + else if (msg.type === "resize") display.resize(msg.width, msg.height); +} + +async function init(msg, ports) { + const t0 = performance.now(); + const name = msg.name || "python"; + py = await loadPyodide({ + indexURL: new URL("../vendor/pyscript/pyodide/", import.meta.url).href, + stdout: (s) => console.log(`[${name}] ${s}`), + stderr: (s) => console.warn(`[${name}] ${s}`), + }); + for (const pkg of msg.packages || []) await py.loadPackage(new URL(pkg, msg.base).href); + for (const [src, dest] of Object.entries(msg.files || {})) { + const resp = await fetch(new URL(src, msg.base)); + if (!resp.ok) throw new Error(`${src}: ${resp.status}`); + const path = "/home/pyodide/" + dest; + py.FS.mkdirTree(path.slice(0, path.lastIndexOf("/")) || "/home/pyodide"); + py.FS.writeFile(path, new Uint8Array(await resp.arrayBuffer())); + } + if (msg.env) { + const env = py.toPy(msg.env); + py.pyimport("os").environ.update(env); + env.destroy(); + } + // MessagePorts arrive in event.ports, in the order of msg.portNames. + for (const [i, portName] of (msg.portNames || []).entries()) self.ports[portName] = ports[i]; + if (msg.canvas) self.display_canvas = msg.canvas; + modules = py.pyimport("sys").modules; + const loaded = performance.now(); + self.postMessage({ type: "loaded", ms: loaded - t0 }); + if (msg.main) py.pyimport(msg.main); + self.postMessage({ type: "ready", ms: performance.now() - t0, loadMs: loaded - t0 }); +} + +self.onmessage = (e) => { + const msg = e.data; + if (msg.type === "init") { + init(msg, e.ports) + .catch((err) => self.postMessage({ type: "error", error: String((err && err.stack) || err) })); + } else if (msg.type === "input" || msg.type === "resize") { + relayInput(msg); + } +};