From 0683acd69eff06ede1322cbed3beb75dcfec625a Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 10 Aug 2026 22:49:36 +0200 Subject: [PATCH] Screenshot from the server instead of the browser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getDisplayMedia costs a picker, a permission, and a "Sharing this tab" bar parked over the projector all night. None of it can be suppressed from the page — the indicator is a security surface with no API, and the Chromium switches that hide it were no help here. The server runs on the machine the screen is plugged into, so it can take the screenshot itself. server/screen.js drives grim for one region and /shot returns the finished crop, which means grab() hands back a URL for an rather than a data URL off a canvas: same synchronous contract, so the animations are untouched, and no tainting to work around either. Coordinates line up because a fullscreen page's CSS pixels are the compositor's logical pixels, which is the space grim's -g takes. The monitor layout comes from hyprctl, so a projector is measured against itself rather than the laptop panel, and -s pins the scale to that monitor: it keeps a mixed-DPI desk deterministic, and it keeps grim native, where resampling costs 440ms against 35ms. Needs a wlroots compositor and grim on PATH; without them the route reports 501 and the player falls back to getDisplayMedia, probed at load because that call cannot be reached through an await without losing transient activation. Also skipped when the page is not on localhost, where the server would be photographing its own screen rather than the viewer's. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01H9v2Xoa6eoeGYpK6gp95k9 --- README.md | 38 ++++++--- public/player/capture.js | 179 ++++++++++++++++++++++++++++----------- public/player/player.js | 14 +-- server/index.js | 36 ++++++++ server/screen.js | 152 +++++++++++++++++++++++++++++++++ 5 files changed, 351 insertions(+), 68 deletions(-) create mode 100644 server/screen.js diff --git a/README.md b/README.md index 355828e..7db5164 100644 --- a/README.md +++ b/README.md @@ -98,27 +98,39 @@ how you look at a single animation without waiting for the cycle to reach it. **Auto** (the default) hands selection back to the shuffle bag. Switching takes effect immediately; while pinned, track changes leave the animation running. -**The corners and trail animations ask to share your screen.** The decks are +**The corners and trail animations photograph the frame.** The decks are cross-origin YouTube iframes, so nothing on the page can read their pixels — -capturing the screen and cropping out the frame is the only way to photograph -what is playing. The prompt appears once, on the first click of **Start**, which -also puts the page into fullscreen. +a screenshot is the only way to capture what is playing. -The crop assumes the captured surface *is* the viewport, so what you pick in the -picker matters: +The server takes it. It runs on the same machine as the screen, so it can grab +the region itself with [`grim`](https://sr.ht/~emersion/grim) and hand back the +finished crop, which the player drops straight into an ``. No picker, no +permission prompt, and no "Sharing this tab" bar parked over the party. This +needs a wlroots compositor — Hyprland, Sway, river — and `grim` on `PATH`; +`server/screen.js` reads the monitor layout from `hyprctl` so the right screen +is measured when a projector is plugged in. + +Without `grim`, the player falls back to `getDisplayMedia` and asks to share +your screen on the first click of **Start**. That path costs a picker and a +sharing indicator, and what you choose in the picker matters: - **Chrome/Chromium** preselects *This tab* — take it, and the mapping is exact. - **Firefox** has no tab capture; the picker offers only windows and screens. - Choose the **entire screen**. Fullscreen is what makes it line up, so if you - leave fullscreen, press F11 to get screenshots back. + Choose the **entire screen**. - **A single window** cannot be used: it includes the tab and URL bars, and Wayland gives the page no window coordinates to subtract. -Declining is fine: the frame still parks in its corner and still glides, just -without the screenshots. The prompt is skipped entirely on a plain-HTTP origin -(i.e. when you open the player from another machine over the network), for the -same reason. Whenever screenshots are off or misaligned, the player says why in -the bottom-left corner and logs it to the console. +**Either way the page has to be fullscreen**, which is why **Start** requests it: +only then does a CSS pixel line up with a screen pixel. Leave fullscreen and the +screenshots pause until you press F11. The exception is Chrome's tab +capture, which is the viewport by definition and lines up in a window too. + +Going without is fine: the frame still parks in its corner and still glides, +just unphotographed. That is what happens when you open the player from another +machine over the network — the server would be screenshotting its own screen +rather than yours, so the player does not ask it to, and a plain-HTTP origin has +no `getDisplayMedia` either. Whenever screenshots are off or misaligned, the +player says why in the bottom-left corner and logs it to the console. ## Adding animations diff --git a/public/player/capture.js b/public/player/capture.js index bf844b0..95ca53c 100644 --- a/public/player/capture.js +++ b/public/player/capture.js @@ -1,30 +1,137 @@ -// Screen capture, used to photograph the video frame. +// Screenshots of the video frame, from one of two backends. // // The decks are cross-origin YouTube iframes, so nothing on the page can read -// their pixels — canvas taints, html2canvas sees an empty box. Capturing the -// screen through getDisplayMedia and cropping out the frame's rect is the only -// way to get the actually-playing image. +// their pixels — canvas taints, html2canvas sees an empty box. Photographing +// the screen is the only way to get the actually-playing image. // -// The crop assumes the captured surface *is* the viewport, so stream pixels map -// to CSS pixels by a single scale factor. Chrome can guarantee that with tab -// capture. Firefox has no tab capture at all — its picker offers only windows -// and screens — so there the operator shares the whole screen and the page goes -// fullscreen to match it. A shared window can never be mapped: it carries the -// tab bar and URL bar, and Wayland reports window.screenX/screenY as 0, so -// there is no offset to subtract. +// Preferred: the server takes the screenshot (see server/screen.js). It runs on +// the machine the projector is plugged into, so it can grab the region itself +// with no picker, no permission, and no "Sharing this tab" bar sitting over the +// party. The response is the finished crop, so `grab` hands back a URL and an +// does the rest. +// +// Fallback: getDisplayMedia, for machines the server cannot screenshot. That +// path crops a live stream on a canvas, and carries all the caveats the server +// path removes — a picker, a sharing indicator, and on Firefox no tab capture +// at all, only whole windows and screens. +// +// Both need the page fullscreen, for the same reason: only then does a CSS +// pixel equal a screen pixel, and a shared window can never be mapped at all — +// it carries the tab bar and URL bar, and Wayland reports window.screenX as 0, +// so there is no offset to subtract. + +// The server screenshots its own screen, which is the player's screen only when +// the two are the same machine. Opened from a laptop across the network, /shot +// would photograph the projector and crop the laptop's coordinates out of it. +const LOCAL_HOSTS = new Set(['localhost', '127.0.0.1', '[::1]', '::1']); /** - * Ask for the screen-share stream. Resolves to a `{ grab, stop }` handle, or to - * null if the operator declines or the API is missing — callers degrade rather - * than fail. + * Whether the server can screenshot for us. Probed at load rather than at the + * click, because getDisplayMedia needs the click's transient activation and + * cannot be reached through an await. + */ +let serverShots = null; + +if (LOCAL_HOSTS.has(location.hostname)) { + // Any rect will do; only the status code is read. + fetch('/shot?x=0&y=0&w=2&h=2&sw=0&sh=0', { method: 'GET', cache: 'no-store' }) + .then((response) => { + serverShots = response.status !== 501 && response.status !== 404; + }) + .catch(() => { + serverShots = false; + }); +} + +/** + * Ask for a screenshot source. Resolves to a `{ grab, stop }` handle, or to + * null if there is none — callers degrade rather than fail. * * `onIssue` receives a human-readable reason whenever screenshots are * unavailable or stop lining up; without it these failures are invisible. * - * Must be called synchronously from a user gesture (getDisplayMedia needs - * transient activation), so kick it off before any await and keep the promise. + * Must be called synchronously from a user gesture: the fallback's + * getDisplayMedia needs transient activation, so kick this off before any await + * and keep the promise. */ export function requestCapture({ onIssue = () => {} } = {}) { + if (serverShots) return Promise.resolve(openServerCapture(onIssue)); + return requestDisplayMedia(onIssue); +} + +/** + * True while the viewport covers the whole screen, which is the one case the + * crop maths can handle. Tested by dimensions rather than by + * `document.fullscreenElement`, because F11 fullscreen is browser-level and + * leaves that property null. + */ +function fillsScreen() { + return window.innerWidth === screen.width && window.innerHeight === screen.height; +} + +/** + * Watch the viewport-covers-screen condition and report only its transitions: + * grab() runs as often as once per animation frame, and a stuck state would + * otherwise repeat the same notice forever. + */ +function trackAlignment(onIssue, initial) { + let mapped = initial; + if (!mapped) { + onIssue('Screenshots need the page fullscreen to line up with the screen — press F11.'); + } + + return function aligned() { + const now = fillsScreen(); + if (now !== mapped) { + mapped = now; + onIssue( + mapped + ? 'Screenshots are lined up again.' + : 'Screenshots paused: the page no longer fills the screen — press F11.', + ); + } + return mapped; + }; +} + +// --- server screenshots ------------------------------------------------- + +function openServerCapture(onIssue) { + const aligned = trackAlignment(onIssue, fillsScreen()); + let nonce = 0; + + return { + /** + * A URL for a fresh photograph of `rect`, in CSS pixels. Safe to hand + * straight to an : the server returns the crop itself, already sized. + * Null while the viewport does not cover the screen. + */ + grab({ x, y, width, height }) { + if (!aligned()) return null; + if (width <= 0 || height <= 0) return null; + + const query = new URLSearchParams({ + x: Math.round(x), + y: Math.round(y), + w: Math.round(width), + h: Math.round(height), + sw: screen.width, + sh: screen.height, + // The server sends no-store, but a nonce also keeps two stills taken at + // the same spot from collapsing into one request in flight. + n: String(++nonce), + }); + return `/shot?${query}`; + }, + + // Nothing is held open: each screenshot is one request. + stop() {}, + }; +} + +// --- getDisplayMedia fallback ------------------------------------------- + +function requestDisplayMedia(onIssue) { // Undefined on a plain-HTTP origin, which is exactly how the README tells // people to reach this page from other machines on the network. if (!navigator.mediaDevices?.getDisplayMedia) { @@ -67,21 +174,6 @@ function describe(error) { return `Screen capture failed (${name}: ${error?.message || 'no detail'}) — running without screenshots.`; } -/** - * True while the viewport covers the whole captured surface, which is the one - * case the crop maths can handle. - * - * `browser` is Chrome's tab capture: the stream is the viewport by definition. - * For a screen it holds only while the page is fullscreen — tested by - * dimensions rather than `document.fullscreenElement`, because F11 fullscreen - * is browser-level and leaves that property null. - */ -function mapsToViewport(surface) { - if (surface === 'browser') return true; - if (surface === 'window') return false; - return window.innerWidth === screen.width && window.innerHeight === screen.height; -} - async function attach(stream, onIssue) { const [track] = stream.getVideoTracks(); if (!track) { @@ -89,12 +181,16 @@ async function attach(stream, onIssue) { return null; } + // 'browser' is Chrome's tab capture: the stream is the viewport by + // definition, so it lines up whether or not the page is fullscreen. const surface = track.getSettings?.().displaySurface ?? null; if (surface === 'window') { onIssue('Sharing a single window cannot be cropped — restart and share the whole screen instead.'); - } else if (!mapsToViewport(surface)) { - onIssue('Screenshots need the page fullscreen to line up with the shared screen — press F11.'); } + const aligned = + surface === 'browser' + ? () => true + : trackAlignment(onIssue, surface !== 'window' && fillsScreen()); const video = document.createElement('video'); video.srcObject = stream; @@ -111,11 +207,6 @@ async function attach(stream, onIssue) { const canvas = document.createElement('canvas'); const context = canvas.getContext('2d'); - // The operator can leave fullscreen mid-party, so the mapping is rechecked on - // every grab. Only the transitions are reported: grab() runs per animation - // frame, and a stuck state would otherwise repeat the notice forever. - let mapped = mapsToViewport(surface); - return { /** * Crop a viewport rect out of the live stream. `rect` is in CSS pixels. @@ -123,17 +214,7 @@ async function attach(stream, onIssue) { * or while the viewport cannot be located inside the captured surface. */ grab({ x, y, width, height }) { - const nowMapped = mapsToViewport(surface); - if (nowMapped !== mapped) { - mapped = nowMapped; - onIssue( - mapped - ? 'Screenshots are lined up again.' - : 'Screenshots paused: the page no longer fills the shared screen — press F11.', - ); - } - if (!mapped) return null; - + if (!aligned()) return null; if (video.readyState < 2 || !video.videoWidth) return null; if (width <= 0 || height <= 0) return null; diff --git a/public/player/player.js b/public/player/player.js index c00fa68..6ea6ed0 100644 --- a/public/player/player.js +++ b/public/player/player.js @@ -49,14 +49,16 @@ async function start() { if (started) return; started = true; - // Before any await: getDisplayMedia needs the click's transient activation, - // which is gone the moment this function yields. It goes first for the same - // reason — it is the call that must not lose the activation to another API. + // Before any await: the getDisplayMedia fallback needs the click's transient + // activation, which is gone the moment this function yields. It goes first for + // the same reason — it is the call that must not lose the activation to + // another API. The server-screenshot path needs no gesture, but cannot be + // chosen without one, so this ordering has to hold either way. const capturing = requestCapture({ onIssue: showNotice }); - // Screenshots only line up while the viewport covers the whole captured - // surface. Firefox cannot share a bare tab, so the operator shares the screen - // and this makes the page match it. Harmless on the tab-capture path. + // Screenshots only line up while the viewport covers the whole screen, since + // that is what makes a CSS pixel a screen pixel. Chrome's tab capture is the + // one exception, and fullscreen does it no harm. document.documentElement.requestFullscreen?.().catch(() => { showNotice('Could not go fullscreen — press F11 so screenshots line up.'); }); diff --git a/server/index.js b/server/index.js index a1b8ab0..fef7711 100644 --- a/server/index.js +++ b/server/index.js @@ -10,6 +10,7 @@ import { WebSocketServer } from 'ws'; import * as config from '../config.js'; import { PORT } from '../config.js'; import { parseVideoId, fetchTitle } from './youtube.js'; +import * as screen from './screen.js'; import * as store from './state.js'; const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); @@ -41,6 +42,41 @@ app.get('/config.js', (_req, res) => { ); }); +// Screenshots of the player's own frame (see server/screen.js). The player +// points an straight at this, so the response has to be the finished +// crop — there is no canvas step on the other end. +app.get('/shot', async (req, res) => { + const rect = { + x: Number(req.query.x), + y: Number(req.query.y), + width: Number(req.query.w), + height: Number(req.query.h), + screenWidth: Number(req.query.sw), + screenHeight: Number(req.query.sh), + }; + if (Object.values(rect).some((value) => !Number.isFinite(value))) { + res.sendStatus(400); + return; + } + + // 501, not 404: the player probes this route to decide whether to fall back + // to getDisplayMedia, and needs to tell "wrong machine" from "no route". + if (!(await screen.available())) { + res.sendStatus(501); + return; + } + + const jpeg = await screen.capture(rect); + if (!jpeg) { + res.sendStatus(422); + return; + } + + // Every request carries a fresh nonce, so a cached copy would only ever be a + // stale photograph of a moment that has passed. + res.type('image/jpeg').set('Cache-Control', 'no-store').send(jpeg); +}); + const server = createServer(app); const wss = new WebSocketServer({ server }); diff --git a/server/screen.js b/server/screen.js new file mode 100644 index 0000000..a6cf2c9 --- /dev/null +++ b/server/screen.js @@ -0,0 +1,152 @@ +// Screen capture, done by the machine instead of by the browser. +// +// The decks are cross-origin YouTube iframes, so the page cannot read their +// pixels. The browser's own answer to that is getDisplayMedia, which costs a +// picker, a permission, and a "Sharing this tab" bar parked over the projector +// all night — and on Firefox cannot share a tab at all. Since the server runs +// on the same machine as the screen it is trying to photograph, it can just +// take the screenshot itself. +// +// grim is the wlroots screencopy client (Hyprland, Sway, river). Elsewhere this +// reports unavailable and the player falls back to getDisplayMedia. + +import { spawn } from 'node:child_process'; + +// A region grab is ~40ms and ~25KB at this quality; the stills are shown at +// screen size for a few seconds, so there is nothing to gain from going higher. +const QUALITY = 80; + +// grim is quick, but a hung compositor must not hold a request open forever. +const TIMEOUT_MS = 2000; + +// hyprctl is cheap, but it is called once per screenshot and the monitor layout +// does not change during a party. +const LAYOUT_TTL_MS = 5000; + +/** True once we know grim can actually run — probed lazily, then remembered. */ +let grimAvailable = null; + +let layout = null; +let layoutAt = 0; + +function run(command, args, { input = null } = {}) { + return new Promise((resolve) => { + let child; + try { + child = spawn(command, args); + } catch { + resolve(null); + return; + } + + const chunks = []; + let settled = false; + + const timer = setTimeout(() => { + settled = true; + child.kill('SIGKILL'); + resolve(null); + }, TIMEOUT_MS); + + child.stdout.on('data', (chunk) => chunks.push(chunk)); + // Without a drain grim can block on a full stderr pipe. + child.stderr.resume(); + child.on('error', () => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(null); + }); + child.on('close', (code) => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(code === 0 ? Buffer.concat(chunks) : null); + }); + + if (input !== null) child.stdin.end(input); + }); +} + +/** + * The monitors, in the compositor's logical coordinate space — the same space + * grim's -g takes, and the same one CSS pixels live in while a page is + * fullscreen. Empty when hyprctl is missing, which the caller reads as "one + * screen, at the origin". + */ +async function monitors() { + const now = Date.now(); + if (layout && now - layoutAt < LAYOUT_TTL_MS) return layout; + + const out = await run('hyprctl', ['monitors', '-j']); + let parsed = []; + if (out) { + try { + parsed = JSON.parse(out.toString()).map((monitor) => ({ + x: monitor.x, + y: monitor.y, + scale: monitor.scale, + // hyprctl reports the mode in device pixels; everything else here, grim + // included, works in logical ones. + width: Math.round(monitor.width / monitor.scale), + height: Math.round(monitor.height / monitor.scale), + })); + } catch { + parsed = []; + } + } + + layout = parsed; + layoutAt = now; + return layout; +} + +/** Whether this machine can be screenshotted at all. */ +export async function available() { + if (grimAvailable === null) { + grimAvailable = (await run('grim', ['-h'])) !== null; + } + return grimAvailable; +} + +/** + * Photograph one rectangle of the screen the player is on. + * + * `rect` is in CSS pixels, which equal logical screen pixels while the page is + * fullscreen — that equality is the whole reason the caller insists on + * fullscreen. `screenWidth`/`screenHeight` name which monitor to measure + * against, so a player on a projector is not cropped against the laptop panel. + * + * Resolves to a JPEG buffer, or null if grim is unusable or the rectangle + * falls outside the screen. + */ +export async function capture({ x, y, width, height, screenWidth, screenHeight }) { + if (!(await available())) return null; + + const screens = await monitors(); + const screen = + screens.find((m) => m.width === screenWidth && m.height === screenHeight) ?? + screens[0] ?? + { x: 0, y: 0, scale: null, width: screenWidth, height: screenHeight }; + + // The frame can hang off an edge mid-bounce. Clip rather than refuse: grim + // fails outright on a region that leaves the output. + const left = Math.max(0, Math.min(x, screen.width)); + const top = Math.max(0, Math.min(y, screen.height)); + const right = Math.max(left, Math.min(x + width, screen.width)); + const bottom = Math.max(top, Math.min(y + height, screen.height)); + const w = Math.round(right - left); + const h = Math.round(bottom - top); + if (w < 1 || h < 1) return null; + + const args = ['-t', 'jpeg', '-q', String(QUALITY)]; + // grim otherwise renders at the largest scale of every monitor it can see, so + // on a mixed-DPI desk the size of a screenshot would depend on which other + // screens happen to be plugged in. Naming the scale also keeps it native: + // asking grim to resample is an order of magnitude slower than not. + // (-o would pin the scale too, but grim rejects it alongside -g.) + if (screen.scale) args.push('-s', String(screen.scale)); + args.push('-g', `${Math.round(screen.x + left)},${Math.round(screen.y + top)} ${w}x${h}`, '-'); + + return run('grim', args); +}