From e5b6cef3fc06b68a535313be447a1776771a9aba Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sun, 1 Nov 2026 05:15:17 -0500 Subject: [PATCH 1/3] fix(core): a resized preview video keeps its picture while its new copy loads Swapping a video to a copy made for its new size emptied it until the copy decoded, so the preview showed black for a moment. The swap now shows a still of the frame it had, as the video's poster, until the new copy can draw. --- packages/core/src/runtime/init.ts | 9 ++- packages/core/src/runtime/mediaProxy.test.ts | 70 +++++++++++++++++++- packages/core/src/runtime/mediaProxy.ts | 47 ++++++++++++- packages/core/src/runtime/proxySrc.ts | 7 ++ 4 files changed, 128 insertions(+), 5 deletions(-) diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 2fd87687192..301fb49b917 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -158,7 +158,7 @@ import { forgetPositionEdit, installPositionEditsSeekReapply, } from "./positionEdits"; -import { unproxiedSrc } from "./proxySrc"; +import { unheldPoster, unproxiedSrc } from "./proxySrc"; import { applyVariableBindings, unproxiedMediaSrc } from "./applyVariableBindings"; import { createColorGradingRuntime, type RuntimeColorGradingApi } from "./colorGrading"; import { COLOR_GRADING_AUTHORED_OPACITY_ATTR } from "../colorGrading"; @@ -385,9 +385,12 @@ const writtenShape = (el: Element): string => JSON.stringify([ Array.from(el.attributes, ({ name, value }) => [ name, - name === "class" ? ownClasses(value) : value, + name === "class" ? ownClasses(value) : name === "poster" ? unheldPoster(el) : value, ]) - .filter(([name, value]) => !RUNTIME_MEDIA_ATTRS.has(name!) && !(name === "class" && !value)) + .filter( + ([name, value]) => + !RUNTIME_MEDIA_ATTRS.has(name!) && !((name === "class" || name === "poster") && !value), + ) .sort(([a], [b]) => (a! < b! ? -1 : 1)), el.hasAttribute("data-var-src") ? null : unproxiedSrc(el), el.innerHTML, diff --git a/packages/core/src/runtime/mediaProxy.test.ts b/packages/core/src/runtime/mediaProxy.test.ts index 6c7eae21870..68b5a1969bc 100644 --- a/packages/core/src/runtime/mediaProxy.test.ts +++ b/packages/core/src/runtime/mediaProxy.test.ts @@ -71,6 +71,26 @@ function isProxied(el: HTMLMediaElement): boolean { /** Lets a started swap ask for its copy (the default stub serves it) and finish. */ const settle = () => new Promise((resolveSettle) => setTimeout(resolveSettle, 0)); +const STILL = "data:image/jpeg;base64,c3RpbGw="; +/** The element shows a frame, and a 2D canvas can draw it (jsdom has none). */ +function showingFrame(el: HTMLVideoElement) { + Object.defineProperty(el, "readyState", { configurable: true, value: 4 }); + Object.defineProperty(el, "videoWidth", { configurable: true, value: 1080 }); + Object.defineProperty(el, "videoHeight", { configurable: true, value: 1920 }); + const draw = vi.fn(); + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({ + drawImage: draw, + } as unknown as CanvasRenderingContext2D); + vi.spyOn(HTMLCanvasElement.prototype, "toDataURL").mockReturnValue(STILL); + return draw; +} +/** The poster the element has each time it loads a new file. */ +function postersAtLoad(el: HTMLVideoElement) { + const posters: (string | null)[] = []; + el.load = vi.fn(() => posters.push(el.getAttribute("poster"))); + return posters; +} + beforeEach(() => { vi.stubGlobal( "fetch", @@ -890,6 +910,51 @@ describe("a copy sized to how large the video shows", () => { expect(el.load).toHaveBeenCalledTimes(2); }); + it("shows a still of its frame while a resized copy loads, then its own poster again", async () => { + const { el, setProxyDisplayScale } = await framedVideo(1080, 1920); + el.setAttribute("poster", "/cover.jpg"); + setProxyDisplayScale(0.5); + await settle(); + const draw = showingFrame(el); + const posters = postersAtLoad(el); + + setProxyDisplayScale(1); + await settle(); + expect(boxOf(el)).toBe("2436x4096"); + expect(posters).toEqual([STILL]); + expect(draw).toHaveBeenCalledWith(el, 0, 0, 640, 1138); + // A scene swap compares the video as written, not with the still the swap put on it. + const { unheldPoster } = await import("./proxySrc"); + expect(unheldPoster(el)).toBe("/cover.jpg"); + + // The new file seeking to the playhead still shows the still; once it can draw there, the authored poster is back. + Object.defineProperty(el, "seeking", { configurable: true, value: true }); + el.dispatchEvent(new Event("loadeddata")); + expect(el.getAttribute("poster")).toBe(STILL); + Object.defineProperty(el, "seeking", { configurable: true, value: false }); + el.dispatchEvent(new Event("seeked")); + expect(el.getAttribute("poster")).toBe("/cover.jpg"); + }); + + it("lets go of the still when the resized copy fails, and holds nothing for a video with no frame yet", async () => { + const { el, setProxyDisplayScale } = await framedVideo(1080, 1920); + setProxyDisplayScale(0.5); + await settle(); + showingFrame(el); + setProxyDisplayScale(1); + await settle(); + expect(el.getAttribute("poster")).toBe(STILL); + // The new file has nothing to draw and fails: holding on would freeze the picture. + Object.defineProperty(el, "readyState", { configurable: true, value: 0 }); + el.dispatchEvent(new Event("error")); + expect(el.hasAttribute("poster")).toBe(false); + + const posters = postersAtLoad(el); + setProxyDisplayScale(2); + await settle(); + expect(posters).toEqual([null]); + }); + it("loads the newest size asked for, even when an older, smaller copy is served first", async () => { const { el, setProxyDisplayScale } = await framedVideo(1080, 1920); setProxyDisplayScale(0.5); @@ -1002,10 +1067,13 @@ describe("preview quality: lighter copies of heavy files", () => { await settle(); expect(isProxied(el)).toBe(true); + showingFrame(el); + const posters = postersAtLoad(el); setPreviewQuality(1); expect(el.getAttribute("src")).toBe("/video.mp4"); - expect(el.load).toHaveBeenCalledTimes(2); + // One more load, and the lighter copy's frame stays up until the original can draw. + expect(posters).toEqual([STILL]); }); it("lightens a video that appears after the level was lowered", async () => { diff --git a/packages/core/src/runtime/mediaProxy.ts b/packages/core/src/runtime/mediaProxy.ts index 38344cb1de9..ca4b1e3bfca 100644 --- a/packages/core/src/runtime/mediaProxy.ts +++ b/packages/core/src/runtime/mediaProxy.ts @@ -4,7 +4,7 @@ import { evictMediaSyncState, HOLD_CAP_MS, isActiveAtPlayhead } from "./media"; import { findInjectedRenderFrame } from "./renderFrameSibling"; import type { RuntimeJson } from "./types"; import { isVideoElement } from "./domRealm"; -import { swappedElements } from "./proxySrc"; +import { heldPosters, swappedElements } from "./proxySrc"; import { restartPlaybackHealth } from "./playbackHealth"; import { failedCopyElements, noteCopy, setCopyReportingLevel } from "./previewCopies"; import { waitForServedProxy } from "./proxyWait"; @@ -211,11 +211,55 @@ function restoreOriginal(el: HTMLMediaElement): void { lightElements.delete(el); evictMediaSyncState(el); restartPlaybackHealth(); + holdShownFrame(el); if (original === null) el.removeAttribute("src"); else el.setAttribute("src", original); el.load(); } +// Wide enough to stand in for the picture for the moment a swap takes; small enough to encode in a few ms. +const HELD_STILL_WIDTH = 640; +const HAVE_CURRENT_DATA = 2; +// The new file can draw (and is not seeking), or it failed and holding would freeze the picture. +const RELEASE_EVENTS = ["loadeddata", "seeked", "error"] as const; + +/** A swapped source empties the video until the new file decodes, which shows black. Until then it shows a still + * of the frame it had, as its poster. */ +function holdShownFrame(el: HTMLMediaElement): void { + if (!isVideoElement(el) || heldPosters.has(el)) return; + const still = stillOf(el); + if (!still) return; + heldPosters.set(el, el.getAttribute("poster")); + el.setAttribute("poster", still); + const release = (event: Event) => { + if (event.type !== "error" && (el.readyState < HAVE_CURRENT_DATA || el.seeking)) return; + for (const type of RELEASE_EVENTS) el.removeEventListener(type, release); + const authored = heldPosters.get(el) ?? null; + heldPosters.delete(el); + if (authored === null) el.removeAttribute("poster"); + else el.setAttribute("poster", authored); + }; + for (const type of RELEASE_EVENTS) el.addEventListener(type, release); +} + +/** The frame the video shows, as a JPEG data URL; null with no frame yet or one the page may not read. */ +function stillOf(el: HTMLVideoElement): string | null { + if (el.readyState < HAVE_CURRENT_DATA || !el.videoWidth || !el.videoHeight) return null; + const scale = Math.min(1, HELD_STILL_WIDTH / el.videoWidth); + const canvas = document.createElement("canvas"); + canvas.width = Math.max(1, Math.round(el.videoWidth * scale)); + canvas.height = Math.max(1, Math.round(el.videoHeight * scale)); + try { + const context = canvas.getContext("2d"); + if (!context) return null; + context.drawImage(el, 0, 0, canvas.width, canvas.height); + return canvas.toDataURL("image/jpeg", 0.85); + } catch { + // A cross-origin frame taints the canvas: the swap goes ahead with nothing held. + return null; + } +} + function loadProxy( el: HTMLMediaElement, href: string, @@ -229,6 +273,7 @@ function loadProxy( // Sync state measured on the previous file would read the new file's buffering as drift. evictMediaSyncState(el); restartPlaybackHealth(); + holdShownFrame(el); el.src = href; el.load(); } diff --git a/packages/core/src/runtime/proxySrc.ts b/packages/core/src/runtime/proxySrc.ts index 9811e9bd0b7..c0b3060fc08 100644 --- a/packages/core/src/runtime/proxySrc.ts +++ b/packages/core/src/runtime/proxySrc.ts @@ -4,3 +4,10 @@ export const swappedElements = new WeakMap(); export function unproxiedSrc(el: Element): string | null { return swappedElements.has(el) ? (swappedElements.get(el) ?? null) : el.getAttribute("src"); } + +/** Videos showing a still of their last frame as poster across a copy swap, each with the poster it had before. */ +export const heldPosters = new WeakMap(); + +export function unheldPoster(el: Element): string | null { + return heldPosters.has(el) ? (heldPosters.get(el) ?? null) : el.getAttribute("poster"); +} From 8dbe6cfc468031aa4cc11a39367b3371a315e8b5 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sun, 1 Nov 2026 05:54:45 -0500 Subject: [PATCH 2/3] fix(core): a held preview still is let go when no file loads back --- packages/core/src/runtime/mediaProxy.test.ts | 24 ++++++++++++++++++++ packages/core/src/runtime/mediaProxy.ts | 20 ++++++++++++---- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/packages/core/src/runtime/mediaProxy.test.ts b/packages/core/src/runtime/mediaProxy.test.ts index 68b5a1969bc..5b72b91db72 100644 --- a/packages/core/src/runtime/mediaProxy.test.ts +++ b/packages/core/src/runtime/mediaProxy.test.ts @@ -1076,6 +1076,30 @@ describe("preview quality: lighter copies of heavy files", () => { expect(posters).toEqual([STILL]); }); + it("lets go of a held still when the original has no src to put back", async () => { + const { video, setPreviewQuality, setProxyDisplayScale } = await stage({ "/video.mp4": HEAVY }); + const el = video(); + el.removeAttribute("src"); + el.append(Object.assign(document.createElement("source"), { src: "/video.mp4" })); + Object.defineProperty(el, "currentSrc", { + configurable: true, + value: el.querySelector("source")!.src, + }); + el.setAttribute("poster", "/cover.jpg"); + setPreviewQuality(0.5); + await settle(); + expect(isProxied(el)).toBe(true); + showingFrame(el); + setProxyDisplayScale(0.5); + await settle(); + expect(el.getAttribute("poster")).toBe(STILL); + + setPreviewQuality(1); + + expect(el.hasAttribute("src")).toBe(false); + expect(el.getAttribute("poster")).toBe("/cover.jpg"); + }); + it("lightens a video that appears after the level was lowered", async () => { const { video, setPreviewQuality, maybeProxyProactively } = await stage({ "/video.mp4": HEAVY, diff --git a/packages/core/src/runtime/mediaProxy.ts b/packages/core/src/runtime/mediaProxy.ts index ca4b1e3bfca..a1418d8ed1a 100644 --- a/packages/core/src/runtime/mediaProxy.ts +++ b/packages/core/src/runtime/mediaProxy.ts @@ -211,9 +211,14 @@ function restoreOriginal(el: HTMLMediaElement): void { lightElements.delete(el); evictMediaSyncState(el); restartPlaybackHealth(); - holdShownFrame(el); - if (original === null) el.removeAttribute("src"); - else el.setAttribute("src", original); + if (original === null) { + // No file loads, so no event would ever let a held still go. + letGoOfStill.get(el)?.(); + el.removeAttribute("src"); + } else { + holdShownFrame(el); + el.setAttribute("src", original); + } el.load(); } @@ -222,6 +227,7 @@ const HELD_STILL_WIDTH = 640; const HAVE_CURRENT_DATA = 2; // The new file can draw (and is not seeking), or it failed and holding would freeze the picture. const RELEASE_EVENTS = ["loadeddata", "seeked", "error"] as const; +const letGoOfStill = new WeakMap void>(); /** A swapped source empties the video until the new file decodes, which shows black. Until then it shows a still * of the frame it had, as its poster. */ @@ -231,15 +237,19 @@ function holdShownFrame(el: HTMLMediaElement): void { if (!still) return; heldPosters.set(el, el.getAttribute("poster")); el.setAttribute("poster", still); - const release = (event: Event) => { - if (event.type !== "error" && (el.readyState < HAVE_CURRENT_DATA || el.seeking)) return; + const letGo = () => { for (const type of RELEASE_EVENTS) el.removeEventListener(type, release); + letGoOfStill.delete(el); const authored = heldPosters.get(el) ?? null; heldPosters.delete(el); if (authored === null) el.removeAttribute("poster"); else el.setAttribute("poster", authored); }; + const release = (event: Event) => { + if (event.type === "error" || (el.readyState >= HAVE_CURRENT_DATA && !el.seeking)) letGo(); + }; for (const type of RELEASE_EVENTS) el.addEventListener(type, release); + letGoOfStill.set(el, letGo); } /** The frame the video shows, as a JPEG data URL; null with no frame yet or one the page may not read. */ From 8b1c1d3891a890622e3207f00be4633e5efee03c Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sun, 1 Nov 2026 06:39:50 -0500 Subject: [PATCH 3/3] chore(core): raise the packed size budget for the held preview still --- scripts/package-size-budgets.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json index ae1bc58ca2d..2a567bb5ec8 100644 --- a/scripts/package-size-budgets.json +++ b/scripts/package-size-budgets.json @@ -8,7 +8,7 @@ "unpacked": 23722874 }, "@hyperframes/core": { - "packed": 1661748, + "packed": 1762752, "unpacked": 4268445 }, "@hyperframes/engine": {