diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 2fd87687192..301fb49b917 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -158,7 +158,7 @@ import { forgetPositionEdit, installPositionEditsSeekReapply, } from "./positionEdits"; -import { unproxiedSrc } from "./proxySrc"; +import { unheldPoster, unproxiedSrc } from "./proxySrc"; import { applyVariableBindings, unproxiedMediaSrc } from "./applyVariableBindings"; import { createColorGradingRuntime, type RuntimeColorGradingApi } from "./colorGrading"; import { COLOR_GRADING_AUTHORED_OPACITY_ATTR } from "../colorGrading"; @@ -385,9 +385,12 @@ const writtenShape = (el: Element): string => JSON.stringify([ Array.from(el.attributes, ({ name, value }) => [ name, - name === "class" ? ownClasses(value) : value, + name === "class" ? ownClasses(value) : name === "poster" ? unheldPoster(el) : value, ]) - .filter(([name, value]) => !RUNTIME_MEDIA_ATTRS.has(name!) && !(name === "class" && !value)) + .filter( + ([name, value]) => + !RUNTIME_MEDIA_ATTRS.has(name!) && !((name === "class" || name === "poster") && !value), + ) .sort(([a], [b]) => (a! < b! ? -1 : 1)), el.hasAttribute("data-var-src") ? null : unproxiedSrc(el), el.innerHTML, diff --git a/packages/core/src/runtime/mediaProxy.test.ts b/packages/core/src/runtime/mediaProxy.test.ts index 6c7eae21870..5b72b91db72 100644 --- a/packages/core/src/runtime/mediaProxy.test.ts +++ b/packages/core/src/runtime/mediaProxy.test.ts @@ -71,6 +71,26 @@ function isProxied(el: HTMLMediaElement): boolean { /** Lets a started swap ask for its copy (the default stub serves it) and finish. */ const settle = () => new Promise((resolveSettle) => setTimeout(resolveSettle, 0)); +const STILL = "data:image/jpeg;base64,c3RpbGw="; +/** The element shows a frame, and a 2D canvas can draw it (jsdom has none). */ +function showingFrame(el: HTMLVideoElement) { + Object.defineProperty(el, "readyState", { configurable: true, value: 4 }); + Object.defineProperty(el, "videoWidth", { configurable: true, value: 1080 }); + Object.defineProperty(el, "videoHeight", { configurable: true, value: 1920 }); + const draw = vi.fn(); + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({ + drawImage: draw, + } as unknown as CanvasRenderingContext2D); + vi.spyOn(HTMLCanvasElement.prototype, "toDataURL").mockReturnValue(STILL); + return draw; +} +/** The poster the element has each time it loads a new file. */ +function postersAtLoad(el: HTMLVideoElement) { + const posters: (string | null)[] = []; + el.load = vi.fn(() => posters.push(el.getAttribute("poster"))); + return posters; +} + beforeEach(() => { vi.stubGlobal( "fetch", @@ -890,6 +910,51 @@ describe("a copy sized to how large the video shows", () => { expect(el.load).toHaveBeenCalledTimes(2); }); + it("shows a still of its frame while a resized copy loads, then its own poster again", async () => { + const { el, setProxyDisplayScale } = await framedVideo(1080, 1920); + el.setAttribute("poster", "/cover.jpg"); + setProxyDisplayScale(0.5); + await settle(); + const draw = showingFrame(el); + const posters = postersAtLoad(el); + + setProxyDisplayScale(1); + await settle(); + expect(boxOf(el)).toBe("2436x4096"); + expect(posters).toEqual([STILL]); + expect(draw).toHaveBeenCalledWith(el, 0, 0, 640, 1138); + // A scene swap compares the video as written, not with the still the swap put on it. + const { unheldPoster } = await import("./proxySrc"); + expect(unheldPoster(el)).toBe("/cover.jpg"); + + // The new file seeking to the playhead still shows the still; once it can draw there, the authored poster is back. + Object.defineProperty(el, "seeking", { configurable: true, value: true }); + el.dispatchEvent(new Event("loadeddata")); + expect(el.getAttribute("poster")).toBe(STILL); + Object.defineProperty(el, "seeking", { configurable: true, value: false }); + el.dispatchEvent(new Event("seeked")); + expect(el.getAttribute("poster")).toBe("/cover.jpg"); + }); + + it("lets go of the still when the resized copy fails, and holds nothing for a video with no frame yet", async () => { + const { el, setProxyDisplayScale } = await framedVideo(1080, 1920); + setProxyDisplayScale(0.5); + await settle(); + showingFrame(el); + setProxyDisplayScale(1); + await settle(); + expect(el.getAttribute("poster")).toBe(STILL); + // The new file has nothing to draw and fails: holding on would freeze the picture. + Object.defineProperty(el, "readyState", { configurable: true, value: 0 }); + el.dispatchEvent(new Event("error")); + expect(el.hasAttribute("poster")).toBe(false); + + const posters = postersAtLoad(el); + setProxyDisplayScale(2); + await settle(); + expect(posters).toEqual([null]); + }); + it("loads the newest size asked for, even when an older, smaller copy is served first", async () => { const { el, setProxyDisplayScale } = await framedVideo(1080, 1920); setProxyDisplayScale(0.5); @@ -1002,10 +1067,37 @@ describe("preview quality: lighter copies of heavy files", () => { await settle(); expect(isProxied(el)).toBe(true); + showingFrame(el); + const posters = postersAtLoad(el); setPreviewQuality(1); expect(el.getAttribute("src")).toBe("/video.mp4"); - expect(el.load).toHaveBeenCalledTimes(2); + // One more load, and the lighter copy's frame stays up until the original can draw. + expect(posters).toEqual([STILL]); + }); + + it("lets go of a held still when the original has no src to put back", async () => { + const { video, setPreviewQuality, setProxyDisplayScale } = await stage({ "/video.mp4": HEAVY }); + const el = video(); + el.removeAttribute("src"); + el.append(Object.assign(document.createElement("source"), { src: "/video.mp4" })); + Object.defineProperty(el, "currentSrc", { + configurable: true, + value: el.querySelector("source")!.src, + }); + el.setAttribute("poster", "/cover.jpg"); + setPreviewQuality(0.5); + await settle(); + expect(isProxied(el)).toBe(true); + showingFrame(el); + setProxyDisplayScale(0.5); + await settle(); + expect(el.getAttribute("poster")).toBe(STILL); + + setPreviewQuality(1); + + expect(el.hasAttribute("src")).toBe(false); + expect(el.getAttribute("poster")).toBe("/cover.jpg"); }); it("lightens a video that appears after the level was lowered", async () => { diff --git a/packages/core/src/runtime/mediaProxy.ts b/packages/core/src/runtime/mediaProxy.ts index 38344cb1de9..a1418d8ed1a 100644 --- a/packages/core/src/runtime/mediaProxy.ts +++ b/packages/core/src/runtime/mediaProxy.ts @@ -4,7 +4,7 @@ import { evictMediaSyncState, HOLD_CAP_MS, isActiveAtPlayhead } from "./media"; import { findInjectedRenderFrame } from "./renderFrameSibling"; import type { RuntimeJson } from "./types"; import { isVideoElement } from "./domRealm"; -import { swappedElements } from "./proxySrc"; +import { heldPosters, swappedElements } from "./proxySrc"; import { restartPlaybackHealth } from "./playbackHealth"; import { failedCopyElements, noteCopy, setCopyReportingLevel } from "./previewCopies"; import { waitForServedProxy } from "./proxyWait"; @@ -211,11 +211,65 @@ function restoreOriginal(el: HTMLMediaElement): void { lightElements.delete(el); evictMediaSyncState(el); restartPlaybackHealth(); - if (original === null) el.removeAttribute("src"); - else el.setAttribute("src", original); + if (original === null) { + // No file loads, so no event would ever let a held still go. + letGoOfStill.get(el)?.(); + el.removeAttribute("src"); + } else { + holdShownFrame(el); + el.setAttribute("src", original); + } el.load(); } +// Wide enough to stand in for the picture for the moment a swap takes; small enough to encode in a few ms. +const HELD_STILL_WIDTH = 640; +const HAVE_CURRENT_DATA = 2; +// The new file can draw (and is not seeking), or it failed and holding would freeze the picture. +const RELEASE_EVENTS = ["loadeddata", "seeked", "error"] as const; +const letGoOfStill = new WeakMap void>(); + +/** A swapped source empties the video until the new file decodes, which shows black. Until then it shows a still + * of the frame it had, as its poster. */ +function holdShownFrame(el: HTMLMediaElement): void { + if (!isVideoElement(el) || heldPosters.has(el)) return; + const still = stillOf(el); + if (!still) return; + heldPosters.set(el, el.getAttribute("poster")); + el.setAttribute("poster", still); + const letGo = () => { + for (const type of RELEASE_EVENTS) el.removeEventListener(type, release); + letGoOfStill.delete(el); + const authored = heldPosters.get(el) ?? null; + heldPosters.delete(el); + if (authored === null) el.removeAttribute("poster"); + else el.setAttribute("poster", authored); + }; + const release = (event: Event) => { + if (event.type === "error" || (el.readyState >= HAVE_CURRENT_DATA && !el.seeking)) letGo(); + }; + for (const type of RELEASE_EVENTS) el.addEventListener(type, release); + letGoOfStill.set(el, letGo); +} + +/** The frame the video shows, as a JPEG data URL; null with no frame yet or one the page may not read. */ +function stillOf(el: HTMLVideoElement): string | null { + if (el.readyState < HAVE_CURRENT_DATA || !el.videoWidth || !el.videoHeight) return null; + const scale = Math.min(1, HELD_STILL_WIDTH / el.videoWidth); + const canvas = document.createElement("canvas"); + canvas.width = Math.max(1, Math.round(el.videoWidth * scale)); + canvas.height = Math.max(1, Math.round(el.videoHeight * scale)); + try { + const context = canvas.getContext("2d"); + if (!context) return null; + context.drawImage(el, 0, 0, canvas.width, canvas.height); + return canvas.toDataURL("image/jpeg", 0.85); + } catch { + // A cross-origin frame taints the canvas: the swap goes ahead with nothing held. + return null; + } +} + function loadProxy( el: HTMLMediaElement, href: string, @@ -229,6 +283,7 @@ function loadProxy( // Sync state measured on the previous file would read the new file's buffering as drift. evictMediaSyncState(el); restartPlaybackHealth(); + holdShownFrame(el); el.src = href; el.load(); } diff --git a/packages/core/src/runtime/proxySrc.ts b/packages/core/src/runtime/proxySrc.ts index 9811e9bd0b7..c0b3060fc08 100644 --- a/packages/core/src/runtime/proxySrc.ts +++ b/packages/core/src/runtime/proxySrc.ts @@ -4,3 +4,10 @@ export const swappedElements = new WeakMap(); export function unproxiedSrc(el: Element): string | null { return swappedElements.has(el) ? (swappedElements.get(el) ?? null) : el.getAttribute("src"); } + +/** Videos showing a still of their last frame as poster across a copy swap, each with the poster it had before. */ +export const heldPosters = new WeakMap(); + +export function unheldPoster(el: Element): string | null { + return heldPosters.has(el) ? (heldPosters.get(el) ?? null) : el.getAttribute("poster"); +} diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json index ae1bc58ca2d..2a567bb5ec8 100644 --- a/scripts/package-size-budgets.json +++ b/scripts/package-size-budgets.json @@ -8,7 +8,7 @@ "unpacked": 23722874 }, "@hyperframes/core": { - "packed": 1661748, + "packed": 1762752, "unpacked": 4268445 }, "@hyperframes/engine": {