fix(core): a resized preview video keeps its picture while its new copy loads

packages/core/src/runtime/init.ts

@@ -158,7 +158,7 @@ import {
   forgetPositionEdit,
   installPositionEditsSeekReapply,
 } from "./positionEdits";
-import { unproxiedSrc } from "./proxySrc";
+import { unheldPoster, unproxiedSrc } from "./proxySrc";
 import { applyVariableBindings, unproxiedMediaSrc } from "./applyVariableBindings";
 import { createColorGradingRuntime, type RuntimeColorGradingApi } from "./colorGrading";
 import { COLOR_GRADING_AUTHORED_OPACITY_ATTR } from "../colorGrading";
@@ -385,9 +385,12 @@ const writtenShape = (el: Element): string =>
   JSON.stringify([
     Array.from(el.attributes, ({ name, value }) => [
       name,
-      name === "class" ? ownClasses(value) : value,
+      name === "class" ? ownClasses(value) : name === "poster" ? unheldPoster(el) : value,
     ])
-      .filter(([name, value]) => !RUNTIME_MEDIA_ATTRS.has(name!) && !(name === "class" && !value))
+      .filter(
+        ([name, value]) =>
+          !RUNTIME_MEDIA_ATTRS.has(name!) && !((name === "class" || name === "poster") && !value),
+      )
       .sort(([a], [b]) => (a! < b! ? -1 : 1)),
     el.hasAttribute("data-var-src") ? null : unproxiedSrc(el),
     el.innerHTML,

packages/core/src/runtime/mediaProxy.test.ts

@@ -71,6 +71,26 @@ function isProxied(el: HTMLMediaElement): boolean {
 /** Lets a started swap ask for its copy (the default stub serves it) and finish. */
 const settle = () => new Promise((resolveSettle) => setTimeout(resolveSettle, 0));
 
+const STILL = "data:image/jpeg;base64,c3RpbGw=";
+/** The element shows a frame, and a 2D canvas can draw it (jsdom has none). */
+function showingFrame(el: HTMLVideoElement) {
+  Object.defineProperty(el, "readyState", { configurable: true, value: 4 });
+  Object.defineProperty(el, "videoWidth", { configurable: true, value: 1080 });
+  Object.defineProperty(el, "videoHeight", { configurable: true, value: 1920 });
+  const draw = vi.fn();
+  vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
+    drawImage: draw,
+  } as unknown as CanvasRenderingContext2D);
+  vi.spyOn(HTMLCanvasElement.prototype, "toDataURL").mockReturnValue(STILL);
+  return draw;
+}
+/** The poster the element has each time it loads a new file. */
+function postersAtLoad(el: HTMLVideoElement) {
+  const posters: (string | null)[] = [];
+  el.load = vi.fn(() => posters.push(el.getAttribute("poster")));
+  return posters;
+}
+
 beforeEach(() => {
   vi.stubGlobal(
     "fetch",
@@ -890,6 +910,51 @@ describe("a copy sized to how large the video shows", () => {
     expect(el.load).toHaveBeenCalledTimes(2);
   });
 
+  it("shows a still of its frame while a resized copy loads, then its own poster again", async () => {
+    const { el, setProxyDisplayScale } = await framedVideo(1080, 1920);
+    el.setAttribute("poster", "/cover.jpg");
+    setProxyDisplayScale(0.5);
+    await settle();
+    const draw = showingFrame(el);
+    const posters = postersAtLoad(el);
+
+    setProxyDisplayScale(1);
+    await settle();
+    expect(boxOf(el)).toBe("2436x4096");
+    expect(posters).toEqual([STILL]);
+    expect(draw).toHaveBeenCalledWith(el, 0, 0, 640, 1138);
+    // A scene swap compares the video as written, not with the still the swap put on it.
+    const { unheldPoster } = await import("./proxySrc");
+    expect(unheldPoster(el)).toBe("/cover.jpg");
+
+    // The new file seeking to the playhead still shows the still; once it can draw there, the authored poster is back.
+    Object.defineProperty(el, "seeking", { configurable: true, value: true });
+    el.dispatchEvent(new Event("loadeddata"));
+    expect(el.getAttribute("poster")).toBe(STILL);
+    Object.defineProperty(el, "seeking", { configurable: true, value: false });
+    el.dispatchEvent(new Event("seeked"));
+    expect(el.getAttribute("poster")).toBe("/cover.jpg");
+  });
+
+  it("lets go of the still when the resized copy fails, and holds nothing for a video with no frame yet", async () => {
+    const { el, setProxyDisplayScale } = await framedVideo(1080, 1920);
+    setProxyDisplayScale(0.5);
+    await settle();
+    showingFrame(el);
+    setProxyDisplayScale(1);
+    await settle();
+    expect(el.getAttribute("poster")).toBe(STILL);
+    // The new file has nothing to draw and fails: holding on would freeze the picture.
+    Object.defineProperty(el, "readyState", { configurable: true, value: 0 });
+    el.dispatchEvent(new Event("error"));
+    expect(el.hasAttribute("poster")).toBe(false);
+
+    const posters = postersAtLoad(el);
+    setProxyDisplayScale(2);
+    await settle();
+    expect(posters).toEqual([null]);
+  });
+
   it("loads the newest size asked for, even when an older, smaller copy is served first", async () => {
     const { el, setProxyDisplayScale } = await framedVideo(1080, 1920);
     setProxyDisplayScale(0.5);
@@ -1002,10 +1067,37 @@ describe("preview quality: lighter copies of heavy files", () => {
     await settle();
     expect(isProxied(el)).toBe(true);
 
+    showingFrame(el);
+    const posters = postersAtLoad(el);
     setPreviewQuality(1);
 
     expect(el.getAttribute("src")).toBe("/video.mp4");
-    expect(el.load).toHaveBeenCalledTimes(2);
+    // One more load, and the lighter copy's frame stays up until the original can draw.
+    expect(posters).toEqual([STILL]);
+  });
+
+  it("lets go of a held still when the original has no src to put back", async () => {
+    const { video, setPreviewQuality, setProxyDisplayScale } = await stage({ "/video.mp4": HEAVY });
+    const el = video();
+    el.removeAttribute("src");
+    el.append(Object.assign(document.createElement("source"), { src: "/video.mp4" }));
+    Object.defineProperty(el, "currentSrc", {
+      configurable: true,
+      value: el.querySelector("source")!.src,
+    });
+    el.setAttribute("poster", "/cover.jpg");
+    setPreviewQuality(0.5);
+    await settle();
+    expect(isProxied(el)).toBe(true);
+    showingFrame(el);
+    setProxyDisplayScale(0.5);
+    await settle();
+    expect(el.getAttribute("poster")).toBe(STILL);
+
+    setPreviewQuality(1);
+
+    expect(el.hasAttribute("src")).toBe(false);
+    expect(el.getAttribute("poster")).toBe("/cover.jpg");
   });
 
   it("lightens a video that appears after the level was lowered", async () => {

packages/core/src/runtime/mediaProxy.ts

@@ -4,7 +4,7 @@ import { evictMediaSyncState, HOLD_CAP_MS, isActiveAtPlayhead } from "./media";
 import { findInjectedRenderFrame } from "./renderFrameSibling";
 import type { RuntimeJson } from "./types";
 import { isVideoElement } from "./domRealm";
-import { swappedElements } from "./proxySrc";
+import { heldPosters, swappedElements } from "./proxySrc";
 import { restartPlaybackHealth } from "./playbackHealth";
 import { failedCopyElements, noteCopy, setCopyReportingLevel } from "./previewCopies";
 import { waitForServedProxy } from "./proxyWait";
@@ -211,11 +211,65 @@ function restoreOriginal(el: HTMLMediaElement): void {
   lightElements.delete(el);
   evictMediaSyncState(el);
   restartPlaybackHealth();
-  if (original === null) el.removeAttribute("src");
-  else el.setAttribute("src", original);
+  if (original === null) {
+    // No file loads, so no event would ever let a held still go.
+    letGoOfStill.get(el)?.();
+    el.removeAttribute("src");
+  } else {
+    holdShownFrame(el);
+    el.setAttribute("src", original);
+  }
   el.load();
 }
 
+// Wide enough to stand in for the picture for the moment a swap takes; small enough to encode in a few ms.
+const HELD_STILL_WIDTH = 640;
+const HAVE_CURRENT_DATA = 2;
+// The new file can draw (and is not seeking), or it failed and holding would freeze the picture.
+const RELEASE_EVENTS = ["loadeddata", "seeked", "error"] as const;
+const letGoOfStill = new WeakMap<HTMLMediaElement, () => void>();
+
+/** A swapped source empties the video until the new file decodes, which shows black. Until then it shows a still
+ * of the frame it had, as its poster. */
+function holdShownFrame(el: HTMLMediaElement): void {
+  if (!isVideoElement(el) || heldPosters.has(el)) return;
+  const still = stillOf(el);
+  if (!still) return;
+  heldPosters.set(el, el.getAttribute("poster"));
+  el.setAttribute("poster", still);
+  const letGo = () => {
+    for (const type of RELEASE_EVENTS) el.removeEventListener(type, release);
+    letGoOfStill.delete(el);
+    const authored = heldPosters.get(el) ?? null;
+    heldPosters.delete(el);
+    if (authored === null) el.removeAttribute("poster");
+    else el.setAttribute("poster", authored);
+  };
+  const release = (event: Event) => {
+    if (event.type === "error" || (el.readyState >= HAVE_CURRENT_DATA && !el.seeking)) letGo();
+  };
+  for (const type of RELEASE_EVENTS) el.addEventListener(type, release);
+  letGoOfStill.set(el, letGo);
+}
+
+/** The frame the video shows, as a JPEG data URL; null with no frame yet or one the page may not read. */
+function stillOf(el: HTMLVideoElement): string | null {
+  if (el.readyState < HAVE_CURRENT_DATA || !el.videoWidth || !el.videoHeight) return null;
+  const scale = Math.min(1, HELD_STILL_WIDTH / el.videoWidth);
+  const canvas = document.createElement("canvas");
+  canvas.width = Math.max(1, Math.round(el.videoWidth * scale));
+  canvas.height = Math.max(1, Math.round(el.videoHeight * scale));
+  try {
+    const context = canvas.getContext("2d");
+    if (!context) return null;
+    context.drawImage(el, 0, 0, canvas.width, canvas.height);
+    return canvas.toDataURL("image/jpeg", 0.85);
+  } catch {
+    // A cross-origin frame taints the canvas: the swap goes ahead with nothing held.
+    return null;
+  }
+}
+
 function loadProxy(
   el: HTMLMediaElement,
   href: string,
@@ -229,6 +283,7 @@ function loadProxy(
   // Sync state measured on the previous file would read the new file's buffering as drift.
   evictMediaSyncState(el);
   restartPlaybackHealth();
+  holdShownFrame(el);
   el.src = href;
   el.load();
 }

packages/core/src/runtime/proxySrc.ts

@@ -4,3 +4,10 @@ export const swappedElements = new WeakMap<Element, string | null>();
 export function unproxiedSrc(el: Element): string | null {
   return swappedElements.has(el) ? (swappedElements.get(el) ?? null) : el.getAttribute("src");
 }
+
+/** Videos showing a still of their last frame as poster across a copy swap, each with the poster it had before. */
+export const heldPosters = new WeakMap<Element, string | null>();
+
+export function unheldPoster(el: Element): string | null {
+  return heldPosters.has(el) ? (heldPosters.get(el) ?? null) : el.getAttribute("poster");
+}

scripts/package-size-budgets.json

@@ -8,7 +8,7 @@
     "unpacked": 23722874
   },
   "@hyperframes/core": {
-    "packed": 1661748,
+    "packed": 1762752,
     "unpacked": 4268445
   },
   "@hyperframes/engine": {