fix(media): iPhone clips preview and export with their AAC voice

packages/cli/src/audio/analysis/decode.test.ts

@@ -0,0 +1,24 @@
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
+import { decodeAudio } from "./decode.js";
+import { writeIphoneClip } from "../../../../studio-server/src/helpers/iphoneClip.test-helpers.js";
+
+const ffmpeg = findFfBinary("ffmpeg");
+
+describe.skipIf(!ffmpeg || !findFfBinary("ffprobe"))("decodeAudio", () => {
+  it("decodes an iPhone clip's AAC voice, not its undecodable spatial track", async () => {
+    const dir = mkdtempSync(join(tmpdir(), "hf-decode-"));
+    try {
+      const decoded = await decodeAudio(writeIphoneClip(ffmpeg ?? "ffmpeg", dir));
+      expect(decoded?.stream).toMatchObject({ track: 1, sampleRate: 48_000, channels: 1 });
+      const samples = decoded?.signal.channels[0] ?? new Float32Array();
+      // ffmpeg's sine peaks at 1/8.
+      expect(Math.max(...samples)).toBeGreaterThan(0.1);
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+});

packages/cli/src/audio/analysis/decode.ts

@@ -1,11 +1,13 @@
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
+import { playableAudioTrack } from "@hyperframes/core/audible-video";
 import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
 import type { AudioSignal } from "@hyperframes/core/audio-analysis";
 
 const run = promisify(execFile);
 
 export interface AudioStreamInfo {
+  track: number;
   sampleRate: number;
   channels: number;
   duration: number;
@@ -17,31 +19,43 @@ function requiredBinary(name: "ffmpeg" | "ffprobe"): string {
   return path;
 }
 
-/** The first audio stream's format, or `null` when the file has no audio. */
-export async function probeAudio(file: string): Promise<AudioStreamInfo | null> {
+/** The playable audio stream's format, or `null` when the file has none. */
+export async function probeAudio(
+  file: string,
+  options: { signal?: AbortSignal; timeout?: number } = {},
+): Promise<AudioStreamInfo | null> {
   const { stdout } = await run(
     requiredBinary("ffprobe"),
     [
       "-v",
       "error",
       "-select_streams",
-      "a:0",
+      "a",
       "-show_streams",
       "-show_format",
       "-of",
       "json",
       "--",
       file,
     ],
-    { maxBuffer: 8 * 1024 * 1024, windowsHide: true },
+    { maxBuffer: 8 * 1024 * 1024, windowsHide: true, ...options },
   );
   const parsed = JSON.parse(stdout) as {
-    streams?: { sample_rate?: string; channels?: number; duration?: string }[];
+    streams?: {
+      codec_type?: string;
+      codec_name?: string;
+      sample_rate?: string;
+      channels?: number;
+      duration?: string;
+    }[];
     format?: { duration?: string };
   };
-  const stream = parsed.streams?.[0];
-  if (!stream) return null;
+  const streams = parsed.streams ?? [];
+  const track = playableAudioTrack(streams);
+  const stream = track === undefined ? undefined : streams[track];
+  if (track === undefined || !stream) return null;
   return {
+    track,
     sampleRate: Number(stream.sample_rate),
     channels: stream.channels ?? 1,
     duration: Number(stream.duration ?? parsed.format?.duration ?? 0),
@@ -65,7 +79,7 @@ export interface Decoded {
   stream: AudioStreamInfo;
 }
 
-/** Decode the first audio stream at its own sample rate, each channel kept apart; `null` when the file has none. */
+/** Decode the playable audio stream at its own sample rate, each channel kept apart; `null` when the file has none. */
 export async function decodeAudio(
   file: string,
   window: DecodeWindow = {},
@@ -91,7 +105,7 @@ export async function decodeAudio(
       file,
       ...length,
       "-map",
-      "0:a:0",
+      `0:a:${info.track}`,
       ...(format.mono ? ["-ac", "1"] : []),
       ...(format.sampleRate ? ["-ar", String(format.sampleRate)] : []),
       "-f",

packages/cli/src/commands/init.transcodeToMp4.windowsHide.test.ts

@@ -3,7 +3,10 @@ import { describe, expect, it, vi } from "vitest";
 
 const { spawnMock } = vi.hoisted(() => ({ spawnMock: vi.fn() }));
 
-vi.mock("node:child_process", () => ({ spawn: spawnMock }));
+vi.mock("node:child_process", async (importOriginal) => ({
+  ...(await importOriginal<typeof import("node:child_process")>()),
+  spawn: spawnMock,
+}));
 vi.mock("../browser/ffmpeg.js", () => ({
   findFFmpeg: () => "/fake/bin/ffmpeg",
   findFFprobe: () => "/fake/bin/ffprobe",

packages/cli/src/whisper/transcribe.signal.test.ts

@@ -1,9 +1,9 @@
-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import * as manager from "./manager.js";
-import { transcribe } from "./transcribe.js";
+import { prepareWav, transcribe } from "./transcribe.js";
 
 describe("whisper transcribe with a signal", () => {
   let dir: string;
@@ -20,6 +20,7 @@ describe("whisper transcribe with a signal", () => {
   });
   afterEach(() => {
     vi.restoreAllMocks();
+    vi.unstubAllEnvs();
     rmSync(dir, { recursive: true, force: true });
   });
 
@@ -50,4 +51,27 @@ describe("whisper transcribe with a signal", () => {
       expect.objectContaining({ signal: controller.signal }),
     );
   });
+
+  it.skipIf(process.platform === "win32")(
+    "stops on abort while the audio track probe is still running",
+    async () => {
+      const hang = join(dir, "hang.sh");
+      const asked = join(dir, "asked.log");
+      writeFileSync(hang, `#!/bin/sh\necho "$*" >> "${asked}"\nexec sleep 60\n`);
+      chmodSync(hang, 0o755);
+      vi.stubEnv("HYPERFRAMES_FFPROBE_PATH", hang);
+      vi.stubEnv("HYPERFRAMES_FFMPEG_PATH", hang);
+      const controller = new AbortController();
+      const reason = new Error("stopped during the probe");
+
+      // The probe has spawned by the time prepareWav returns; nothing else has run yet.
+      const preparing = prepareWav(join(dir, "voice.m4a"), undefined, controller.signal);
+      controller.abort(reason);
+
+      await expect(preparing).rejects.toBe(reason);
+      expect(readFileSync(asked, { encoding: "utf8", flag: "a+" })).not.toContain(
+        "format=duration",
+      );
+    },
+  );
 });

packages/cli/src/whisper/transcribe.test.ts

@@ -2,6 +2,8 @@ import { chmodSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node
 import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
 import { describe, expect, it, test } from "vitest";
+import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
+import { writeIphoneClip } from "../../../studio-server/src/helpers/iphoneClip.test-helpers.js";
 import {
   getPreparedWavDurationSeconds,
   initialModelForLanguage,
@@ -289,6 +291,21 @@ describe("getPreparedWavDurationSeconds", () => {
 });
 
 describe.skipIf(process.platform === "win32")("prepareWav", () => {
+  const ffmpeg = findFfBinary("ffmpeg");
+  it.skipIf(!ffmpeg || !findFfBinary("ffprobe"))(
+    "extracts an iPhone clip's AAC voice, not its undecodable spatial track",
+    async () => {
+      const dir = mkdtempSync(join(tmpdir(), "hf-prepare-wav-"));
+      try {
+        const wav = await prepareWav(writeIphoneClip(ffmpeg ?? "ffmpeg", dir));
+        expect(getPreparedWavDurationSeconds(wav)).toBeCloseTo(1, 1);
+        rmSync(wav, { force: true });
+      } finally {
+        rmSync(dir, { recursive: true, force: true });
+      }
+    },
+  );
+
   it("removes the partial WAV when ffmpeg fails or is stopped", async () => {
     const dir = mkdtempSync(join(tmpdir(), "hf-prepare-wav-"));
     const ffmpeg = join(dir, "ffmpeg");

packages/cli/src/whisper/transcribe.ts

@@ -7,6 +7,7 @@ import { existsSync, readFileSync, mkdirSync, rmSync, statSync, unlinkSync } fro
 import { basename, join, extname } from "node:path";
 import { homedir, tmpdir } from "node:os";
 import { randomUUID } from "node:crypto";
+import { probeAudio } from "../audio/analysis/decode.js";
 import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js";
 import { stoppedByCancelSignal } from "../utils/renderCancellation.js";
 import { stoppableExecFile } from "../utils/stoppableExecFile.js";
@@ -341,6 +342,15 @@ async function runFfmpeg(
   }
 }
 
+/** Selects the playable audio track; ffmpeg's own pick when ffprobe gives no answer in time. */
+async function audioTrackMap(file: string, signal?: AbortSignal): Promise<string[]> {
+  const stream = await probeAudio(file, { signal, timeout: 10_000 }).catch(() => {
+    signal?.throwIfAborted();
+    return null;
+  });
+  return stream ? ["-map", `0:a:${stream.track}`] : [];
+}
+
 /**
  * Extract audio from a video file as 16kHz mono WAV (whisper requirement).
  */
@@ -352,9 +362,10 @@ async function extractAudio(videoPath: string, signal?: AbortSignal): Promise<st
     );
   }
   const wavPath = tempWavPath();
+  const map = await audioTrackMap(videoPath, signal);
   await runFfmpeg(
     ffmpegPath,
-    ["-i", videoPath, "-vn", "-ar", "16000", "-ac", "1", "-f", "wav"],
+    ["-i", videoPath, ...map, "-vn", "-ar", "16000", "-ac", "1", "-f", "wav"],
     wavPath,
     resolveAudioPreparationTimeoutMs(getMediaDurationSeconds(videoPath)),
     signal,
@@ -409,9 +420,10 @@ async function prepareAudio(audioPath: string, signal?: AbortSignal): Promise<st
     throw new Error(`ffmpeg is required to prepare audio. Install: ${getFFmpegInstallHint()}`);
   }
   const wavPath = tempWavPath();
+  const map = await audioTrackMap(audioPath, signal);
   await runFfmpeg(
     ffmpegPath,
-    ["-i", audioPath, "-ar", "16000", "-ac", "1", "-f", "wav"],
+    ["-i", audioPath, ...map, "-ar", "16000", "-ac", "1", "-f", "wav"],
     wavPath,
     resolveAudioPreparationTimeoutMs(getMediaDurationSeconds(audioPath)),
     signal,

packages/core/src/audibleVideo.test.ts

@@ -3,6 +3,7 @@ import {
   AUDIBLE_MEDIA_SELECTOR,
   audibleVideoNeedsWebAudio,
   isAudibleVideoElement,
+  playableAudioTrack,
 } from "./audibleVideo";
 
 const el = (tagName: string, attrs: Record<string, string>) => ({
@@ -66,3 +67,19 @@ describe("audibleVideoNeedsWebAudio", () => {
     expect(audibleVideoNeedsWebAudio(fields)).toBe(true);
   });
 });
+
+describe("playableAudioTrack", () => {
+  const video = { codec_type: "video", codec_name: "hevc" };
+  const aac = { codec_type: "audio", codec_name: "aac" };
+  const spatial = { codec_type: "audio", codec_name: "unknown" };
+
+  it("skips an iPhone spatial track ffprobe cannot name, counting audio streams only", () => {
+    expect(playableAudioTrack([video, spatial, aac])).toBe(1);
+    expect(playableAudioTrack([video, aac, spatial])).toBe(0);
+    expect(playableAudioTrack([video, { codec_type: "audio" }, aac])).toBe(1);
+  });
+  it("has no answer when no audio stream plays", () => {
+    expect(playableAudioTrack([video, spatial])).toBeUndefined();
+    expect(playableAudioTrack([video])).toBeUndefined();
+  });
+});

packages/core/src/audibleVideo.ts

@@ -25,3 +25,12 @@ export function audibleVideoNeedsWebAudio(fields: {
     Boolean(fields.fxChain || fields.automation || fields.audioGroup || fields.masterChain)
   );
 }
+
+export function playableAudioTrack(
+  streams: ReadonlyArray<{ codec_type?: string; codec_name?: string }>,
+): number | undefined {
+  const track = streams
+    .filter((stream) => stream.codec_type === "audio")
+    .findIndex((stream) => Boolean(stream.codec_name) && stream.codec_name !== "unknown");
+  return track === -1 ? undefined : track;
+}

packages/engine/src/services/audioMixer.level.test.ts

@@ -1,5 +1,5 @@
 import { spawnSync } from "node:child_process";
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterEach, describe, expect, it } from "vitest";
@@ -55,6 +55,29 @@ function firstAudibleSeconds(path: string): number {
   throw new Error(`No audible sample found in ${path}`);
 }
 
+/** An iPhone-shaped clip: a spatial-audio track ffmpeg cannot decode first, the AAC voice second. */
+function iphoneClip(dir: string): string {
+  const base = join(dir, "base.mov");
+  const setup = spawnSync(
+    getFfmpegBinary(),
+    [
+      ...["-nostdin", "-v", "error", "-f", "lavfi", "-i", "testsrc=s=64x64:d=1"],
+      ...["-f", "lavfi", "-i", "anoisesrc=d=1,aformat=channel_layouts=4.0"],
+      ...["-f", "lavfi", "-i", "sine=frequency=1000:duration=1:sample_rate=48000"],
+      ...["-map", "0:v", "-map", "1:a", "-map", "2:a", "-c:v", "libx264"],
+      ...["-c:a:0", "pcm_s16le", "-c:a:1", "aac", "-movflags", "+faststart", base],
+    ],
+    { encoding: "utf-8" },
+  );
+  expect(setup.status, setup.stderr).toBe(0);
+  // The header comes first (faststart), so the first `sowt` is the PCM track's sample entry.
+  const bytes = readFileSync(base);
+  bytes.write("apac", bytes.indexOf("sowt"), "latin1");
+  const clip = join(dir, "iphone.mov");
+  writeFileSync(clip, bytes);
+  return clip;
+}
+
 // Same real-ffmpeg exposure as audioMixer.grouping.test.ts, which timed out on a
 // Windows runner at vitest's default 5s. This one has not failed yet; it is one
 // spawn slower away from it.
@@ -166,6 +189,53 @@ describe.skipIf(!HAS_FFMPEG)(
       expect(result.success).toBe(true);
       expect(firstAudibleSeconds(outputPath)).toBeCloseTo(2, 2);
     });
+
+    it.each([
+      ["at normal speed", 1],
+      [
+        "on a speed ramp",
+        {
+          target: "rate",
+          points: [
+            { t: 0, v: 1 },
+            { t: 0.5, v: 1 },
+          ],
+        },
+      ],
+    ])(
+      "exports an iPhone clip's AAC voice, not its undecodable spatial track, %s",
+      async (_, rate) => {
+        const projectDir = mkdtempSync(join(tmpdir(), "hf-iphone-"));
+        const workDir = mkdtempSync(join(tmpdir(), "hf-iphone-work-"));
+        tempDirs.push(projectDir, workDir);
+        const outputPath = join(projectDir, MIXED_AUDIO_FILENAME);
+        iphoneClip(projectDir);
+
+        const result = await processCompositionAudio(
+          [
+            {
+              id: "clip",
+              src: "iphone.mov",
+              start: 0,
+              end: 0.5,
+              mediaStart: 0,
+              playbackRate: rate,
+              layer: 0,
+              volume: 1,
+              type: "video",
+            },
+          ],
+          projectDir,
+          workDir,
+          outputPath,
+          0.5,
+        );
+
+        expect(result.success).toBe(true);
+        // The 1 kHz sine sits near -21 dB; a dropped track mixes to digital silence.
+        expect(meanVolumeDb(outputPath)).toBeGreaterThan(-30);
+      },
+    );
   },
   60_000,
 );

packages/engine/src/services/audioMixer.order.test.ts

@@ -24,7 +24,7 @@ vi.mock("./aacEncode.js", async (importOriginal) => ({
 }));
 
 vi.mock("../utils/ffprobe.js", () => ({
-  extractAudioMetadata: vi.fn(async () => ({ durationSeconds: 1, channels: 2 })),
+  extractAudioMetadata: vi.fn(async () => ({ durationSeconds: 1, channels: 2, audioTrack: 0 })),
 }));
 
 import { processCompositionAudio, type AudioElement } from "./audioMixer.js";