diff --git a/packages/cli/src/audio/analysis/decode.test.ts b/packages/cli/src/audio/analysis/decode.test.ts new file mode 100644 index 00000000000..25e82ee087d --- /dev/null +++ b/packages/cli/src/audio/analysis/decode.test.ts @@ -0,0 +1,24 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; +import { decodeAudio } from "./decode.js"; +import { writeIphoneClip } from "../../../../studio-server/src/helpers/iphoneClip.test-helpers.js"; + +const ffmpeg = findFfBinary("ffmpeg"); + +describe.skipIf(!ffmpeg || !findFfBinary("ffprobe"))("decodeAudio", () => { + it("decodes an iPhone clip's AAC voice, not its undecodable spatial track", async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-decode-")); + try { + const decoded = await decodeAudio(writeIphoneClip(ffmpeg ?? "ffmpeg", dir)); + expect(decoded?.stream).toMatchObject({ track: 1, sampleRate: 48_000, channels: 1 }); + const samples = decoded?.signal.channels[0] ?? new Float32Array(); + // ffmpeg's sine peaks at 1/8. + expect(Math.max(...samples)).toBeGreaterThan(0.1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/cli/src/audio/analysis/decode.ts b/packages/cli/src/audio/analysis/decode.ts index 9ba709330a3..439c69d4131 100644 --- a/packages/cli/src/audio/analysis/decode.ts +++ b/packages/cli/src/audio/analysis/decode.ts @@ -1,11 +1,13 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; +import { playableAudioTrack } from "@hyperframes/core/audible-video"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; import type { AudioSignal } from "@hyperframes/core/audio-analysis"; const run = promisify(execFile); export interface AudioStreamInfo { + track: number; sampleRate: number; channels: number; duration: number; @@ -17,15 +19,18 @@ function requiredBinary(name: "ffmpeg" | "ffprobe"): string { return path; } -/** The first audio stream's format, or `null` when the file has no audio. */ -export async function probeAudio(file: string): Promise { +/** The playable audio stream's format, or `null` when the file has none. */ +export async function probeAudio( + file: string, + options: { signal?: AbortSignal; timeout?: number } = {}, +): Promise { const { stdout } = await run( requiredBinary("ffprobe"), [ "-v", "error", "-select_streams", - "a:0", + "a", "-show_streams", "-show_format", "-of", @@ -33,15 +38,24 @@ export async function probeAudio(file: string): Promise "--", file, ], - { maxBuffer: 8 * 1024 * 1024, windowsHide: true }, + { maxBuffer: 8 * 1024 * 1024, windowsHide: true, ...options }, ); const parsed = JSON.parse(stdout) as { - streams?: { sample_rate?: string; channels?: number; duration?: string }[]; + streams?: { + codec_type?: string; + codec_name?: string; + sample_rate?: string; + channels?: number; + duration?: string; + }[]; format?: { duration?: string }; }; - const stream = parsed.streams?.[0]; - if (!stream) return null; + const streams = parsed.streams ?? []; + const track = playableAudioTrack(streams); + const stream = track === undefined ? undefined : streams[track]; + if (track === undefined || !stream) return null; return { + track, sampleRate: Number(stream.sample_rate), channels: stream.channels ?? 1, duration: Number(stream.duration ?? parsed.format?.duration ?? 0), @@ -65,7 +79,7 @@ export interface Decoded { stream: AudioStreamInfo; } -/** Decode the first audio stream at its own sample rate, each channel kept apart; `null` when the file has none. */ +/** Decode the playable audio stream at its own sample rate, each channel kept apart; `null` when the file has none. */ export async function decodeAudio( file: string, window: DecodeWindow = {}, @@ -91,7 +105,7 @@ export async function decodeAudio( file, ...length, "-map", - "0:a:0", + `0:a:${info.track}`, ...(format.mono ? ["-ac", "1"] : []), ...(format.sampleRate ? ["-ar", String(format.sampleRate)] : []), "-f", diff --git a/packages/cli/src/commands/init.transcodeToMp4.windowsHide.test.ts b/packages/cli/src/commands/init.transcodeToMp4.windowsHide.test.ts index 05c11c3229a..daf37800100 100644 --- a/packages/cli/src/commands/init.transcodeToMp4.windowsHide.test.ts +++ b/packages/cli/src/commands/init.transcodeToMp4.windowsHide.test.ts @@ -3,7 +3,10 @@ import { describe, expect, it, vi } from "vitest"; const { spawnMock } = vi.hoisted(() => ({ spawnMock: vi.fn() })); -vi.mock("node:child_process", () => ({ spawn: spawnMock })); +vi.mock("node:child_process", async (importOriginal) => ({ + ...(await importOriginal()), + spawn: spawnMock, +})); vi.mock("../browser/ffmpeg.js", () => ({ findFFmpeg: () => "/fake/bin/ffmpeg", findFFprobe: () => "/fake/bin/ffprobe", diff --git a/packages/cli/src/whisper/transcribe.signal.test.ts b/packages/cli/src/whisper/transcribe.signal.test.ts index 35f9db6b4e4..c2502afce0a 100644 --- a/packages/cli/src/whisper/transcribe.signal.test.ts +++ b/packages/cli/src/whisper/transcribe.signal.test.ts @@ -1,9 +1,9 @@ -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import * as manager from "./manager.js"; -import { transcribe } from "./transcribe.js"; +import { prepareWav, transcribe } from "./transcribe.js"; describe("whisper transcribe with a signal", () => { let dir: string; @@ -20,6 +20,7 @@ describe("whisper transcribe with a signal", () => { }); afterEach(() => { vi.restoreAllMocks(); + vi.unstubAllEnvs(); rmSync(dir, { recursive: true, force: true }); }); @@ -50,4 +51,27 @@ describe("whisper transcribe with a signal", () => { expect.objectContaining({ signal: controller.signal }), ); }); + + it.skipIf(process.platform === "win32")( + "stops on abort while the audio track probe is still running", + async () => { + const hang = join(dir, "hang.sh"); + const asked = join(dir, "asked.log"); + writeFileSync(hang, `#!/bin/sh\necho "$*" >> "${asked}"\nexec sleep 60\n`); + chmodSync(hang, 0o755); + vi.stubEnv("HYPERFRAMES_FFPROBE_PATH", hang); + vi.stubEnv("HYPERFRAMES_FFMPEG_PATH", hang); + const controller = new AbortController(); + const reason = new Error("stopped during the probe"); + + // The probe has spawned by the time prepareWav returns; nothing else has run yet. + const preparing = prepareWav(join(dir, "voice.m4a"), undefined, controller.signal); + controller.abort(reason); + + await expect(preparing).rejects.toBe(reason); + expect(readFileSync(asked, { encoding: "utf8", flag: "a+" })).not.toContain( + "format=duration", + ); + }, + ); }); diff --git a/packages/cli/src/whisper/transcribe.test.ts b/packages/cli/src/whisper/transcribe.test.ts index 4d2f29cc4a7..84f0aac405f 100644 --- a/packages/cli/src/whisper/transcribe.test.ts +++ b/packages/cli/src/whisper/transcribe.test.ts @@ -2,6 +2,8 @@ import { chmodSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node import { homedir, tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it, test } from "vitest"; +import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; +import { writeIphoneClip } from "../../../studio-server/src/helpers/iphoneClip.test-helpers.js"; import { getPreparedWavDurationSeconds, initialModelForLanguage, @@ -289,6 +291,21 @@ describe("getPreparedWavDurationSeconds", () => { }); describe.skipIf(process.platform === "win32")("prepareWav", () => { + const ffmpeg = findFfBinary("ffmpeg"); + it.skipIf(!ffmpeg || !findFfBinary("ffprobe"))( + "extracts an iPhone clip's AAC voice, not its undecodable spatial track", + async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-prepare-wav-")); + try { + const wav = await prepareWav(writeIphoneClip(ffmpeg ?? "ffmpeg", dir)); + expect(getPreparedWavDurationSeconds(wav)).toBeCloseTo(1, 1); + rmSync(wav, { force: true }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, + ); + it("removes the partial WAV when ffmpeg fails or is stopped", async () => { const dir = mkdtempSync(join(tmpdir(), "hf-prepare-wav-")); const ffmpeg = join(dir, "ffmpeg"); diff --git a/packages/cli/src/whisper/transcribe.ts b/packages/cli/src/whisper/transcribe.ts index e7430a9c545..68bcba05708 100644 --- a/packages/cli/src/whisper/transcribe.ts +++ b/packages/cli/src/whisper/transcribe.ts @@ -7,6 +7,7 @@ import { existsSync, readFileSync, mkdirSync, rmSync, statSync, unlinkSync } fro import { basename, join, extname } from "node:path"; import { homedir, tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; +import { probeAudio } from "../audio/analysis/decode.js"; import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js"; import { stoppedByCancelSignal } from "../utils/renderCancellation.js"; import { stoppableExecFile } from "../utils/stoppableExecFile.js"; @@ -341,6 +342,15 @@ async function runFfmpeg( } } +/** Selects the playable audio track; ffmpeg's own pick when ffprobe gives no answer in time. */ +async function audioTrackMap(file: string, signal?: AbortSignal): Promise { + const stream = await probeAudio(file, { signal, timeout: 10_000 }).catch(() => { + signal?.throwIfAborted(); + return null; + }); + return stream ? ["-map", `0:a:${stream.track}`] : []; +} + /** * Extract audio from a video file as 16kHz mono WAV (whisper requirement). */ @@ -352,9 +362,10 @@ async function extractAudio(videoPath: string, signal?: AbortSignal): Promise) => ({ @@ -66,3 +67,19 @@ describe("audibleVideoNeedsWebAudio", () => { expect(audibleVideoNeedsWebAudio(fields)).toBe(true); }); }); + +describe("playableAudioTrack", () => { + const video = { codec_type: "video", codec_name: "hevc" }; + const aac = { codec_type: "audio", codec_name: "aac" }; + const spatial = { codec_type: "audio", codec_name: "unknown" }; + + it("skips an iPhone spatial track ffprobe cannot name, counting audio streams only", () => { + expect(playableAudioTrack([video, spatial, aac])).toBe(1); + expect(playableAudioTrack([video, aac, spatial])).toBe(0); + expect(playableAudioTrack([video, { codec_type: "audio" }, aac])).toBe(1); + }); + it("has no answer when no audio stream plays", () => { + expect(playableAudioTrack([video, spatial])).toBeUndefined(); + expect(playableAudioTrack([video])).toBeUndefined(); + }); +}); diff --git a/packages/core/src/audibleVideo.ts b/packages/core/src/audibleVideo.ts index 6c3b9aead37..0e92b4e3dda 100644 --- a/packages/core/src/audibleVideo.ts +++ b/packages/core/src/audibleVideo.ts @@ -25,3 +25,12 @@ export function audibleVideoNeedsWebAudio(fields: { Boolean(fields.fxChain || fields.automation || fields.audioGroup || fields.masterChain) ); } + +export function playableAudioTrack( + streams: ReadonlyArray<{ codec_type?: string; codec_name?: string }>, +): number | undefined { + const track = streams + .filter((stream) => stream.codec_type === "audio") + .findIndex((stream) => Boolean(stream.codec_name) && stream.codec_name !== "unknown"); + return track === -1 ? undefined : track; +} diff --git a/packages/engine/src/services/audioMixer.level.test.ts b/packages/engine/src/services/audioMixer.level.test.ts index 3a5b02257be..26f60244d36 100644 --- a/packages/engine/src/services/audioMixer.level.test.ts +++ b/packages/engine/src/services/audioMixer.level.test.ts @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -55,6 +55,29 @@ function firstAudibleSeconds(path: string): number { throw new Error(`No audible sample found in ${path}`); } +/** An iPhone-shaped clip: a spatial-audio track ffmpeg cannot decode first, the AAC voice second. */ +function iphoneClip(dir: string): string { + const base = join(dir, "base.mov"); + const setup = spawnSync( + getFfmpegBinary(), + [ + ...["-nostdin", "-v", "error", "-f", "lavfi", "-i", "testsrc=s=64x64:d=1"], + ...["-f", "lavfi", "-i", "anoisesrc=d=1,aformat=channel_layouts=4.0"], + ...["-f", "lavfi", "-i", "sine=frequency=1000:duration=1:sample_rate=48000"], + ...["-map", "0:v", "-map", "1:a", "-map", "2:a", "-c:v", "libx264"], + ...["-c:a:0", "pcm_s16le", "-c:a:1", "aac", "-movflags", "+faststart", base], + ], + { encoding: "utf-8" }, + ); + expect(setup.status, setup.stderr).toBe(0); + // The header comes first (faststart), so the first `sowt` is the PCM track's sample entry. + const bytes = readFileSync(base); + bytes.write("apac", bytes.indexOf("sowt"), "latin1"); + const clip = join(dir, "iphone.mov"); + writeFileSync(clip, bytes); + return clip; +} + // Same real-ffmpeg exposure as audioMixer.grouping.test.ts, which timed out on a // Windows runner at vitest's default 5s. This one has not failed yet; it is one // spawn slower away from it. @@ -166,6 +189,53 @@ describe.skipIf(!HAS_FFMPEG)( expect(result.success).toBe(true); expect(firstAudibleSeconds(outputPath)).toBeCloseTo(2, 2); }); + + it.each([ + ["at normal speed", 1], + [ + "on a speed ramp", + { + target: "rate", + points: [ + { t: 0, v: 1 }, + { t: 0.5, v: 1 }, + ], + }, + ], + ])( + "exports an iPhone clip's AAC voice, not its undecodable spatial track, %s", + async (_, rate) => { + const projectDir = mkdtempSync(join(tmpdir(), "hf-iphone-")); + const workDir = mkdtempSync(join(tmpdir(), "hf-iphone-work-")); + tempDirs.push(projectDir, workDir); + const outputPath = join(projectDir, MIXED_AUDIO_FILENAME); + iphoneClip(projectDir); + + const result = await processCompositionAudio( + [ + { + id: "clip", + src: "iphone.mov", + start: 0, + end: 0.5, + mediaStart: 0, + playbackRate: rate, + layer: 0, + volume: 1, + type: "video", + }, + ], + projectDir, + workDir, + outputPath, + 0.5, + ); + + expect(result.success).toBe(true); + // The 1 kHz sine sits near -21 dB; a dropped track mixes to digital silence. + expect(meanVolumeDb(outputPath)).toBeGreaterThan(-30); + }, + ); }, 60_000, ); diff --git a/packages/engine/src/services/audioMixer.order.test.ts b/packages/engine/src/services/audioMixer.order.test.ts index 0754d145b7c..d62e39e33d8 100644 --- a/packages/engine/src/services/audioMixer.order.test.ts +++ b/packages/engine/src/services/audioMixer.order.test.ts @@ -24,7 +24,7 @@ vi.mock("./aacEncode.js", async (importOriginal) => ({ })); vi.mock("../utils/ffprobe.js", () => ({ - extractAudioMetadata: vi.fn(async () => ({ durationSeconds: 1, channels: 2 })), + extractAudioMetadata: vi.fn(async () => ({ durationSeconds: 1, channels: 2, audioTrack: 0 })), })); import { processCompositionAudio, type AudioElement } from "./audioMixer.js"; diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.ts index 8f5d6b0e1b0..540a60504eb 100644 --- a/packages/engine/src/services/audioMixer.test.ts +++ b/packages/engine/src/services/audioMixer.test.ts @@ -20,6 +20,7 @@ const { runFfmpegMock, capturedFilterScripts, extractAudioMetadataMock } = vi.ho sampleRate: 48_000, channels: 2, audioCodec: "aac", + audioTrack: 0, })), runFfmpegMock: vi.fn(async (args: string[]) => { const legacyIdx = args.indexOf("-filter_complex_script"); @@ -162,6 +163,7 @@ describe("processCompositionAudio", () => { sampleRate: 48_000, channels: 2, audioCodec: "aac", + audioTrack: 0, }); applyAudioFxChainMock.mockClear(); capturedFilterScripts.length = 0; diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts index 87055ada2d6..eda01fa5750 100644 --- a/packages/engine/src/services/audioMixer.ts +++ b/packages/engine/src/services/audioMixer.ts @@ -125,6 +125,7 @@ const MAX_RAMP_SLICES = 240; /** Bake a rate lane as consecutive source slices, each stretched by its mean `sourceTimeAt` rate, keeping pitch. */ function buildRampFilterComplex( + input: string, lane: RateSpec & object, duration: number, headFilter: string | null, @@ -133,7 +134,7 @@ function buildRampFilterComplex( const slices = Math.min(MAX_RAMP_SLICES, Math.max(1, Math.ceil(duration / RAMP_SLICE_SECONDS))); const step = duration / slices; const split = Array.from({ length: slices }, (_, i) => `[s${i}]`).join(""); - const parts = [`[0:a]${headFilter ? `${headFilter},` : ""}asplit=${slices}${split}`]; + const parts = [`[${input}]${headFilter ? `${headFilter},` : ""}asplit=${slices}${split}`]; for (let i = 0; i < slices; i += 1) { const from = sourceTimeAt(lane, i * step); const to = sourceTimeAt(lane, (i + 1) * step); @@ -161,11 +162,13 @@ function preparedAudioOutputArgs( headFilter: string | null, ): string[] { const channelFilter = metadata?.channels === 1 ? STEREO_CHANNEL_FILTER : null; + const input = metadata ? `0:a:${metadata.audioTrack}` : "0:a"; const outputArgs = channelFilter ? [] : ["-ac", "2"]; if (typeof playbackRate === "object") { - const graph = buildRampFilterComplex(playbackRate, duration, headFilter, channelFilter); + const graph = buildRampFilterComplex(input, playbackRate, duration, headFilter, channelFilter); return ["-filter_complex", graph, "-map", "[out]", ...outputArgs]; } + if (metadata) outputArgs.push("-map", input); const filters = [headFilter, channelFilter, buildAtempoFilter(playbackRate)].filter(Boolean); if (filters.length > 0) outputArgs.push("-af", filters.join(",")); return outputArgs; diff --git a/packages/engine/src/services/chunkEncoder.test.ts b/packages/engine/src/services/chunkEncoder.test.ts index 4433c23bcd9..a72772eae2f 100644 --- a/packages/engine/src/services/chunkEncoder.test.ts +++ b/packages/engine/src/services/chunkEncoder.test.ts @@ -681,6 +681,7 @@ describe("muxVideoWithAudio audio codec handling", () => { sampleRate: 48000, channels: 2, audioCodec: "aac", + audioTrack: 0, })); vi.resetModules(); vi.doMock("child_process", () => ({ spawn })); @@ -714,6 +715,7 @@ describe("muxVideoWithAudio audio codec handling", () => { sampleRate: 48000, channels: 2, audioCodec: "mp3", + audioTrack: 0, })); vi.resetModules(); vi.doMock("child_process", () => ({ spawn })); diff --git a/packages/engine/src/utils/ffprobe.ts b/packages/engine/src/utils/ffprobe.ts index 8f4228d1e82..81c4bd8b022 100644 --- a/packages/engine/src/utils/ffprobe.ts +++ b/packages/engine/src/utils/ffprobe.ts @@ -11,6 +11,7 @@ import { pixelFormatHasAlpha, videoStreamHasAlpha, } from "@hyperframes/parsers/media-alpha"; +import { playableAudioTrack } from "@hyperframes/core/audible-video"; import { urlInputArgs } from "./urlInput.js"; import { runFfmpeg } from "./runFfmpeg.js"; import { FFPROBE_PATH_ENV, getFfprobeBinary } from "./ffmpegBinaries.js"; @@ -238,6 +239,8 @@ export interface AudioMetadata { sampleRate: number; channels: number; audioCodec: string; + /** Which audio stream these fields describe: N in ffmpeg's `0:a:N` (`playableAudioTrack`). */ + audioTrack: number; bitrate?: number; /** Seconds after the file's media time 0 at which its last audio or video stream starts. */ latestStreamLeadSeconds?: number; @@ -1209,8 +1212,14 @@ export async function extractAudioMetadata( async function probeAudioMetadata(filePath: string, signal?: AbortSignal): Promise { const output = await probeMediaOutput(filePath, signal); - const audioStream = output.streams.find((s) => s.codec_type === "audio"); - if (!audioStream) throw new Error("[FFmpeg] No audio stream found"); + const audioTrack = playableAudioTrack(output.streams); + const audioStream = + audioTrack === undefined + ? undefined + : output.streams.filter((s) => s.codec_type === "audio")[audioTrack]; + if (audioTrack === undefined || !audioStream) { + throw new Error("[FFmpeg] No playable audio stream found"); + } let durationSeconds = output.format.duration ? parseFloat(output.format.duration) : 0; const streamDuration = audioStream.duration ? parseFloat(audioStream.duration) : undefined; @@ -1221,6 +1230,7 @@ async function probeAudioMetadata(filePath: string, signal?: AbortSignal): Promi if (sampleRate > 0) { const decoded = await probeDecodedAudioDuration( filePath, + audioTrack, sampleRate, audioStartSeconds(audioStream, output.format), durationSeconds, @@ -1237,6 +1247,7 @@ async function probeAudioMetadata(filePath: string, signal?: AbortSignal): Promi sampleRate, channels: audioStream.channels || 2, audioCodec, + audioTrack, bitrate: output.format.bit_rate ? parseInt(output.format.bit_rate) : undefined, latestStreamLeadSeconds: latestStreamLeadSeconds(output), audioStreamLeadSeconds: streamLeadSeconds(output, audioStream), @@ -1291,6 +1302,7 @@ function reconcileAudioDuration( */ async function probeDecodedAudioDuration( filePath: string, + audioTrack: number, sampleRate: number, startSeconds: number, containerDurationSeconds: number, @@ -1301,7 +1313,7 @@ async function probeDecodedAudioDuration( ): Promise<{ timestamp: number; nbSamples: number } | undefined> => { const args = [ "-select_streams", - "a:0", + `a:${audioTrack}`, "-show_entries", "frame=best_effort_timestamp_time,nb_samples", "-of", diff --git a/packages/studio-server/src/helpers/childProcess.windowsHide.test.ts b/packages/studio-server/src/helpers/childProcess.windowsHide.test.ts index 025363107c3..435614a7e09 100644 --- a/packages/studio-server/src/helpers/childProcess.windowsHide.test.ts +++ b/packages/studio-server/src/helpers/childProcess.windowsHide.test.ts @@ -45,6 +45,7 @@ describe("Studio child-process options", () => { spawnMock.mockReturnValue(proc); const peaks = decodeAudioPeaks("/tmp/audio.wav"); + await vi.waitFor(() => expect(spawnMock).toHaveBeenCalled()); proc.stdout.emit("data", Buffer.from(new Float32Array([0.5]).buffer)); proc.emit("close", 0); await peaks; diff --git a/packages/studio-server/src/helpers/iphoneClip.test-helpers.ts b/packages/studio-server/src/helpers/iphoneClip.test-helpers.ts new file mode 100644 index 00000000000..21b65de4f4c --- /dev/null +++ b/packages/studio-server/src/helpers/iphoneClip.test-helpers.ts @@ -0,0 +1,20 @@ +import { execFileSync } from "node:child_process"; +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +export function writeIphoneClip(ffmpeg: string, dir: string, seconds = 1): string { + const base = join(dir, "base.mov"); + execFileSync(ffmpeg, [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", `testsrc=s=64x64:d=${seconds}`], + ...["-f", "lavfi", "-i", `anoisesrc=d=${seconds},aformat=channel_layouts=4.0`], + ...["-f", "lavfi", "-i", `sine=frequency=1000:duration=${seconds}:sample_rate=48000`], + ...["-map", "0:v", "-map", "1:a", "-map", "2:a", "-c:v", "libx264"], + ...["-c:a:0", "pcm_s16le", "-c:a:1", "aac", "-movflags", "+faststart", base], + ]); + // Apple's spatial-audio fourcc on the PCM track (the first `sowt`, as faststart puts the header first). + const bytes = readFileSync(base); + bytes.write("apac", bytes.indexOf("sowt"), "latin1"); + const clip = join(dir, "iphone.mov"); + writeFileSync(clip, bytes); + return clip; +} diff --git a/packages/studio-server/src/helpers/loudness.test.ts b/packages/studio-server/src/helpers/loudness.test.ts index 638abb66c63..378f0543315 100644 --- a/packages/studio-server/src/helpers/loudness.test.ts +++ b/packages/studio-server/src/helpers/loudness.test.ts @@ -1,8 +1,14 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it } from "vitest"; +import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; +import { writeIphoneClip } from "./iphoneClip.test-helpers.js"; import { absoluteLoudnessPlan, audioTags, loudnessMeasureArgs, + measureAudio, updateAudioVolume, } from "./loudness.js"; @@ -204,3 +210,25 @@ describe("audioTags on hostile markup", () => { expect(audioTags(html).map((t) => t.id)).toEqual(["vo"]); }); }); + +describe("measureAudio on an iPhone clip", () => { + const ffmpeg = findFfBinary("ffmpeg"); + it.skipIf(!ffmpeg || !findFfBinary("ffprobe"))( + "measures the AAC voice, not the undecodable spatial track", + async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-loudness-")); + try { + const clip = writeIphoneClip(ffmpeg ?? "ffmpeg", dir, 2); + const measured = await measureAudio(ffmpeg ?? "ffmpeg", clip, { + mediaStart: 0, + duration: null, + }); + // ffmpeg's sine is 1/8 amplitude, about -21 LUFS. + expect(measured.integratedLufs).toBeGreaterThan(-26); + expect(measured.integratedLufs).toBeLessThan(-16); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/packages/studio-server/src/helpers/loudness.ts b/packages/studio-server/src/helpers/loudness.ts index f104894205f..cb4882b4eb0 100644 --- a/packages/studio-server/src/helpers/loudness.ts +++ b/packages/studio-server/src/helpers/loudness.ts @@ -6,6 +6,7 @@ import { formatAudioGain, MAX_AUDIO_GAIN_DB } from "@hyperframes/core/audio-gain import { decodeAuthoredAttribute } from "@hyperframes/parsers"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; import { readMediaOffsetSeconds } from "@hyperframes/parsers/media-duration"; +import { probeMediaMetadata } from "./mediaMetadata.js"; import { requestedProjectPath } from "./requestSubPath.js"; const execFileAsync = promisify(execFile); @@ -454,12 +455,13 @@ export function updateAudioVolume(html: string, id: string, volume: number): str * -61.8 LUFS: output-side `-t 4` reported -33.8 LUFS, input-side reports -61.8, * the same as physically cutting the file first. */ -export function loudnessMeasureArgs(file: string, tag: MeasuredWindow): string[] { +export function loudnessMeasureArgs(file: string, tag: MeasuredWindow, audioTrack = 0): string[] { const args = ["-hide_banner", "-nostats"]; if (tag.mediaStart > 0) args.push("-ss", String(tag.mediaStart)); const sourceSpan = tag.duration === null ? null : tag.duration * (tag.playbackRate ?? 1); if (sourceSpan !== null && sourceSpan > 0) args.push("-t", String(sourceSpan)); - args.push("-i", file, "-map", "0:a:0", "-vn", "-af", "ebur128=peak=true", "-f", "null", "-"); + args.push("-i", file, "-map", `0:a:${audioTrack}`); + args.push("-vn", "-af", "ebur128=peak=true", "-f", "null", "-"); return args; } @@ -471,7 +473,8 @@ export async function measureAudio( file: string, tag: MeasuredWindow, ): Promise { - const result = await execFileAsync(ffmpegPath, loudnessMeasureArgs(file, tag), { + const { audioTrack } = await probeMediaMetadata(file); + const result = await execFileAsync(ffmpegPath, loudnessMeasureArgs(file, tag, audioTrack), { encoding: "utf8", maxBuffer: 16 * 1024 * 1024, timeout: FFMPEG_TIMEOUT_MS, diff --git a/packages/studio-server/src/helpers/mediaMetadata.test.ts b/packages/studio-server/src/helpers/mediaMetadata.test.ts index 73006c0e4c4..0e614ca7435 100644 --- a/packages/studio-server/src/helpers/mediaMetadata.test.ts +++ b/packages/studio-server/src/helpers/mediaMetadata.test.ts @@ -102,6 +102,19 @@ describe("probeMediaMetadata", () => { })); expect(silent).toMatchObject({ kind: "video", hasAudio: false }); + // An iPhone spatial track alone plays nowhere, so it is not audio to drop. + const spatialOnly = await probeMediaMetadata("/tmp/iphone.mov", () => ({ + status: 0, + stdout: JSON.stringify({ + streams: [ + { codec_type: "video", codec_name: "hevc" }, + { codec_type: "audio", codec_name: "unknown" }, + ], + }), + stderr: "", + })); + expect(spatialOnly).toMatchObject({ kind: "video", hasAudio: false }); + // An audio file is not a video; the flag is a video question only. const audioFile = await probeMediaMetadata("/tmp/music.wav", () => ({ status: 0, diff --git a/packages/studio-server/src/helpers/mediaMetadata.ts b/packages/studio-server/src/helpers/mediaMetadata.ts index 78fff6cb5c1..2d530c07e18 100644 --- a/packages/studio-server/src/helpers/mediaMetadata.ts +++ b/packages/studio-server/src/helpers/mediaMetadata.ts @@ -5,6 +5,7 @@ import { parseFirstFrameColour, type ToneMapSourceColour, } from "@hyperframes/core"; +import { playableAudioTrack } from "@hyperframes/core/audible-video"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; import { videoStreamHasAlpha } from "@hyperframes/parsers/media-alpha"; @@ -77,6 +78,7 @@ export interface MediaMetadata { /** Video stream carries alpha (pix_fmt or the VP8/VP9 alpha_mode tag). */ hasAlpha?: boolean; durationSeconds?: number; + audioTrack?: number; probeError?: string; } @@ -243,7 +245,8 @@ export async function probeMediaMetadata( }); const metadata: MediaMetadata = { kind, color: classifyMediaColor(stream) }; if (kind === "video") { - metadata.hasAudio = (parsed.streams ?? []).some((item) => item.codec_type === "audio"); + metadata.audioTrack = playableAudioTrack(parsed.streams ?? []); + metadata.hasAudio = metadata.audioTrack !== undefined; metadata.hasAlpha = stream !== undefined && videoStreamHasAlpha(stream); const duration = Number(parsed.format?.duration); if (Number.isFinite(duration) && duration > 0) metadata.durationSeconds = duration; diff --git a/packages/studio-server/src/helpers/peakMap.decodeFailure.test.ts b/packages/studio-server/src/helpers/peakMap.decodeFailure.test.ts index a2f24c8c409..a51a2c0addd 100644 --- a/packages/studio-server/src/helpers/peakMap.decodeFailure.test.ts +++ b/packages/studio-server/src/helpers/peakMap.decodeFailure.test.ts @@ -21,7 +21,10 @@ describe("decodePeakMap, decode failure", () => { writeFileSync(file, `#!/bin/sh\n${body}\n`); chmodSync(file, 0o755); }; - script("ffprobe", "echo 2"); + script( + "ffprobe", + `echo '{"streams":[{"codec_type":"audio","codec_name":"aac","channels":2}]}'`, + ); script("ffmpeg", "head -c 38400 /dev/zero; exit 1"); await expect(decodePeakMap("truncated.mov")).rejects.toThrow(/exit 1/); }, diff --git a/packages/studio-server/src/helpers/peakMap.test.ts b/packages/studio-server/src/helpers/peakMap.test.ts index 6299574e757..b434a431063 100644 --- a/packages/studio-server/src/helpers/peakMap.test.ts +++ b/packages/studio-server/src/helpers/peakMap.test.ts @@ -11,6 +11,7 @@ import { decodePeakMap, finishPeaks, } from "./peakMap.js"; +import { writeIphoneClip } from "./iphoneClip.test-helpers.js"; function stereo(frames: Array<[number, number]>): Buffer { const buf = Buffer.alloc(frames.length * 8); @@ -66,7 +67,7 @@ describe("buildPeakMapCacheKey", () => { expect(a).toBe(buildPeakMapCacheKey("a/talk.mp4", { size: 10, mtimeMs: 1 })); expect(a).not.toBe(buildPeakMapCacheKey("a/talk.mp4", { size: 11, mtimeMs: 1 })); expect(a).not.toBe(buildPeakMapCacheKey("a/talk.mp4", { size: 10, mtimeMs: 2 })); - expect(a).toMatch(/^peaks-v2_a_talk\.mp4_10-1\.json$/); + expect(a).toMatch(/^peaks-v3_a_talk\.mp4_10-1\.json$/); }); }); @@ -146,4 +147,18 @@ describe("decodePeakMap", () => { } }, ); + + it.skipIf(!ffmpeg)( + "reads an iPhone clip's AAC voice, not its undecodable spatial track", + async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-peaks-")); + try { + const peaks = await decodePeakMap(writeIphoneClip(ffmpeg ?? "ffmpeg", dir)); + // ffmpeg's sine peaks at 1/8. + expect(Math.max(...peaks)).toBeGreaterThan(0.1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, + ); }); diff --git a/packages/studio-server/src/helpers/peakMap.ts b/packages/studio-server/src/helpers/peakMap.ts index 743a94b2c43..a4fc60a711e 100644 --- a/packages/studio-server/src/helpers/peakMap.ts +++ b/packages/studio-server/src/helpers/peakMap.ts @@ -1,11 +1,12 @@ import { execFile, spawn } from "node:child_process"; import { promisify } from "node:util"; +import { playableAudioTrack } from "@hyperframes/core/audible-video"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; export const PEAK_BIN_SECONDS = 0.05; const PEAK_SAMPLE_RATE = 48_000; const execFileAsync = promisify(execFile); -const PEAK_MAP_CACHE_VERSION = "peaks-v2"; +const PEAK_MAP_CACHE_VERSION = "peaks-v3"; /** * Running max-abs per bin over interleaved f32le at the source's own channel @@ -61,30 +62,33 @@ export function finishPeaks(acc: PeakAccumulator): number[] { return acc.bins.map((peak) => Number(peak.toFixed(4))); } -async function probeChannels(mediaPath: string): Promise { +async function probeAudioTrack(mediaPath: string): Promise<{ track: number; channels: number }> { const { stdout } = await execFileAsync( findFfBinary("ffprobe") ?? "ffprobe", [ "-v", "error", "-select_streams", - "a:0", + "a", "-show_entries", - "stream=channels", + "stream=codec_type,codec_name,channels", "-of", - "csv=p=0", + "json", "--", mediaPath, ], { encoding: "utf8", timeout: 30_000, windowsHide: true }, ); - const channels = Number.parseInt(stdout.trim(), 10); - if (!Number.isFinite(channels) || channels < 1) throw new Error("no audio stream"); - return channels; + const streams: Array<{ codec_type?: string; codec_name?: string; channels?: number }> = + JSON.parse(stdout).streams ?? []; + const track = playableAudioTrack(streams); + const channels = track === undefined ? undefined : streams[track]?.channels; + if (track === undefined || !channels || channels < 1) throw new Error("no audio stream"); + return { track, channels }; } export async function decodePeakMap(mediaPath: string): Promise { - const channels = await probeChannels(mediaPath); + const { track, channels } = await probeAudioTrack(mediaPath); return new Promise((resolvePromise, reject) => { const proc = spawn( findFfBinary("ffmpeg") ?? "ffmpeg", @@ -94,7 +98,7 @@ export async function decodePeakMap(mediaPath: string): Promise { "-i", mediaPath, "-map", - "0:a:0", + `0:a:${track}`, "-vn", "-ar", String(PEAK_SAMPLE_RATE), diff --git a/packages/studio-server/src/helpers/proxyTranscoder.test.ts b/packages/studio-server/src/helpers/proxyTranscoder.test.ts index 6982a19c578..9c216e7342c 100644 --- a/packages/studio-server/src/helpers/proxyTranscoder.test.ts +++ b/packages/studio-server/src/helpers/proxyTranscoder.test.ts @@ -16,6 +16,7 @@ import { basename, join } from "node:path"; import { hdrToSdrToneMapFilter } from "@hyperframes/core"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { writeIphoneClip } from "./iphoneClip.test-helpers.js"; const FFMPEG_PATH = "/usr/bin/ffmpeg"; const realFfmpeg = findFfBinary("ffmpeg"); @@ -774,6 +775,68 @@ describe("resolveProxy", () => { 60_000, ); + it.skipIf(!realFfmpeg || !findFfBinary("ffprobe"))( + "proxies an iPhone clip with its AAC voice when its first audio track cannot be decoded", + async () => { + vi.resetModules(); + vi.doUnmock("./mediaMetadata.js"); + const { resolveProxy } = await import("./proxyTranscoder.js"); + const projectDir = tmpProject(); + const sourcePath = writeIphoneClip(realFfmpeg!, projectDir, 0.5); + + const proxyPath = await resolveProxy(projectDir, sourcePath); + const streams = execFileSync( + findFfBinary("ffprobe")!, + ["-v", "error", "-show_entries", "stream=codec_name", "-of", "csv=p=0", "--", proxyPath], + { encoding: "utf8" }, + ); + expect(streams.trim().split("\n")).toEqual(["h264", "aac"]); + }, + 60_000, + ); + + it.skipIf(!realFfmpeg || !findFfBinary("ffprobe") || process.platform === "win32")( + "keeps the sound of a symlinked clip whose target has no extension", + async () => { + vi.resetModules(); + vi.doUnmock("./mediaMetadata.js"); + const { resolveProxy } = await import("./proxyTranscoder.js"); + const projectDir = tmpProject(); + const blob = join(projectDir, "blob"); + execFileSync(realFfmpeg!, [ + "-v", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=size=64x64:rate=10:duration=0.5", + "-f", + "lavfi", + "-i", + "sine=frequency=440:duration=0.5", + "-c:v", + "libx264", + "-c:a", + "aac", + "-f", + "mov", + "-y", + blob, + ]); + const clip = join(projectDir, "clip.mov"); + symlinkSync(blob, clip); + + const proxyPath = await resolveProxy(projectDir, clip); + const streams = execFileSync( + findFfBinary("ffprobe")!, + ["-v", "error", "-show_entries", "stream=codec_name", "-of", "csv=p=0", "--", proxyPath], + { encoding: "utf8" }, + ); + expect(streams.trim().split("\n")).toEqual(["h264", "aac"]); + }, + 60_000, + ); + it("dedupes two concurrent same-key calls to one spawn", async () => { const { spawn, calls } = createSpawnSpy(); const { resolveProxy } = await loadModule(spawn, FFMPEG_PATH); diff --git a/packages/studio-server/src/helpers/proxyTranscoder.ts b/packages/studio-server/src/helpers/proxyTranscoder.ts index 18b6e823119..16100520dfb 100644 --- a/packages/studio-server/src/helpers/proxyTranscoder.ts +++ b/packages/studio-server/src/helpers/proxyTranscoder.ts @@ -19,7 +19,7 @@ import { } from "@hyperframes/core"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; import { alphaDecoderFor } from "@hyperframes/parsers/media-alpha"; -import { probeFirstFrameColour, probeMediaMetadata } from "./mediaMetadata.js"; +import { probeFirstFrameColour, probeMediaMetadata, type MediaMetadata } from "./mediaMetadata.js"; import { cleanupProxyCache } from "./proxyCache.js"; import { PROXY_VARIANT_CONFIG, type ProxyVariant } from "./mediaCodecMap.js"; import { mkdirWithinProject, realpath, realProjectRoot } from "./safePath.js"; @@ -574,6 +574,12 @@ const VP8_ARGS = [ "libopus", ]; +function streamMaps(metadata: MediaMetadata): string[] { + if (metadata.probeError || metadata.kind !== "video") return []; + const audio = metadata.audioTrack === undefined ? [] : ["-map", `0:a:${metadata.audioTrack}`]; + return ["-map", "0:V:0?", ...audio]; +} + async function buildVideoFilter( ffmpegPath: string, sourcePath: string, @@ -629,6 +635,7 @@ async function runFfmpeg( "-i", sourcePath, ...(range ? ["-t", String(range.lengthSeconds)] : []), + ...streamMaps(metadata), "-vf", videoFilter, ...(variant === "vp8" ? VP8_ARGS : H264_ARGS), diff --git a/packages/studio-server/src/helpers/waveform.test.ts b/packages/studio-server/src/helpers/waveform.test.ts index 45478fc25cb..cef0a205edd 100644 --- a/packages/studio-server/src/helpers/waveform.test.ts +++ b/packages/studio-server/src/helpers/waveform.test.ts @@ -5,6 +5,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; import { buildWaveformCacheKey, decodeAudioPeaks } from "./waveform.js"; +import { writeIphoneClip } from "./iphoneClip.test-helpers.js"; describe("buildWaveformCacheKey", () => { it("is stable for the same file", () => { @@ -71,4 +72,17 @@ describe("decodeAudioPeaks on a video file", () => { rmSync(dir, { recursive: true, force: true }); } }); + + it.skipIf(!ffmpeg || !findFfBinary("ffprobe"))( + "reads an iPhone clip's AAC voice, not its undecodable spatial track", + async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-wave-video-")); + try { + const peaks = await decodeAudioPeaks(writeIphoneClip(ffmpeg ?? "ffmpeg", dir)); + expect(Math.max(...peaks)).toBeCloseTo(1, 3); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, + ); }); diff --git a/packages/studio-server/src/helpers/waveform.ts b/packages/studio-server/src/helpers/waveform.ts index 997f6a15065..82efe751258 100644 --- a/packages/studio-server/src/helpers/waveform.ts +++ b/packages/studio-server/src/helpers/waveform.ts @@ -11,10 +11,11 @@ import { } from "node:fs"; import { dirname, join } from "node:path"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; +import { probeMediaMetadata } from "./mediaMetadata.js"; const SAMPLE_RATE = 4000; const PEAK_COUNT = 4000; -const WAVEFORM_CACHE_VERSION = "v2"; +const WAVEFORM_CACHE_VERSION = "v3"; /** * Cache filename for one asset's peaks, keyed on its content as well as its name. @@ -58,13 +59,16 @@ function computePeaks(floats: Float32Array, count: number): number[] { return peaks.map((p) => p / maxPeak); } -export function decodeAudioPeaks(audioPath: string): Promise { +export async function decodeAudioPeaks(audioPath: string): Promise { + const { audioTrack } = await probeMediaMetadata(audioPath); + const map = audioTrack === undefined ? [] : ["-map", `0:a:${audioTrack}`]; return new Promise((resolvePromise, reject) => { const proc = spawn( findFfBinary("ffmpeg") ?? "ffmpeg", [ "-i", audioPath, + ...map, "-af", "atrim=start_sample=1152", "-f",