From 75b84269693c86c36aa2833eca0d554b78b7ec1b Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sun, 25 Oct 2026 19:17:26 -0400 Subject: [PATCH 1/3] fix(media): iPhone clips preview and export with their AAC voice An iPhone video carries a spatial-audio track ffmpeg cannot decode next to its AAC track. Preview proxies, waveforms, peak maps, loudness, audio extraction and transcription now use the first audio track ffprobe can name, not ffmpeg's own pick (most channels), which failed with 'Error opening output files: Invalid argument'. --- .../cli/src/audio/analysis/decode.test.ts | 24 +++++++ packages/cli/src/audio/analysis/decode.ts | 32 ++++++--- .../init.transcodeToMp4.windowsHide.test.ts | 5 +- .../cli/src/whisper/transcribe.signal.test.ts | 28 +++++++- packages/cli/src/whisper/transcribe.test.ts | 17 +++++ packages/cli/src/whisper/transcribe.ts | 16 ++++- packages/core/src/audibleVideo.test.ts | 17 +++++ packages/core/src/audibleVideo.ts | 9 +++ .../src/services/audioMixer.level.test.ts | 72 ++++++++++++++++++- .../src/services/audioMixer.order.test.ts | 2 +- .../engine/src/services/audioMixer.test.ts | 2 + packages/engine/src/services/audioMixer.ts | 7 +- .../engine/src/services/chunkEncoder.test.ts | 2 + packages/engine/src/utils/ffprobe.ts | 18 ++++- .../helpers/childProcess.windowsHide.test.ts | 1 + .../src/helpers/iphoneClip.test-helpers.ts | 20 ++++++ .../src/helpers/loudness.test.ts | 28 ++++++++ .../studio-server/src/helpers/loudness.ts | 9 ++- .../src/helpers/mediaMetadata.test.ts | 13 ++++ .../src/helpers/mediaMetadata.ts | 5 +- .../src/helpers/peakMap.decodeFailure.test.ts | 5 +- .../studio-server/src/helpers/peakMap.test.ts | 17 ++++- packages/studio-server/src/helpers/peakMap.ts | 24 ++++--- .../src/helpers/proxyTranscoder.test.ts | 63 ++++++++++++++++ .../src/helpers/proxyTranscoder.ts | 10 ++- .../src/helpers/waveform.test.ts | 14 ++++ .../studio-server/src/helpers/waveform.ts | 8 ++- 27 files changed, 428 insertions(+), 40 deletions(-) create mode 100644 packages/cli/src/audio/analysis/decode.test.ts create mode 100644 packages/studio-server/src/helpers/iphoneClip.test-helpers.ts diff --git a/packages/cli/src/audio/analysis/decode.test.ts b/packages/cli/src/audio/analysis/decode.test.ts new file mode 100644 index 00000000000..25e82ee087d --- /dev/null +++ b/packages/cli/src/audio/analysis/decode.test.ts @@ -0,0 +1,24 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; +import { decodeAudio } from "./decode.js"; +import { writeIphoneClip } from "../../../../studio-server/src/helpers/iphoneClip.test-helpers.js"; + +const ffmpeg = findFfBinary("ffmpeg"); + +describe.skipIf(!ffmpeg || !findFfBinary("ffprobe"))("decodeAudio", () => { + it("decodes an iPhone clip's AAC voice, not its undecodable spatial track", async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-decode-")); + try { + const decoded = await decodeAudio(writeIphoneClip(ffmpeg ?? "ffmpeg", dir)); + expect(decoded?.stream).toMatchObject({ track: 1, sampleRate: 48_000, channels: 1 }); + const samples = decoded?.signal.channels[0] ?? new Float32Array(); + // ffmpeg's sine peaks at 1/8. + expect(Math.max(...samples)).toBeGreaterThan(0.1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/cli/src/audio/analysis/decode.ts b/packages/cli/src/audio/analysis/decode.ts index 9ba709330a3..439c69d4131 100644 --- a/packages/cli/src/audio/analysis/decode.ts +++ b/packages/cli/src/audio/analysis/decode.ts @@ -1,11 +1,13 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; +import { playableAudioTrack } from "@hyperframes/core/audible-video"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; import type { AudioSignal } from "@hyperframes/core/audio-analysis"; const run = promisify(execFile); export interface AudioStreamInfo { + track: number; sampleRate: number; channels: number; duration: number; @@ -17,15 +19,18 @@ function requiredBinary(name: "ffmpeg" | "ffprobe"): string { return path; } -/** The first audio stream's format, or `null` when the file has no audio. */ -export async function probeAudio(file: string): Promise { +/** The playable audio stream's format, or `null` when the file has none. */ +export async function probeAudio( + file: string, + options: { signal?: AbortSignal; timeout?: number } = {}, +): Promise { const { stdout } = await run( requiredBinary("ffprobe"), [ "-v", "error", "-select_streams", - "a:0", + "a", "-show_streams", "-show_format", "-of", @@ -33,15 +38,24 @@ export async function probeAudio(file: string): Promise "--", file, ], - { maxBuffer: 8 * 1024 * 1024, windowsHide: true }, + { maxBuffer: 8 * 1024 * 1024, windowsHide: true, ...options }, ); const parsed = JSON.parse(stdout) as { - streams?: { sample_rate?: string; channels?: number; duration?: string }[]; + streams?: { + codec_type?: string; + codec_name?: string; + sample_rate?: string; + channels?: number; + duration?: string; + }[]; format?: { duration?: string }; }; - const stream = parsed.streams?.[0]; - if (!stream) return null; + const streams = parsed.streams ?? []; + const track = playableAudioTrack(streams); + const stream = track === undefined ? undefined : streams[track]; + if (track === undefined || !stream) return null; return { + track, sampleRate: Number(stream.sample_rate), channels: stream.channels ?? 1, duration: Number(stream.duration ?? parsed.format?.duration ?? 0), @@ -65,7 +79,7 @@ export interface Decoded { stream: AudioStreamInfo; } -/** Decode the first audio stream at its own sample rate, each channel kept apart; `null` when the file has none. */ +/** Decode the playable audio stream at its own sample rate, each channel kept apart; `null` when the file has none. */ export async function decodeAudio( file: string, window: DecodeWindow = {}, @@ -91,7 +105,7 @@ export async function decodeAudio( file, ...length, "-map", - "0:a:0", + `0:a:${info.track}`, ...(format.mono ? ["-ac", "1"] : []), ...(format.sampleRate ? ["-ar", String(format.sampleRate)] : []), "-f", diff --git a/packages/cli/src/commands/init.transcodeToMp4.windowsHide.test.ts b/packages/cli/src/commands/init.transcodeToMp4.windowsHide.test.ts index 05c11c3229a..daf37800100 100644 --- a/packages/cli/src/commands/init.transcodeToMp4.windowsHide.test.ts +++ b/packages/cli/src/commands/init.transcodeToMp4.windowsHide.test.ts @@ -3,7 +3,10 @@ import { describe, expect, it, vi } from "vitest"; const { spawnMock } = vi.hoisted(() => ({ spawnMock: vi.fn() })); -vi.mock("node:child_process", () => ({ spawn: spawnMock })); +vi.mock("node:child_process", async (importOriginal) => ({ + ...(await importOriginal()), + spawn: spawnMock, +})); vi.mock("../browser/ffmpeg.js", () => ({ findFFmpeg: () => "/fake/bin/ffmpeg", findFFprobe: () => "/fake/bin/ffprobe", diff --git a/packages/cli/src/whisper/transcribe.signal.test.ts b/packages/cli/src/whisper/transcribe.signal.test.ts index 35f9db6b4e4..c2502afce0a 100644 --- a/packages/cli/src/whisper/transcribe.signal.test.ts +++ b/packages/cli/src/whisper/transcribe.signal.test.ts @@ -1,9 +1,9 @@ -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import * as manager from "./manager.js"; -import { transcribe } from "./transcribe.js"; +import { prepareWav, transcribe } from "./transcribe.js"; describe("whisper transcribe with a signal", () => { let dir: string; @@ -20,6 +20,7 @@ describe("whisper transcribe with a signal", () => { }); afterEach(() => { vi.restoreAllMocks(); + vi.unstubAllEnvs(); rmSync(dir, { recursive: true, force: true }); }); @@ -50,4 +51,27 @@ describe("whisper transcribe with a signal", () => { expect.objectContaining({ signal: controller.signal }), ); }); + + it.skipIf(process.platform === "win32")( + "stops on abort while the audio track probe is still running", + async () => { + const hang = join(dir, "hang.sh"); + const asked = join(dir, "asked.log"); + writeFileSync(hang, `#!/bin/sh\necho "$*" >> "${asked}"\nexec sleep 60\n`); + chmodSync(hang, 0o755); + vi.stubEnv("HYPERFRAMES_FFPROBE_PATH", hang); + vi.stubEnv("HYPERFRAMES_FFMPEG_PATH", hang); + const controller = new AbortController(); + const reason = new Error("stopped during the probe"); + + // The probe has spawned by the time prepareWav returns; nothing else has run yet. + const preparing = prepareWav(join(dir, "voice.m4a"), undefined, controller.signal); + controller.abort(reason); + + await expect(preparing).rejects.toBe(reason); + expect(readFileSync(asked, { encoding: "utf8", flag: "a+" })).not.toContain( + "format=duration", + ); + }, + ); }); diff --git a/packages/cli/src/whisper/transcribe.test.ts b/packages/cli/src/whisper/transcribe.test.ts index 4d2f29cc4a7..84f0aac405f 100644 --- a/packages/cli/src/whisper/transcribe.test.ts +++ b/packages/cli/src/whisper/transcribe.test.ts @@ -2,6 +2,8 @@ import { chmodSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node import { homedir, tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it, test } from "vitest"; +import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; +import { writeIphoneClip } from "../../../studio-server/src/helpers/iphoneClip.test-helpers.js"; import { getPreparedWavDurationSeconds, initialModelForLanguage, @@ -289,6 +291,21 @@ describe("getPreparedWavDurationSeconds", () => { }); describe.skipIf(process.platform === "win32")("prepareWav", () => { + const ffmpeg = findFfBinary("ffmpeg"); + it.skipIf(!ffmpeg || !findFfBinary("ffprobe"))( + "extracts an iPhone clip's AAC voice, not its undecodable spatial track", + async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-prepare-wav-")); + try { + const wav = await prepareWav(writeIphoneClip(ffmpeg ?? "ffmpeg", dir)); + expect(getPreparedWavDurationSeconds(wav)).toBeCloseTo(1, 1); + rmSync(wav, { force: true }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, + ); + it("removes the partial WAV when ffmpeg fails or is stopped", async () => { const dir = mkdtempSync(join(tmpdir(), "hf-prepare-wav-")); const ffmpeg = join(dir, "ffmpeg"); diff --git a/packages/cli/src/whisper/transcribe.ts b/packages/cli/src/whisper/transcribe.ts index e7430a9c545..68bcba05708 100644 --- a/packages/cli/src/whisper/transcribe.ts +++ b/packages/cli/src/whisper/transcribe.ts @@ -7,6 +7,7 @@ import { existsSync, readFileSync, mkdirSync, rmSync, statSync, unlinkSync } fro import { basename, join, extname } from "node:path"; import { homedir, tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; +import { probeAudio } from "../audio/analysis/decode.js"; import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js"; import { stoppedByCancelSignal } from "../utils/renderCancellation.js"; import { stoppableExecFile } from "../utils/stoppableExecFile.js"; @@ -341,6 +342,15 @@ async function runFfmpeg( } } +/** Selects the playable audio track; ffmpeg's own pick when ffprobe gives no answer in time. */ +async function audioTrackMap(file: string, signal?: AbortSignal): Promise { + const stream = await probeAudio(file, { signal, timeout: 10_000 }).catch(() => { + signal?.throwIfAborted(); + return null; + }); + return stream ? ["-map", `0:a:${stream.track}`] : []; +} + /** * Extract audio from a video file as 16kHz mono WAV (whisper requirement). */ @@ -352,9 +362,10 @@ async function extractAudio(videoPath: string, signal?: AbortSignal): Promise) => ({ @@ -66,3 +67,19 @@ describe("audibleVideoNeedsWebAudio", () => { expect(audibleVideoNeedsWebAudio(fields)).toBe(true); }); }); + +describe("playableAudioTrack", () => { + const video = { codec_type: "video", codec_name: "hevc" }; + const aac = { codec_type: "audio", codec_name: "aac" }; + const spatial = { codec_type: "audio", codec_name: "unknown" }; + + it("skips an iPhone spatial track ffprobe cannot name, counting audio streams only", () => { + expect(playableAudioTrack([video, spatial, aac])).toBe(1); + expect(playableAudioTrack([video, aac, spatial])).toBe(0); + expect(playableAudioTrack([video, { codec_type: "audio" }, aac])).toBe(1); + }); + it("has no answer when no audio stream plays", () => { + expect(playableAudioTrack([video, spatial])).toBeUndefined(); + expect(playableAudioTrack([video])).toBeUndefined(); + }); +}); diff --git a/packages/core/src/audibleVideo.ts b/packages/core/src/audibleVideo.ts index 6c3b9aead37..0e92b4e3dda 100644 --- a/packages/core/src/audibleVideo.ts +++ b/packages/core/src/audibleVideo.ts @@ -25,3 +25,12 @@ export function audibleVideoNeedsWebAudio(fields: { Boolean(fields.fxChain || fields.automation || fields.audioGroup || fields.masterChain) ); } + +export function playableAudioTrack( + streams: ReadonlyArray<{ codec_type?: string; codec_name?: string }>, +): number | undefined { + const track = streams + .filter((stream) => stream.codec_type === "audio") + .findIndex((stream) => Boolean(stream.codec_name) && stream.codec_name !== "unknown"); + return track === -1 ? undefined : track; +} diff --git a/packages/engine/src/services/audioMixer.level.test.ts b/packages/engine/src/services/audioMixer.level.test.ts index 3a5b02257be..26f60244d36 100644 --- a/packages/engine/src/services/audioMixer.level.test.ts +++ b/packages/engine/src/services/audioMixer.level.test.ts @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -55,6 +55,29 @@ function firstAudibleSeconds(path: string): number { throw new Error(`No audible sample found in ${path}`); } +/** An iPhone-shaped clip: a spatial-audio track ffmpeg cannot decode first, the AAC voice second. */ +function iphoneClip(dir: string): string { + const base = join(dir, "base.mov"); + const setup = spawnSync( + getFfmpegBinary(), + [ + ...["-nostdin", "-v", "error", "-f", "lavfi", "-i", "testsrc=s=64x64:d=1"], + ...["-f", "lavfi", "-i", "anoisesrc=d=1,aformat=channel_layouts=4.0"], + ...["-f", "lavfi", "-i", "sine=frequency=1000:duration=1:sample_rate=48000"], + ...["-map", "0:v", "-map", "1:a", "-map", "2:a", "-c:v", "libx264"], + ...["-c:a:0", "pcm_s16le", "-c:a:1", "aac", "-movflags", "+faststart", base], + ], + { encoding: "utf-8" }, + ); + expect(setup.status, setup.stderr).toBe(0); + // The header comes first (faststart), so the first `sowt` is the PCM track's sample entry. + const bytes = readFileSync(base); + bytes.write("apac", bytes.indexOf("sowt"), "latin1"); + const clip = join(dir, "iphone.mov"); + writeFileSync(clip, bytes); + return clip; +} + // Same real-ffmpeg exposure as audioMixer.grouping.test.ts, which timed out on a // Windows runner at vitest's default 5s. This one has not failed yet; it is one // spawn slower away from it. @@ -166,6 +189,53 @@ describe.skipIf(!HAS_FFMPEG)( expect(result.success).toBe(true); expect(firstAudibleSeconds(outputPath)).toBeCloseTo(2, 2); }); + + it.each([ + ["at normal speed", 1], + [ + "on a speed ramp", + { + target: "rate", + points: [ + { t: 0, v: 1 }, + { t: 0.5, v: 1 }, + ], + }, + ], + ])( + "exports an iPhone clip's AAC voice, not its undecodable spatial track, %s", + async (_, rate) => { + const projectDir = mkdtempSync(join(tmpdir(), "hf-iphone-")); + const workDir = mkdtempSync(join(tmpdir(), "hf-iphone-work-")); + tempDirs.push(projectDir, workDir); + const outputPath = join(projectDir, MIXED_AUDIO_FILENAME); + iphoneClip(projectDir); + + const result = await processCompositionAudio( + [ + { + id: "clip", + src: "iphone.mov", + start: 0, + end: 0.5, + mediaStart: 0, + playbackRate: rate, + layer: 0, + volume: 1, + type: "video", + }, + ], + projectDir, + workDir, + outputPath, + 0.5, + ); + + expect(result.success).toBe(true); + // The 1 kHz sine sits near -21 dB; a dropped track mixes to digital silence. + expect(meanVolumeDb(outputPath)).toBeGreaterThan(-30); + }, + ); }, 60_000, ); diff --git a/packages/engine/src/services/audioMixer.order.test.ts b/packages/engine/src/services/audioMixer.order.test.ts index 0754d145b7c..d62e39e33d8 100644 --- a/packages/engine/src/services/audioMixer.order.test.ts +++ b/packages/engine/src/services/audioMixer.order.test.ts @@ -24,7 +24,7 @@ vi.mock("./aacEncode.js", async (importOriginal) => ({ })); vi.mock("../utils/ffprobe.js", () => ({ - extractAudioMetadata: vi.fn(async () => ({ durationSeconds: 1, channels: 2 })), + extractAudioMetadata: vi.fn(async () => ({ durationSeconds: 1, channels: 2, audioTrack: 0 })), })); import { processCompositionAudio, type AudioElement } from "./audioMixer.js"; diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.ts index 8f5d6b0e1b0..540a60504eb 100644 --- a/packages/engine/src/services/audioMixer.test.ts +++ b/packages/engine/src/services/audioMixer.test.ts @@ -20,6 +20,7 @@ const { runFfmpegMock, capturedFilterScripts, extractAudioMetadataMock } = vi.ho sampleRate: 48_000, channels: 2, audioCodec: "aac", + audioTrack: 0, })), runFfmpegMock: vi.fn(async (args: string[]) => { const legacyIdx = args.indexOf("-filter_complex_script"); @@ -162,6 +163,7 @@ describe("processCompositionAudio", () => { sampleRate: 48_000, channels: 2, audioCodec: "aac", + audioTrack: 0, }); applyAudioFxChainMock.mockClear(); capturedFilterScripts.length = 0; diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts index 87055ada2d6..eda01fa5750 100644 --- a/packages/engine/src/services/audioMixer.ts +++ b/packages/engine/src/services/audioMixer.ts @@ -125,6 +125,7 @@ const MAX_RAMP_SLICES = 240; /** Bake a rate lane as consecutive source slices, each stretched by its mean `sourceTimeAt` rate, keeping pitch. */ function buildRampFilterComplex( + input: string, lane: RateSpec & object, duration: number, headFilter: string | null, @@ -133,7 +134,7 @@ function buildRampFilterComplex( const slices = Math.min(MAX_RAMP_SLICES, Math.max(1, Math.ceil(duration / RAMP_SLICE_SECONDS))); const step = duration / slices; const split = Array.from({ length: slices }, (_, i) => `[s${i}]`).join(""); - const parts = [`[0:a]${headFilter ? `${headFilter},` : ""}asplit=${slices}${split}`]; + const parts = [`[${input}]${headFilter ? `${headFilter},` : ""}asplit=${slices}${split}`]; for (let i = 0; i < slices; i += 1) { const from = sourceTimeAt(lane, i * step); const to = sourceTimeAt(lane, (i + 1) * step); @@ -161,11 +162,13 @@ function preparedAudioOutputArgs( headFilter: string | null, ): string[] { const channelFilter = metadata?.channels === 1 ? STEREO_CHANNEL_FILTER : null; + const input = metadata ? `0:a:${metadata.audioTrack}` : "0:a"; const outputArgs = channelFilter ? [] : ["-ac", "2"]; if (typeof playbackRate === "object") { - const graph = buildRampFilterComplex(playbackRate, duration, headFilter, channelFilter); + const graph = buildRampFilterComplex(input, playbackRate, duration, headFilter, channelFilter); return ["-filter_complex", graph, "-map", "[out]", ...outputArgs]; } + if (metadata) outputArgs.push("-map", input); const filters = [headFilter, channelFilter, buildAtempoFilter(playbackRate)].filter(Boolean); if (filters.length > 0) outputArgs.push("-af", filters.join(",")); return outputArgs; diff --git a/packages/engine/src/services/chunkEncoder.test.ts b/packages/engine/src/services/chunkEncoder.test.ts index 4433c23bcd9..a72772eae2f 100644 --- a/packages/engine/src/services/chunkEncoder.test.ts +++ b/packages/engine/src/services/chunkEncoder.test.ts @@ -681,6 +681,7 @@ describe("muxVideoWithAudio audio codec handling", () => { sampleRate: 48000, channels: 2, audioCodec: "aac", + audioTrack: 0, })); vi.resetModules(); vi.doMock("child_process", () => ({ spawn })); @@ -714,6 +715,7 @@ describe("muxVideoWithAudio audio codec handling", () => { sampleRate: 48000, channels: 2, audioCodec: "mp3", + audioTrack: 0, })); vi.resetModules(); vi.doMock("child_process", () => ({ spawn })); diff --git a/packages/engine/src/utils/ffprobe.ts b/packages/engine/src/utils/ffprobe.ts index 8f4228d1e82..81c4bd8b022 100644 --- a/packages/engine/src/utils/ffprobe.ts +++ b/packages/engine/src/utils/ffprobe.ts @@ -11,6 +11,7 @@ import { pixelFormatHasAlpha, videoStreamHasAlpha, } from "@hyperframes/parsers/media-alpha"; +import { playableAudioTrack } from "@hyperframes/core/audible-video"; import { urlInputArgs } from "./urlInput.js"; import { runFfmpeg } from "./runFfmpeg.js"; import { FFPROBE_PATH_ENV, getFfprobeBinary } from "./ffmpegBinaries.js"; @@ -238,6 +239,8 @@ export interface AudioMetadata { sampleRate: number; channels: number; audioCodec: string; + /** Which audio stream these fields describe: N in ffmpeg's `0:a:N` (`playableAudioTrack`). */ + audioTrack: number; bitrate?: number; /** Seconds after the file's media time 0 at which its last audio or video stream starts. */ latestStreamLeadSeconds?: number; @@ -1209,8 +1212,14 @@ export async function extractAudioMetadata( async function probeAudioMetadata(filePath: string, signal?: AbortSignal): Promise { const output = await probeMediaOutput(filePath, signal); - const audioStream = output.streams.find((s) => s.codec_type === "audio"); - if (!audioStream) throw new Error("[FFmpeg] No audio stream found"); + const audioTrack = playableAudioTrack(output.streams); + const audioStream = + audioTrack === undefined + ? undefined + : output.streams.filter((s) => s.codec_type === "audio")[audioTrack]; + if (audioTrack === undefined || !audioStream) { + throw new Error("[FFmpeg] No playable audio stream found"); + } let durationSeconds = output.format.duration ? parseFloat(output.format.duration) : 0; const streamDuration = audioStream.duration ? parseFloat(audioStream.duration) : undefined; @@ -1221,6 +1230,7 @@ async function probeAudioMetadata(filePath: string, signal?: AbortSignal): Promi if (sampleRate > 0) { const decoded = await probeDecodedAudioDuration( filePath, + audioTrack, sampleRate, audioStartSeconds(audioStream, output.format), durationSeconds, @@ -1237,6 +1247,7 @@ async function probeAudioMetadata(filePath: string, signal?: AbortSignal): Promi sampleRate, channels: audioStream.channels || 2, audioCodec, + audioTrack, bitrate: output.format.bit_rate ? parseInt(output.format.bit_rate) : undefined, latestStreamLeadSeconds: latestStreamLeadSeconds(output), audioStreamLeadSeconds: streamLeadSeconds(output, audioStream), @@ -1291,6 +1302,7 @@ function reconcileAudioDuration( */ async function probeDecodedAudioDuration( filePath: string, + audioTrack: number, sampleRate: number, startSeconds: number, containerDurationSeconds: number, @@ -1301,7 +1313,7 @@ async function probeDecodedAudioDuration( ): Promise<{ timestamp: number; nbSamples: number } | undefined> => { const args = [ "-select_streams", - "a:0", + `a:${audioTrack}`, "-show_entries", "frame=best_effort_timestamp_time,nb_samples", "-of", diff --git a/packages/studio-server/src/helpers/childProcess.windowsHide.test.ts b/packages/studio-server/src/helpers/childProcess.windowsHide.test.ts index 025363107c3..435614a7e09 100644 --- a/packages/studio-server/src/helpers/childProcess.windowsHide.test.ts +++ b/packages/studio-server/src/helpers/childProcess.windowsHide.test.ts @@ -45,6 +45,7 @@ describe("Studio child-process options", () => { spawnMock.mockReturnValue(proc); const peaks = decodeAudioPeaks("/tmp/audio.wav"); + await vi.waitFor(() => expect(spawnMock).toHaveBeenCalled()); proc.stdout.emit("data", Buffer.from(new Float32Array([0.5]).buffer)); proc.emit("close", 0); await peaks; diff --git a/packages/studio-server/src/helpers/iphoneClip.test-helpers.ts b/packages/studio-server/src/helpers/iphoneClip.test-helpers.ts new file mode 100644 index 00000000000..21b65de4f4c --- /dev/null +++ b/packages/studio-server/src/helpers/iphoneClip.test-helpers.ts @@ -0,0 +1,20 @@ +import { execFileSync } from "node:child_process"; +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +export function writeIphoneClip(ffmpeg: string, dir: string, seconds = 1): string { + const base = join(dir, "base.mov"); + execFileSync(ffmpeg, [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", `testsrc=s=64x64:d=${seconds}`], + ...["-f", "lavfi", "-i", `anoisesrc=d=${seconds},aformat=channel_layouts=4.0`], + ...["-f", "lavfi", "-i", `sine=frequency=1000:duration=${seconds}:sample_rate=48000`], + ...["-map", "0:v", "-map", "1:a", "-map", "2:a", "-c:v", "libx264"], + ...["-c:a:0", "pcm_s16le", "-c:a:1", "aac", "-movflags", "+faststart", base], + ]); + // Apple's spatial-audio fourcc on the PCM track (the first `sowt`, as faststart puts the header first). + const bytes = readFileSync(base); + bytes.write("apac", bytes.indexOf("sowt"), "latin1"); + const clip = join(dir, "iphone.mov"); + writeFileSync(clip, bytes); + return clip; +} diff --git a/packages/studio-server/src/helpers/loudness.test.ts b/packages/studio-server/src/helpers/loudness.test.ts index 638abb66c63..378f0543315 100644 --- a/packages/studio-server/src/helpers/loudness.test.ts +++ b/packages/studio-server/src/helpers/loudness.test.ts @@ -1,8 +1,14 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it } from "vitest"; +import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; +import { writeIphoneClip } from "./iphoneClip.test-helpers.js"; import { absoluteLoudnessPlan, audioTags, loudnessMeasureArgs, + measureAudio, updateAudioVolume, } from "./loudness.js"; @@ -204,3 +210,25 @@ describe("audioTags on hostile markup", () => { expect(audioTags(html).map((t) => t.id)).toEqual(["vo"]); }); }); + +describe("measureAudio on an iPhone clip", () => { + const ffmpeg = findFfBinary("ffmpeg"); + it.skipIf(!ffmpeg || !findFfBinary("ffprobe"))( + "measures the AAC voice, not the undecodable spatial track", + async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-loudness-")); + try { + const clip = writeIphoneClip(ffmpeg ?? "ffmpeg", dir, 2); + const measured = await measureAudio(ffmpeg ?? "ffmpeg", clip, { + mediaStart: 0, + duration: null, + }); + // ffmpeg's sine is 1/8 amplitude, about -21 LUFS. + expect(measured.integratedLufs).toBeGreaterThan(-26); + expect(measured.integratedLufs).toBeLessThan(-16); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/packages/studio-server/src/helpers/loudness.ts b/packages/studio-server/src/helpers/loudness.ts index f104894205f..cb4882b4eb0 100644 --- a/packages/studio-server/src/helpers/loudness.ts +++ b/packages/studio-server/src/helpers/loudness.ts @@ -6,6 +6,7 @@ import { formatAudioGain, MAX_AUDIO_GAIN_DB } from "@hyperframes/core/audio-gain import { decodeAuthoredAttribute } from "@hyperframes/parsers"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; import { readMediaOffsetSeconds } from "@hyperframes/parsers/media-duration"; +import { probeMediaMetadata } from "./mediaMetadata.js"; import { requestedProjectPath } from "./requestSubPath.js"; const execFileAsync = promisify(execFile); @@ -454,12 +455,13 @@ export function updateAudioVolume(html: string, id: string, volume: number): str * -61.8 LUFS: output-side `-t 4` reported -33.8 LUFS, input-side reports -61.8, * the same as physically cutting the file first. */ -export function loudnessMeasureArgs(file: string, tag: MeasuredWindow): string[] { +export function loudnessMeasureArgs(file: string, tag: MeasuredWindow, audioTrack = 0): string[] { const args = ["-hide_banner", "-nostats"]; if (tag.mediaStart > 0) args.push("-ss", String(tag.mediaStart)); const sourceSpan = tag.duration === null ? null : tag.duration * (tag.playbackRate ?? 1); if (sourceSpan !== null && sourceSpan > 0) args.push("-t", String(sourceSpan)); - args.push("-i", file, "-map", "0:a:0", "-vn", "-af", "ebur128=peak=true", "-f", "null", "-"); + args.push("-i", file, "-map", `0:a:${audioTrack}`); + args.push("-vn", "-af", "ebur128=peak=true", "-f", "null", "-"); return args; } @@ -471,7 +473,8 @@ export async function measureAudio( file: string, tag: MeasuredWindow, ): Promise { - const result = await execFileAsync(ffmpegPath, loudnessMeasureArgs(file, tag), { + const { audioTrack } = await probeMediaMetadata(file); + const result = await execFileAsync(ffmpegPath, loudnessMeasureArgs(file, tag, audioTrack), { encoding: "utf8", maxBuffer: 16 * 1024 * 1024, timeout: FFMPEG_TIMEOUT_MS, diff --git a/packages/studio-server/src/helpers/mediaMetadata.test.ts b/packages/studio-server/src/helpers/mediaMetadata.test.ts index 73006c0e4c4..0e614ca7435 100644 --- a/packages/studio-server/src/helpers/mediaMetadata.test.ts +++ b/packages/studio-server/src/helpers/mediaMetadata.test.ts @@ -102,6 +102,19 @@ describe("probeMediaMetadata", () => { })); expect(silent).toMatchObject({ kind: "video", hasAudio: false }); + // An iPhone spatial track alone plays nowhere, so it is not audio to drop. + const spatialOnly = await probeMediaMetadata("/tmp/iphone.mov", () => ({ + status: 0, + stdout: JSON.stringify({ + streams: [ + { codec_type: "video", codec_name: "hevc" }, + { codec_type: "audio", codec_name: "unknown" }, + ], + }), + stderr: "", + })); + expect(spatialOnly).toMatchObject({ kind: "video", hasAudio: false }); + // An audio file is not a video; the flag is a video question only. const audioFile = await probeMediaMetadata("/tmp/music.wav", () => ({ status: 0, diff --git a/packages/studio-server/src/helpers/mediaMetadata.ts b/packages/studio-server/src/helpers/mediaMetadata.ts index 78fff6cb5c1..2d530c07e18 100644 --- a/packages/studio-server/src/helpers/mediaMetadata.ts +++ b/packages/studio-server/src/helpers/mediaMetadata.ts @@ -5,6 +5,7 @@ import { parseFirstFrameColour, type ToneMapSourceColour, } from "@hyperframes/core"; +import { playableAudioTrack } from "@hyperframes/core/audible-video"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; import { videoStreamHasAlpha } from "@hyperframes/parsers/media-alpha"; @@ -77,6 +78,7 @@ export interface MediaMetadata { /** Video stream carries alpha (pix_fmt or the VP8/VP9 alpha_mode tag). */ hasAlpha?: boolean; durationSeconds?: number; + audioTrack?: number; probeError?: string; } @@ -243,7 +245,8 @@ export async function probeMediaMetadata( }); const metadata: MediaMetadata = { kind, color: classifyMediaColor(stream) }; if (kind === "video") { - metadata.hasAudio = (parsed.streams ?? []).some((item) => item.codec_type === "audio"); + metadata.audioTrack = playableAudioTrack(parsed.streams ?? []); + metadata.hasAudio = metadata.audioTrack !== undefined; metadata.hasAlpha = stream !== undefined && videoStreamHasAlpha(stream); const duration = Number(parsed.format?.duration); if (Number.isFinite(duration) && duration > 0) metadata.durationSeconds = duration; diff --git a/packages/studio-server/src/helpers/peakMap.decodeFailure.test.ts b/packages/studio-server/src/helpers/peakMap.decodeFailure.test.ts index a2f24c8c409..a51a2c0addd 100644 --- a/packages/studio-server/src/helpers/peakMap.decodeFailure.test.ts +++ b/packages/studio-server/src/helpers/peakMap.decodeFailure.test.ts @@ -21,7 +21,10 @@ describe("decodePeakMap, decode failure", () => { writeFileSync(file, `#!/bin/sh\n${body}\n`); chmodSync(file, 0o755); }; - script("ffprobe", "echo 2"); + script( + "ffprobe", + `echo '{"streams":[{"codec_type":"audio","codec_name":"aac","channels":2}]}'`, + ); script("ffmpeg", "head -c 38400 /dev/zero; exit 1"); await expect(decodePeakMap("truncated.mov")).rejects.toThrow(/exit 1/); }, diff --git a/packages/studio-server/src/helpers/peakMap.test.ts b/packages/studio-server/src/helpers/peakMap.test.ts index 6299574e757..b434a431063 100644 --- a/packages/studio-server/src/helpers/peakMap.test.ts +++ b/packages/studio-server/src/helpers/peakMap.test.ts @@ -11,6 +11,7 @@ import { decodePeakMap, finishPeaks, } from "./peakMap.js"; +import { writeIphoneClip } from "./iphoneClip.test-helpers.js"; function stereo(frames: Array<[number, number]>): Buffer { const buf = Buffer.alloc(frames.length * 8); @@ -66,7 +67,7 @@ describe("buildPeakMapCacheKey", () => { expect(a).toBe(buildPeakMapCacheKey("a/talk.mp4", { size: 10, mtimeMs: 1 })); expect(a).not.toBe(buildPeakMapCacheKey("a/talk.mp4", { size: 11, mtimeMs: 1 })); expect(a).not.toBe(buildPeakMapCacheKey("a/talk.mp4", { size: 10, mtimeMs: 2 })); - expect(a).toMatch(/^peaks-v2_a_talk\.mp4_10-1\.json$/); + expect(a).toMatch(/^peaks-v3_a_talk\.mp4_10-1\.json$/); }); }); @@ -146,4 +147,18 @@ describe("decodePeakMap", () => { } }, ); + + it.skipIf(!ffmpeg)( + "reads an iPhone clip's AAC voice, not its undecodable spatial track", + async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-peaks-")); + try { + const peaks = await decodePeakMap(writeIphoneClip(ffmpeg ?? "ffmpeg", dir)); + // ffmpeg's sine peaks at 1/8. + expect(Math.max(...peaks)).toBeGreaterThan(0.1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, + ); }); diff --git a/packages/studio-server/src/helpers/peakMap.ts b/packages/studio-server/src/helpers/peakMap.ts index 743a94b2c43..a4fc60a711e 100644 --- a/packages/studio-server/src/helpers/peakMap.ts +++ b/packages/studio-server/src/helpers/peakMap.ts @@ -1,11 +1,12 @@ import { execFile, spawn } from "node:child_process"; import { promisify } from "node:util"; +import { playableAudioTrack } from "@hyperframes/core/audible-video"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; export const PEAK_BIN_SECONDS = 0.05; const PEAK_SAMPLE_RATE = 48_000; const execFileAsync = promisify(execFile); -const PEAK_MAP_CACHE_VERSION = "peaks-v2"; +const PEAK_MAP_CACHE_VERSION = "peaks-v3"; /** * Running max-abs per bin over interleaved f32le at the source's own channel @@ -61,30 +62,33 @@ export function finishPeaks(acc: PeakAccumulator): number[] { return acc.bins.map((peak) => Number(peak.toFixed(4))); } -async function probeChannels(mediaPath: string): Promise { +async function probeAudioTrack(mediaPath: string): Promise<{ track: number; channels: number }> { const { stdout } = await execFileAsync( findFfBinary("ffprobe") ?? "ffprobe", [ "-v", "error", "-select_streams", - "a:0", + "a", "-show_entries", - "stream=channels", + "stream=codec_type,codec_name,channels", "-of", - "csv=p=0", + "json", "--", mediaPath, ], { encoding: "utf8", timeout: 30_000, windowsHide: true }, ); - const channels = Number.parseInt(stdout.trim(), 10); - if (!Number.isFinite(channels) || channels < 1) throw new Error("no audio stream"); - return channels; + const streams: Array<{ codec_type?: string; codec_name?: string; channels?: number }> = + JSON.parse(stdout).streams ?? []; + const track = playableAudioTrack(streams); + const channels = track === undefined ? undefined : streams[track]?.channels; + if (track === undefined || !channels || channels < 1) throw new Error("no audio stream"); + return { track, channels }; } export async function decodePeakMap(mediaPath: string): Promise { - const channels = await probeChannels(mediaPath); + const { track, channels } = await probeAudioTrack(mediaPath); return new Promise((resolvePromise, reject) => { const proc = spawn( findFfBinary("ffmpeg") ?? "ffmpeg", @@ -94,7 +98,7 @@ export async function decodePeakMap(mediaPath: string): Promise { "-i", mediaPath, "-map", - "0:a:0", + `0:a:${track}`, "-vn", "-ar", String(PEAK_SAMPLE_RATE), diff --git a/packages/studio-server/src/helpers/proxyTranscoder.test.ts b/packages/studio-server/src/helpers/proxyTranscoder.test.ts index 6982a19c578..9c216e7342c 100644 --- a/packages/studio-server/src/helpers/proxyTranscoder.test.ts +++ b/packages/studio-server/src/helpers/proxyTranscoder.test.ts @@ -16,6 +16,7 @@ import { basename, join } from "node:path"; import { hdrToSdrToneMapFilter } from "@hyperframes/core"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { writeIphoneClip } from "./iphoneClip.test-helpers.js"; const FFMPEG_PATH = "/usr/bin/ffmpeg"; const realFfmpeg = findFfBinary("ffmpeg"); @@ -774,6 +775,68 @@ describe("resolveProxy", () => { 60_000, ); + it.skipIf(!realFfmpeg || !findFfBinary("ffprobe"))( + "proxies an iPhone clip with its AAC voice when its first audio track cannot be decoded", + async () => { + vi.resetModules(); + vi.doUnmock("./mediaMetadata.js"); + const { resolveProxy } = await import("./proxyTranscoder.js"); + const projectDir = tmpProject(); + const sourcePath = writeIphoneClip(realFfmpeg!, projectDir, 0.5); + + const proxyPath = await resolveProxy(projectDir, sourcePath); + const streams = execFileSync( + findFfBinary("ffprobe")!, + ["-v", "error", "-show_entries", "stream=codec_name", "-of", "csv=p=0", "--", proxyPath], + { encoding: "utf8" }, + ); + expect(streams.trim().split("\n")).toEqual(["h264", "aac"]); + }, + 60_000, + ); + + it.skipIf(!realFfmpeg || !findFfBinary("ffprobe") || process.platform === "win32")( + "keeps the sound of a symlinked clip whose target has no extension", + async () => { + vi.resetModules(); + vi.doUnmock("./mediaMetadata.js"); + const { resolveProxy } = await import("./proxyTranscoder.js"); + const projectDir = tmpProject(); + const blob = join(projectDir, "blob"); + execFileSync(realFfmpeg!, [ + "-v", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=size=64x64:rate=10:duration=0.5", + "-f", + "lavfi", + "-i", + "sine=frequency=440:duration=0.5", + "-c:v", + "libx264", + "-c:a", + "aac", + "-f", + "mov", + "-y", + blob, + ]); + const clip = join(projectDir, "clip.mov"); + symlinkSync(blob, clip); + + const proxyPath = await resolveProxy(projectDir, clip); + const streams = execFileSync( + findFfBinary("ffprobe")!, + ["-v", "error", "-show_entries", "stream=codec_name", "-of", "csv=p=0", "--", proxyPath], + { encoding: "utf8" }, + ); + expect(streams.trim().split("\n")).toEqual(["h264", "aac"]); + }, + 60_000, + ); + it("dedupes two concurrent same-key calls to one spawn", async () => { const { spawn, calls } = createSpawnSpy(); const { resolveProxy } = await loadModule(spawn, FFMPEG_PATH); diff --git a/packages/studio-server/src/helpers/proxyTranscoder.ts b/packages/studio-server/src/helpers/proxyTranscoder.ts index 18b6e823119..f53049d30d4 100644 --- a/packages/studio-server/src/helpers/proxyTranscoder.ts +++ b/packages/studio-server/src/helpers/proxyTranscoder.ts @@ -19,7 +19,7 @@ import { } from "@hyperframes/core"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; import { alphaDecoderFor } from "@hyperframes/parsers/media-alpha"; -import { probeFirstFrameColour, probeMediaMetadata } from "./mediaMetadata.js"; +import { probeFirstFrameColour, probeMediaMetadata, type MediaMetadata } from "./mediaMetadata.js"; import { cleanupProxyCache } from "./proxyCache.js"; import { PROXY_VARIANT_CONFIG, type ProxyVariant } from "./mediaCodecMap.js"; import { mkdirWithinProject, realpath, realProjectRoot } from "./safePath.js"; @@ -574,6 +574,13 @@ const VP8_ARGS = [ "libopus", ]; +/** The probed video plus its playable audio track; ffmpeg's own pick only when the probe failed. */ +function streamMaps(metadata: MediaMetadata): string[] { + if (metadata.probeError) return []; + const audio = metadata.audioTrack === undefined ? [] : ["-map", `0:a:${metadata.audioTrack}`]; + return ["-map", "0:V:0?", ...audio]; +} + async function buildVideoFilter( ffmpegPath: string, sourcePath: string, @@ -629,6 +636,7 @@ async function runFfmpeg( "-i", sourcePath, ...(range ? ["-t", String(range.lengthSeconds)] : []), + ...streamMaps(metadata), "-vf", videoFilter, ...(variant === "vp8" ? VP8_ARGS : H264_ARGS), diff --git a/packages/studio-server/src/helpers/waveform.test.ts b/packages/studio-server/src/helpers/waveform.test.ts index 45478fc25cb..cef0a205edd 100644 --- a/packages/studio-server/src/helpers/waveform.test.ts +++ b/packages/studio-server/src/helpers/waveform.test.ts @@ -5,6 +5,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; import { buildWaveformCacheKey, decodeAudioPeaks } from "./waveform.js"; +import { writeIphoneClip } from "./iphoneClip.test-helpers.js"; describe("buildWaveformCacheKey", () => { it("is stable for the same file", () => { @@ -71,4 +72,17 @@ describe("decodeAudioPeaks on a video file", () => { rmSync(dir, { recursive: true, force: true }); } }); + + it.skipIf(!ffmpeg || !findFfBinary("ffprobe"))( + "reads an iPhone clip's AAC voice, not its undecodable spatial track", + async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-wave-video-")); + try { + const peaks = await decodeAudioPeaks(writeIphoneClip(ffmpeg ?? "ffmpeg", dir)); + expect(Math.max(...peaks)).toBeCloseTo(1, 3); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, + ); }); diff --git a/packages/studio-server/src/helpers/waveform.ts b/packages/studio-server/src/helpers/waveform.ts index 997f6a15065..82efe751258 100644 --- a/packages/studio-server/src/helpers/waveform.ts +++ b/packages/studio-server/src/helpers/waveform.ts @@ -11,10 +11,11 @@ import { } from "node:fs"; import { dirname, join } from "node:path"; import { findFfBinary } from "@hyperframes/parsers/ff-binaries"; +import { probeMediaMetadata } from "./mediaMetadata.js"; const SAMPLE_RATE = 4000; const PEAK_COUNT = 4000; -const WAVEFORM_CACHE_VERSION = "v2"; +const WAVEFORM_CACHE_VERSION = "v3"; /** * Cache filename for one asset's peaks, keyed on its content as well as its name. @@ -58,13 +59,16 @@ function computePeaks(floats: Float32Array, count: number): number[] { return peaks.map((p) => p / maxPeak); } -export function decodeAudioPeaks(audioPath: string): Promise { +export async function decodeAudioPeaks(audioPath: string): Promise { + const { audioTrack } = await probeMediaMetadata(audioPath); + const map = audioTrack === undefined ? [] : ["-map", `0:a:${audioTrack}`]; return new Promise((resolvePromise, reject) => { const proc = spawn( findFfBinary("ffmpeg") ?? "ffmpeg", [ "-i", audioPath, + ...map, "-af", "atrim=start_sample=1152", "-f", From 7a270abdbdf624c10b06c81461bccdb84893b303 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sun, 25 Oct 2026 19:33:02 -0400 Subject: [PATCH 2/3] refactor(studio-server): trim a comment the comment-share check rejects --- packages/studio-server/src/helpers/proxyTranscoder.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/studio-server/src/helpers/proxyTranscoder.ts b/packages/studio-server/src/helpers/proxyTranscoder.ts index f53049d30d4..0da6366419a 100644 --- a/packages/studio-server/src/helpers/proxyTranscoder.ts +++ b/packages/studio-server/src/helpers/proxyTranscoder.ts @@ -574,7 +574,6 @@ const VP8_ARGS = [ "libopus", ]; -/** The probed video plus its playable audio track; ffmpeg's own pick only when the probe failed. */ function streamMaps(metadata: MediaMetadata): string[] { if (metadata.probeError) return []; const audio = metadata.audioTrack === undefined ? [] : ["-map", `0:a:${metadata.audioTrack}`]; From e711c12835b0610714b51af65f736800e6611acd Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sun, 25 Oct 2026 20:12:47 -0400 Subject: [PATCH 3/3] fix(studio-server): only a probed video gets stream maps in the preview copy --- packages/studio-server/src/helpers/proxyTranscoder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/studio-server/src/helpers/proxyTranscoder.ts b/packages/studio-server/src/helpers/proxyTranscoder.ts index 0da6366419a..16100520dfb 100644 --- a/packages/studio-server/src/helpers/proxyTranscoder.ts +++ b/packages/studio-server/src/helpers/proxyTranscoder.ts @@ -575,7 +575,7 @@ const VP8_ARGS = [ ]; function streamMaps(metadata: MediaMetadata): string[] { - if (metadata.probeError) return []; + if (metadata.probeError || metadata.kind !== "video") return []; const audio = metadata.audioTrack === undefined ? [] : ["-map", `0:a:${metadata.audioTrack}`]; return ["-map", "0:V:0?", ...audio]; }