diff --git a/.changeset/flexfec-subscription.md b/.changeset/flexfec-subscription.md new file mode 100644 index 000000000..27599e3ab --- /dev/null +++ b/.changeset/flexfec-subscription.md @@ -0,0 +1,5 @@ +--- +"client-sdk-android": minor +--- + +Add `LiveKit.enableFlexFec` to negotiate FlexFEC, and `RemoteTrackPublication.setFecProtection` to request FlexFEC repair packets from the server for a subscribed video track, with disabled, adaptive, fixed and minimum modes and low, medium and high levels. diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/LiveKit.kt b/livekit-android-sdk/src/main/java/io/livekit/android/LiveKit.kt index c281b5aa8..967532ea6 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/LiveKit.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/LiveKit.kt @@ -64,6 +64,27 @@ object LiveKit { @JvmStatic var enableWebRTCLogging: Boolean = false + /** + * Enables FlexFEC forward error correction for video. + * + * When enabled, connections negotiate `flexfec-03`, which lets subscribed video tracks + * receive repair packets from the server, see + * [io.livekit.android.room.track.RemoteTrackPublication.setFecProtection]. + * Video tracks published without simulcast also send FlexFEC, at a rate WebRTC adapts to the + * reported loss; WebRTC does not protect simulcast tracks with FlexFEC. + * Disabled by default. + * + * This is a process-wide WebRTC setting, so it must be set before the first [Room] is + * created or [init] is called; later changes have no effect. + * + * ``` + * LiveKit.enableFlexFec = true + * val room = LiveKit.create(appContext) + * ``` + */ + @JvmStatic + var enableFlexFec: Boolean = false + /** * Certain WebRTC classes need to be initialized prior to use. * diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt b/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt index 91e3ad0ff..6f1aafbf6 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt @@ -91,6 +91,10 @@ internal object RTCModule { * it. */ private const val DEFAULT_FIELD_TRIALS = "WebRTC-IceHandshakeDtls/Enabled/" + private const val FLEXFEC_FIELD_TRIALS = "WebRTC-FlexFEC-03-Advertised/Enabled/WebRTC-FlexFEC-03/Enabled/" + + private val fieldTrials: String + get() = if (LiveKit.enableFlexFec) DEFAULT_FIELD_TRIALS + FLEXFEC_FIELD_TRIALS else DEFAULT_FIELD_TRIALS /** * Certain classes require libwebrtc to be initialized prior to use. @@ -122,7 +126,7 @@ internal object RTCModule { PeerConnectionFactory.InitializationOptions .builder(appContext) .setNativeLibraryName("lkjingle_peerconnection_so") - .setFieldTrials(DEFAULT_FIELD_TRIALS) + .setFieldTrials(fieldTrials) .setInjectableLogger( { s, severity, s2 -> if (!LiveKit.enableWebRTCLogging) { diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/SignalClient.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/SignalClient.kt index 875b52051..2903c71df 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/SignalClient.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/SignalClient.kt @@ -529,12 +529,14 @@ constructor( sendRequest(request) } + @Suppress("LongParameterList") fun sendUpdateTrackSettings( sid: String, disabled: Boolean, videoDimensions: Track.Dimensions?, videoQuality: LivekitModels.VideoQuality?, fps: Int?, + fecProtection: LivekitRtc.FECProtection? = null, ) { val trackSettings = LivekitRtc.UpdateTrackSettings.newBuilder() .addTrackSids(sid) @@ -553,6 +555,10 @@ constructor( if (fps != null) { setFps(fps) } + + if (fecProtection != null) { + fec = fecProtection + } } val request = LivekitRtc.SignalRequest.newBuilder() diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/FecProtection.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/FecProtection.kt new file mode 100644 index 000000000..d9807d076 --- /dev/null +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/FecProtection.kt @@ -0,0 +1,92 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.room.track + +import livekit.LivekitRtc + +/** + * Proactive FlexFEC protection the server adds to a subscribed video track. + * + * The server sends repair packets alongside the track's media, so the receiver can recover lost + * packets without waiting for a retransmission. + * + * @see RemoteTrackPublication.setFecProtection + */ +sealed class FecProtection { + /** No repair packets. This is the default. */ + data object Disabled : FecProtection() + + /** + * Protection follows the observed packet loss, using the same algorithm as libwebrtc. + */ + data object Adaptive : FecProtection() + + /** + * Constant protection at [level], regardless of packet loss. + */ + data class Fixed(val level: FecProtectionLevel) : FecProtection() + + /** + * Protection never drops below [level]. It rises adaptively under higher packet loss, and + * returns to [level] once the loss subsides. + */ + data class Minimum(val level: FecProtectionLevel) : FecProtection() + + internal fun toProto(): LivekitRtc.FECProtection { + val builder = LivekitRtc.FECProtection.newBuilder() + when (this) { + Disabled -> builder.mode = LivekitRtc.FECProtectionMode.FEC_MODE_DISABLED + Adaptive -> builder.mode = LivekitRtc.FECProtectionMode.FEC_MODE_ADAPTIVE + is Fixed -> { + builder.mode = LivekitRtc.FECProtectionMode.FEC_MODE_FIXED + builder.level = level.toProto() + } + + is Minimum -> { + builder.mode = LivekitRtc.FECProtectionMode.FEC_MODE_MINIMUM + builder.level = level.toProto() + } + } + return builder.build() + } +} + +/** + * The share of packets spent on FlexFEC repairs, used by [FecProtection.Fixed] and + * [FecProtection.Minimum]. + * + * Frames that span only a few packets get proportionally more repairs. + */ +enum class FecProtectionLevel { + /** Approximately 15% repair packet overhead. */ + LOW, + + /** Approximately 25% repair packet overhead. */ + MEDIUM, + + /** Approximately 35% repair packet overhead. */ + HIGH, + ; + + internal fun toProto(): LivekitRtc.FECProtectionLevel { + return when (this) { + LOW -> LivekitRtc.FECProtectionLevel.FEC_LEVEL_LOW + MEDIUM -> LivekitRtc.FECProtectionLevel.FEC_LEVEL_MEDIUM + HIGH -> LivekitRtc.FECProtectionLevel.FEC_LEVEL_HIGH + } + } +} diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/RemoteTrackPublication.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/RemoteTrackPublication.kt index 0f04a588d..6537e2481 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/RemoteTrackPublication.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/RemoteTrackPublication.kt @@ -1,5 +1,5 @@ /* - * Copyright 2023-2025 LiveKit, Inc. + * Copyright 2023-2026 LiveKit, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -75,6 +75,15 @@ class RemoteTrackPublication( private var videoDimensions: Track.Dimensions? = null private var fps: Int? = null + /** + * The FlexFEC protection requested with [setFecProtection]. + */ + val fecProtection: FecProtection + get() = requestedFecProtection ?: FecProtection.Disabled + + // null until requested, so updates leave the server's setting unchanged + private var requestedFecProtection: FecProtection? = null + var subscriptionAllowed: Boolean = true internal set @@ -204,6 +213,29 @@ class RemoteTrackPublication( sendUpdateTrackSettings.invoke() } + /** + * Requests proactive FlexFEC repair packets from the server for this video track. + * + * Repairs let the receiver recover lost packets without waiting for a retransmission, at the + * cost of extra bandwidth. The protection changes without renegotiation, and is applied even + * when the track's quality is managed by adaptive stream. + * + * Repairs are only sent when the server has downstream FlexFEC enabled and FlexFEC was + * negotiated, which requires [io.livekit.android.LiveKit.enableFlexFec]. Otherwise the request + * has no effect. + * + * ``` + * publication.setFecProtection(FecProtection.Minimum(FecProtectionLevel.MEDIUM)) + * ``` + */ + fun setFecProtection(protection: FecProtection) { + if (!subscribed || track !is VideoTrack || protection == requestedFecProtection) { + return + } + requestedFecProtection = protection + sendUpdateTrackSettings.invoke() + } + private fun handleVisibilityChanged(isVisible: Boolean) { disabled = !isVisible sendUpdateTrackSettings.invoke() @@ -237,6 +269,7 @@ class RemoteTrackPublication( videoDimensions, videoQuality?.toProto(), fps, + requestedFecProtection?.toProto(), ) } diff --git a/livekit-android-test/src/androidTest/AndroidManifest.xml b/livekit-android-test/src/androidTest/AndroidManifest.xml new file mode 100644 index 000000000..5f9b23641 --- /dev/null +++ b/livekit-android-test/src/androidTest/AndroidManifest.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/livekit-android-test/src/androidTest/java/io/livekit/android/room/track/FlexFecOnDeviceTest.kt b/livekit-android-test/src/androidTest/java/io/livekit/android/room/track/FlexFecOnDeviceTest.kt new file mode 100644 index 000000000..2d7358f9f --- /dev/null +++ b/livekit-android-test/src/androidTest/java/io/livekit/android/room/track/FlexFecOnDeviceTest.kt @@ -0,0 +1,198 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.room.track + +import android.util.Log +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import io.livekit.android.LiveKit +import io.livekit.android.RoomOptions +import io.livekit.android.events.RoomEvent +import io.livekit.android.room.Room +import io.livekit.android.util.LoggingLevel +import kotlinx.coroutines.async +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.filterIsInstance +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withTimeout +import livekit.org.webrtc.RTCStatsReport +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Assume.assumeTrue +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.coroutines.resume + +/** + * Subscribes to a video track published by another participant on a real LiveKit server, and + * checks that the server sends FlexFEC repair packets as requested through + * [RemoteTrackPublication.setFecProtection]. + * + * Needs a server with `rtc.flexfec.downstream_enabled` and a participant already publishing video + * in the room, so it is skipped unless the instrumentation arguments `lkUrl` and `lkToken` are + * given: + * + * ``` + * ./gradlew :livekit-android-test:connectedDebugAndroidTest \ + * -Pandroid.testInstrumentationRunnerArguments.class=io.livekit.android.room.track.FlexFecOnDeviceTest \ + * -Pandroid.testInstrumentationRunnerArguments.lkUrl=ws://192.168.1.2:7880 \ + * -Pandroid.testInstrumentationRunnerArguments.lkToken= + * ``` + * + * Repairs are counted per frame and rounded like libwebrtc's `NumFecPackets`, so the measured + * ratios depend on the frame sizes and only approximate the nominal 15/25/35%. The highest layer + * is subscribed throughout to keep the frame sizes comparable between modes. No loss is + * simulated, so adaptive protection is only checked to stay below a fixed level. + */ +@RunWith(AndroidJUnit4::class) +class FlexFecOnDeviceTest { + + private companion object { + const val TAG = "FlexFecOnDeviceTest" + const val SUBSCRIBE_TIMEOUT_MS = 30_000L + const val SETTLE_MS = 2_000L + const val MEASURE_MS = 6_000L + const val STATS_POLL_MS = 500L + } + + private data class Counters(val mediaPackets: Long, val fecPackets: Long, val fecDiscarded: Long) + + private data class Window(val mediaPackets: Long, val fecPackets: Long, val fecDiscarded: Long) { + val fecRatio: Double + get() = if (mediaPackets == 0L) 0.0 else fecPackets.toDouble() / mediaPackets + } + + private lateinit var room: Room + private lateinit var publication: RemoteTrackPublication + private lateinit var trackSid: String + + @Before + fun setUp(): Unit = runBlocking { + val args = InstrumentationRegistry.getArguments() + val url = args.getString("lkUrl") + val token = args.getString("lkToken") + assumeTrue("lkUrl and lkToken instrumentation arguments are required", url != null && token != null) + + LiveKit.loggingLevel = LoggingLevel.INFO + LiveKit.enableFlexFec = true + + val context = InstrumentationRegistry.getInstrumentation().targetContext + room = LiveKit.create(context, RoomOptions(adaptiveStream = false)) + + withTimeout(SUBSCRIBE_TIMEOUT_MS) { + val subscribed = room.events.events + .filterIsInstance() + val connect = async { subscribed.first { it.track is VideoTrack } } + room.connect(url!!, token!!) + val event = connect.await() + publication = event.publication as RemoteTrackPublication + trackSid = event.publication.sid + } + Log.i(TAG, "subscribed to video track $trackSid") + + publication.setVideoQuality(VideoQuality.HIGH) + withTimeout(SUBSCRIBE_TIMEOUT_MS) { + while (readCounters()?.mediaPackets?.let { it > 0 } != true) { + delay(STATS_POLL_MS) + } + } + } + + @After + fun tearDown() { + if (::room.isInitialized) { + room.disconnect() + room.release() + } + } + + @Test + fun serverSendsRepairsAsRequested(): Unit = runBlocking { + val disabledAtStart = measure("initial (not requested)") + assertTrue("media is flowing", disabledAtStart.mediaPackets > 0) + assertEquals("no repairs before FEC is requested", 0L, disabledAtStart.fecPackets) + + val fixedLow = measure(FecProtection.Fixed(FecProtectionLevel.LOW)) + val fixedMedium = measure(FecProtection.Fixed(FecProtectionLevel.MEDIUM)) + val fixedHigh = measure(FecProtection.Fixed(FecProtectionLevel.HIGH)) + val minimumMedium = measure(FecProtection.Minimum(FecProtectionLevel.MEDIUM)) + val adaptive = measure(FecProtection.Adaptive) + val disabled = measure(FecProtection.Disabled) + + assertTrue("media keeps flowing", disabled.mediaPackets > 0) + assertTrue("fixed LOW sends repairs", fixedLow.fecRatio >= 0.10) + assertTrue("fixed MEDIUM protects at least as much as LOW", fixedMedium.fecRatio >= fixedLow.fecRatio * 0.9) + assertTrue("fixed HIGH protects more than LOW", fixedHigh.fecRatio > fixedLow.fecRatio) + assertTrue("fixed HIGH protects at least 20%", fixedHigh.fecRatio >= 0.20) + assertEquals( + "minimum MEDIUM matches fixed MEDIUM without loss", + fixedMedium.fecRatio, + minimumMedium.fecRatio, + 0.05, + ) + assertTrue("adaptive without loss stays below fixed LOW", adaptive.fecRatio < fixedLow.fecRatio) + assertEquals("disabling stops the repairs", 0L, disabled.fecPackets) + listOf(fixedLow, fixedMedium, fixedHigh, minimumMedium).forEach { + assertEquals("repairs are not discarded", 0L, it.fecDiscarded) + } + } + + private suspend fun measure(protection: FecProtection): Window { + publication.setFecProtection(protection) + assertEquals(protection, publication.fecProtection) + return measure(protection.toString()) + } + + private suspend fun measure(label: String): Window { + delay(SETTLE_MS) + val start = requireNotNull(readCounters()) { "no inbound video stats" } + delay(MEASURE_MS) + val end = requireNotNull(readCounters()) { "no inbound video stats" } + val window = Window( + mediaPackets = end.mediaPackets - start.mediaPackets, + fecPackets = end.fecPackets - start.fecPackets, + fecDiscarded = end.fecDiscarded - start.fecDiscarded, + ) + Log.i( + TAG, + "$label: media=${window.mediaPackets} fec=${window.fecPackets} " + + "discarded=${window.fecDiscarded} ratio=${"%.3f".format(window.fecRatio)}", + ) + return window + } + + private suspend fun readCounters(): Counters? { + val report = suspendCancellableCoroutine { cont -> + room.getSubscriberRTCStats { cont.resume(it) } + } + val inbound = report.statsMap.values.firstOrNull { + it.type == "inbound-rtp" && it.members["kind"] == "video" + } + val members = inbound?.members ?: return null + return Counters( + mediaPackets = members.long("packetsReceived"), + fecPackets = members.long("fecPacketsReceived"), + fecDiscarded = members.long("fecPacketsDiscarded"), + ) + } + + private fun Map.long(key: String): Long = (this[key] as? Number)?.toLong() ?: 0L +} diff --git a/livekit-android-test/src/androidTest/java/io/livekit/android/room/track/FlexFecPublishOnDeviceTest.kt b/livekit-android-test/src/androidTest/java/io/livekit/android/room/track/FlexFecPublishOnDeviceTest.kt new file mode 100644 index 000000000..18e36072e --- /dev/null +++ b/livekit-android-test/src/androidTest/java/io/livekit/android/room/track/FlexFecPublishOnDeviceTest.kt @@ -0,0 +1,222 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.room.track + +import android.content.Context +import android.os.SystemClock +import android.util.Log +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import io.livekit.android.LiveKit +import io.livekit.android.RoomOptions +import io.livekit.android.room.Room +import io.livekit.android.room.participant.VideoTrackPublishOptions +import io.livekit.android.util.LoggingLevel +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import livekit.org.webrtc.CapturerObserver +import livekit.org.webrtc.JavaI420Buffer +import livekit.org.webrtc.SurfaceTextureHelper +import livekit.org.webrtc.VideoCapturer +import livekit.org.webrtc.VideoFrame +import org.junit.After +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Assume.assumeTrue +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import java.util.concurrent.Executors +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.TimeUnit +import kotlin.random.Random + +/** + * Publishes a synthetic video track to a real LiveKit server with [LiveKit.enableFlexFec], and + * checks how FlexFEC is negotiated on the publisher peer connection. + * + * libwebrtc only protects a single media stream with FlexFEC, so a FlexFEC SSRC is only signaled + * when the track is published without simulcast. The repairs themselves follow libwebrtc's + * loss-based protection, so none are sent without loss, and the publisher's stats do not count + * them. To see them arrive, run against a server with `rtc.flexfec.upstream_enabled`, add upstream + * loss, and watch the server's `livekit_fec_total{direction="incoming"}` metric while the track is + * held for `lkPublishSeconds`. + * + * Skipped unless the instrumentation arguments `lkUrl` and `lkPublisherToken` are given: + * + * ``` + * ./gradlew :livekit-android-test:connectedDebugAndroidTest \ + * -Pandroid.testInstrumentationRunnerArguments.class=io.livekit.android.room.track.FlexFecPublishOnDeviceTest \ + * -Pandroid.testInstrumentationRunnerArguments.lkUrl=ws://192.168.1.2:7880 \ + * -Pandroid.testInstrumentationRunnerArguments.lkPublisherToken= \ + * -Pandroid.testInstrumentationRunnerArguments.lkPublishSeconds=30 + * ``` + */ +@RunWith(AndroidJUnit4::class) +class FlexFecPublishOnDeviceTest { + + private companion object { + const val TAG = "FlexFecPublishTest" + const val WIDTH = 640 + const val HEIGHT = 480 + const val FPS = 30 + const val NEGOTIATION_MS = 3_000L + const val STATS_INTERVAL_MS = 5_000L + const val DEFAULT_PUBLISH_SECONDS = 10L + } + + private data class PublisherSdp(val offer: String, val answer: String) { + val offersFlexFec: Boolean + get() = offer.lines().any { it.startsWith("a=rtpmap:") && "flexfec-03" in it } + val signalsFlexFecSsrc: Boolean + get() = offer.lines().any { it.startsWith("a=ssrc-group:FEC-FR") } + val answersFlexFec: Boolean + get() = answer.lines().any { it.startsWith("a=rtpmap:") && "flexfec-03" in it } + } + + private lateinit var room: Room + private lateinit var url: String + private lateinit var token: String + private var publishSeconds = DEFAULT_PUBLISH_SECONDS + + @Before + fun setUp() { + val args = InstrumentationRegistry.getArguments() + val url = args.getString("lkUrl") + val token = args.getString("lkPublisherToken") + assumeTrue("lkUrl and lkPublisherToken instrumentation arguments are required", url != null && token != null) + this.url = url!! + this.token = token!! + publishSeconds = args.getString("lkPublishSeconds")?.toLongOrNull() ?: DEFAULT_PUBLISH_SECONDS + + LiveKit.loggingLevel = LoggingLevel.INFO + LiveKit.enableFlexFec = true + } + + @After + fun tearDown() { + if (::room.isInitialized) { + room.disconnect() + room.release() + } + } + + @Test + fun withoutSimulcastSignalsFlexFec(): Unit = runBlocking { + val sdp = publish(simulcast = false) + + assertTrue("offer includes flexfec-03", sdp.offersFlexFec) + assertTrue("offer signals a FEC-FR SSRC", sdp.signalsFlexFecSsrc) + assertTrue("server accepts flexfec-03", sdp.answersFlexFec) + } + + @Test + fun withSimulcastDoesNotSignalFlexFec(): Unit = runBlocking { + val sdp = publish(simulcast = true) + + assertFalse("libwebrtc only protects a single stream with FlexFEC", sdp.signalsFlexFecSsrc) + } + + private suspend fun publish(simulcast: Boolean): PublisherSdp { + val context = InstrumentationRegistry.getInstrumentation().targetContext + room = LiveKit.create(context, RoomOptions(adaptiveStream = false)) + room.connect(url, token) + + val local = room.localParticipant + val track = local.createVideoTrack("synthetic", NoiseCapturer()) + track.startCapture() + local.publishVideoTrack(track, VideoTrackPublishOptions(simulcast = simulcast)) + delay(NEGOTIATION_MS) + + val sdp = requireNotNull( + room.engine.transport?.publisher?.withPeerConnection { + PublisherSdp(localDescription?.description.orEmpty(), remoteDescription?.description.orEmpty()) + }, + ) { "publisher peer connection is closed" } + Log.i( + TAG, + "simulcast=$simulcast offersFlexFec=${sdp.offersFlexFec} " + + "signalsFlexFecSsrc=${sdp.signalsFlexFecSsrc} answersFlexFec=${sdp.answersFlexFec}", + ) + + val end = SystemClock.elapsedRealtime() + publishSeconds * 1000 + var packetsSent = 0L + while (SystemClock.elapsedRealtime() < end) { + delay(STATS_INTERVAL_MS) + val stats = track.getRTCStats() ?: continue + val outbound = stats.statsMap.values.filter { it.type == "outbound-rtp" } + packetsSent = outbound.sumOf { (it.members["packetsSent"] as? Number)?.toLong() ?: 0L } + val retransmitted = outbound.sumOf { (it.members["retransmittedPacketsSent"] as? Number)?.toLong() ?: 0L } + val fractionLost = stats.statsMap.values + .filter { it.type == "remote-inbound-rtp" } + .map { it.members["fractionLost"] } + Log.i( + TAG, + "simulcast=$simulcast packetsSent=$packetsSent retransmitted=$retransmitted " + + "reportedFractionLost=$fractionLost", + ) + } + track.stopCapture() + + assertTrue("media is sent", packetsSent > 0) + return sdp + } + + /** + * Pushes noisy I420 frames, so the encoder produces multi-packet frames without a camera. + */ + private class NoiseCapturer : VideoCapturer { + private var observer: CapturerObserver? = null + private var executor: ScheduledExecutorService? = null + private val random = Random(0) + + override fun initialize(helper: SurfaceTextureHelper?, context: Context?, observer: CapturerObserver) { + this.observer = observer + } + + override fun startCapture(width: Int, height: Int, framerate: Int) { + observer?.onCapturerStarted(true) + executor = Executors.newSingleThreadScheduledExecutor().also { + it.scheduleAtFixedRate(::pushFrame, 0, 1000L / FPS, TimeUnit.MILLISECONDS) + } + } + + private fun pushFrame() { + val buffer = JavaI420Buffer.allocate(WIDTH, HEIGHT) + val luma = ByteArray(buffer.dataY.capacity()) + random.nextBytes(luma) + buffer.dataY.put(luma) + val frame = VideoFrame(buffer, 0, System.nanoTime()) + observer?.onFrameCaptured(frame) + frame.release() + } + + override fun stopCapture() { + executor?.shutdownNow() + executor = null + observer?.onCapturerStopped() + } + + override fun changeCaptureFormat(width: Int, height: Int, framerate: Int) = Unit + + override fun dispose() { + stopCapture() + } + + override fun isScreencast(): Boolean = false + } +} diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/track/RemoteTrackPublicationTest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/track/RemoteTrackPublicationTest.kt index 77a433452..f28996f73 100644 --- a/livekit-android-test/src/test/java/io/livekit/android/room/track/RemoteTrackPublicationTest.kt +++ b/livekit-android-test/src/test/java/io/livekit/android/room/track/RemoteTrackPublicationTest.kt @@ -1,5 +1,5 @@ /* - * Copyright 2023-2025 LiveKit, Inc. + * Copyright 2023-2026 LiveKit, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -84,6 +84,104 @@ class RemoteTrackPublicationTest : MockE2ETest() { assertEquals(VideoQuality.LOW, lastRequest.trackSetting.quality) } + private suspend fun subscribeRemoteVideo(): RemoteTrackPublication { + connect() + + wsFactory.listener.onMessage( + wsFactory.ws, + TestData.PARTICIPANT_JOIN.toOkioByteString(), + ) + + room.onAddTrack( + MockRtpReceiver.create(), + MockVideoStreamTrack(), + arrayOf( + MockMediaStream( + id = createMediaStreamId( + TestData.REMOTE_PARTICIPANT.sid, + TestData.REMOTE_VIDEO_TRACK.sid, + ), + ), + ), + ) + + return room.remoteParticipants.values.first() + .videoTrackPublications.first() + .first as RemoteTrackPublication + } + + private fun lastTrackSetting(): LivekitRtc.UpdateTrackSettings { + val requests = wsFactory.ws.sentRequests.map { + LivekitRtc.SignalRequest.newBuilder().mergeFrom(it.toPBByteString()).build() + } + val lastTrackSetting = requests.lastOrNull { it.hasTrackSetting() } + assertTrue("no track setting sent", lastTrackSetting != null) + return lastTrackSetting!!.trackSetting + } + + @Test + fun fecProtectionOmittedUntilRequested() = runTest { + room.adaptiveStream = false + val remoteVideoPub = subscribeRemoteVideo() + advanceUntilIdle() + wsFactory.ws.clearRequests() + + remoteVideoPub.setVideoFps(30) + advanceUntilIdle() + + assertFalse(lastTrackSetting().hasFec()) + assertEquals(FecProtection.Disabled, remoteVideoPub.fecProtection) + } + + @Test + fun fecProtectionSentAndKept() = runTest { + room.adaptiveStream = false + val remoteVideoPub = subscribeRemoteVideo() + advanceUntilIdle() + wsFactory.ws.clearRequests() + + remoteVideoPub.setFecProtection(FecProtection.Fixed(FecProtectionLevel.HIGH)) + advanceUntilIdle() + + assertEquals(FecProtection.Fixed(FecProtectionLevel.HIGH), remoteVideoPub.fecProtection) + assertTrue(lastTrackSetting().hasFec()) + assertEquals(LivekitRtc.FECProtectionMode.FEC_MODE_FIXED, lastTrackSetting().fec.mode) + assertEquals(LivekitRtc.FECProtectionLevel.FEC_LEVEL_HIGH, lastTrackSetting().fec.level) + + // later settings updates keep the requested level + remoteVideoPub.setVideoFps(30) + advanceUntilIdle() + assertEquals(30, lastTrackSetting().fps) + assertEquals(LivekitRtc.FECProtectionMode.FEC_MODE_FIXED, lastTrackSetting().fec.mode) + assertEquals(LivekitRtc.FECProtectionLevel.FEC_LEVEL_HIGH, lastTrackSetting().fec.level) + + remoteVideoPub.setFecProtection(FecProtection.Adaptive) + advanceUntilIdle() + assertEquals(LivekitRtc.FECProtectionMode.FEC_MODE_ADAPTIVE, lastTrackSetting().fec.mode) + assertEquals(LivekitRtc.FECProtectionLevel.FEC_LEVEL_UNSPECIFIED, lastTrackSetting().fec.level) + + // disabling is sent explicitly rather than omitted + remoteVideoPub.setFecProtection(FecProtection.Disabled) + advanceUntilIdle() + assertTrue(lastTrackSetting().hasFec()) + assertEquals(LivekitRtc.FECProtectionMode.FEC_MODE_DISABLED, lastTrackSetting().fec.mode) + } + + @Test + fun fecProtectionWithAdaptiveStream() = runTest { + room.adaptiveStream = true + val remoteVideoPub = subscribeRemoteVideo() + advanceUntilIdle() + wsFactory.ws.clearRequests() + + // applies even when adaptive stream manages the track's quality + remoteVideoPub.setFecProtection(FecProtection.Minimum(FecProtectionLevel.MEDIUM)) + advanceUntilIdle() + + assertEquals(LivekitRtc.FECProtectionMode.FEC_MODE_MINIMUM, lastTrackSetting().fec.mode) + assertEquals(LivekitRtc.FECProtectionLevel.FEC_LEVEL_MEDIUM, lastTrackSetting().fec.level) + } + @Test fun subscriptionStatus() = runTest { connect() diff --git a/protocol b/protocol index 30d320b79..3a115c89c 160000 --- a/protocol +++ b/protocol @@ -1 +1 @@ -Subproject commit 30d320b796396930fbfa1d82932bfbfdea314353 +Subproject commit 3a115c89c9285e738e6db8e681af0acfd10570db