Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/single-pc-opus-nack.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"client-sdk-android": patch
---

Negotiate Opus NACK on every audio section in single peer connection mode, including send-only offers created before any subscriptions, enabling retransmission of lost audio in both publish and subscribe directions when RED is disabled, such as with end-to-end encryption. Accept server-offered Opus NACK in dual peer connection subscriber answers, with fallback if the modified answer is rejected.
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ package io.livekit.android.room
import io.livekit.android.room.util.MediaConstraintKeys
import io.livekit.android.room.util.PeerConnectionStateObservable
import io.livekit.android.room.util.createAnswer
import io.livekit.android.room.util.setLocalDescription
import io.livekit.android.room.util.waitUntilConnected
import io.livekit.android.util.Either
import io.livekit.android.util.LKLog
Expand Down Expand Up @@ -89,14 +88,7 @@ internal class DualPeerConnectionTransport(
val answer = subscriber.withPeerConnection { createAnswer(MediaConstraints()) }.nullSafe()
if (answer.isRight()) { return answer }

val outcome = subscriber.withPeerConnection { setLocalDescription(answer.value) }.nullSafe()
if (outcome.isRight()) {
// This is a failure
return outcome
} else {
// The caller wants the answer on success
return answer
}
return subscriber.setAnswerWithOpusNack(answer.value, offer)
}

override suspend fun applyServerAnswer(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* Copyright 2023-2026 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package io.livekit.android.room

import android.gov.nist.javax.sdp.fields.AttributeField
import android.javax.sdp.MediaDescription
import android.javax.sdp.SdpFactory
import io.livekit.android.webrtc.getRtps

/**
* Adds plain NACK feedback to every Opus payload in an audio section.
*
* The SFU negotiates audio feedback from the first audio section, so single-PC send-only
* sections must advertise it even before any subscriptions exist. Wildcard NACK already
* covers every payload; `nack pli` is separate feedback and does not imply plain NACK.
*/
internal fun ensureOpusNack(mediaDesc: MediaDescription) {
if (mediaDesc.media.mediaType != "audio") return
val opusPayloads = mediaDesc.opusPayloads()
if (opusPayloads.isEmpty()) return
val nackPayloads = mediaDesc.nackPayloads()
if ("*" in nackPayloads) return

for (payload in opusPayloads - nackPayloads) {
mediaDesc.addAttribute(
AttributeField().apply {
name = "rtcp-fb"
value = "$payload nack"
},
)
}
}

/** Restores offered Opus NACK in the corresponding answer sections, matched by MID. */
internal fun mungeOpusNackForAnswer(answer: String, offer: String, factory: SdpFactory): String {
if (!offer.contains("a=rtcp-fb:")) return answer
val nackMids = factory.createSessionDescription(offer).getMediaDescriptions(true)
.filterIsInstance<MediaDescription>()
.filter { section ->
if (section.media.mediaType != "audio") return@filter false
val opus = section.opusPayloads()
val feedback = section.nackPayloads()
opus.isNotEmpty() && ("*" in feedback || opus.any { it in feedback })
}
.mapNotNull { it.getAttribute("mid") }
.toSet()
if (nackMids.isEmpty()) return answer
val document = factory.createSessionDescription(answer)
var changed = false
for (section in document.getMediaDescriptions(true).filterIsInstance<MediaDescription>()) {
if (section.getAttribute("mid") !in nackMids) continue
val before = section.toString()
ensureOpusNack(section)
changed = changed || section.toString() != before
}
return if (changed) document.toString() else answer
}

/** Returns all Opus payload types in this media section. */
private fun MediaDescription.opusPayloads(): Set<String> = getRtps()
.map { it.second }
.filter { it.codec.equals("opus", ignoreCase = true) }
.map { it.payload.toString() }
.toSet()

/** Returns payload types with plain NACK feedback, including wildcard feedback. */
private fun MediaDescription.nackPayloads(): Set<String> = getAttributes(true)
.filterIsInstance<AttributeField>()
.filter { it.attribute.name == "rtcp-fb" }
.mapNotNull { attribute ->
val fields = attribute.value?.trim()?.split(rtcpFeedbackWhitespace) ?: return@mapNotNull null
if (fields.size == 2 && fields[1] == "nack") fields[0] else null
}
.toSet()

private val rtcpFeedbackWhitespace = "\\s+".toRegex()
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ import kotlin.time.Duration.Companion.milliseconds
*/
internal class PeerConnectionTransport
@AssistedInject
@Suppress("LongParameterList") // Dagger combines assisted settings with injected dependencies.
constructor(
@Assisted config: RTCConfiguration,
@Assisted pcObserver: PeerConnection.Observer,
Expand All @@ -81,6 +82,7 @@ constructor(
connectionFactory: PeerConnectionFactory,
private val sdpFactory: SdpFactory,
private val rtcThreadToken: RTCThreadToken,
@Assisted private val singlePeerConnection: Boolean = false,
) {
private val coroutineScope = CoroutineScope(ioDispatcher + SupervisorJob())

Expand Down Expand Up @@ -262,7 +264,9 @@ constructor(
}
for (mediaDesc in mediaDescs) {
if (mediaDesc.media.mediaType == "audio") {
// TODO
if (singlePeerConnection) {
ensureOpusNack(mediaDesc)
}
} else if (mediaDesc.media.mediaType == "video") {
ensureVideoDDExtensionForSVC(mediaDesc)
ensureCodecBitrates(
Expand Down Expand Up @@ -296,6 +300,22 @@ constructor(
}
}

/** Applies an answer accepting Opus NACK only on the MIDs that offered it. */
suspend fun setAnswerWithOpusNack(answer: SessionDescription, offer: SessionDescription): Either<SessionDescription, String?> {
val munged = mungeOpusNackForAnswer(answer.description, offer.description, sdpFactory)
if (munged != answer.description) {
val description = SessionDescription(answer.type, munged)
val result = withPeerConnection { setLocalDescription(description) }
if (result is Either.Left) return Either.Left(description)
LKLog.w { "Opus NACK answer rejected, falling back to original SDP" }
}
return when (val result = withPeerConnection { setLocalDescription(answer) }) {
is Either.Left -> Either.Left(answer)
is Either.Right -> result
null -> Either.Right("PCT closed")
}
}

private suspend fun setMungedSdp(sdp: SessionDescription, mungedDescription: String, remote: Boolean = false): SessionDescription {
val mungedSdp = SessionDescription(sdp.type, mungedDescription)

Expand Down Expand Up @@ -482,6 +502,7 @@ constructor(
config: RTCConfiguration,
pcObserver: PeerConnection.Observer,
listener: Listener?,
singlePeerConnection: Boolean = false,
): PeerConnectionTransport
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,7 @@ internal constructor(
rtcConfig,
publisherObserver,
publisherObserver,
singlePeerConnection = connectOptions.singlePeerConnection,
)

val connectionStateListener: PeerConnectionStateListener = { newState ->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,13 @@ import livekit.LivekitRtc.MediaSectionsRequirement
import livekit.LivekitRtc.SignalTarget
import livekit.org.webrtc.IceCandidate
import livekit.org.webrtc.MediaStreamTrack
import livekit.org.webrtc.MockPeerConnectionFactory
import livekit.org.webrtc.PeerConnection
import livekit.org.webrtc.PeerConnection.IceConnectionState
import livekit.org.webrtc.PeerConnection.PeerConnectionState
import livekit.org.webrtc.PeerConnection.RTCConfiguration
import livekit.org.webrtc.PeerConnectionFactory
import livekit.org.webrtc.RtpTransceiver
import livekit.org.webrtc.SdpObserver
import livekit.org.webrtc.SessionDescription
import livekit.org.webrtc.WebRTCInitializer
import org.junit.Assert.assertEquals
Expand All @@ -50,6 +51,7 @@ import org.junit.Assert.assertTrue
import org.junit.Before
import org.junit.Test
import org.mockito.kotlin.any
import org.mockito.kotlin.doAnswer
import org.mockito.kotlin.mock
import org.mockito.kotlin.never
import org.mockito.kotlin.spy
Expand Down Expand Up @@ -85,17 +87,23 @@ class DualPeerConnectionTransportTest {
subscriber = createPeerConnectionTransport()
}

private fun createPeerConnectionTransport(): PeerConnectionTransport = spy(
PeerConnectionTransport(
config = RTCConfiguration(emptyList()),
pcObserver = mock<PeerConnection.Observer>(),
listener = null,
ioDispatcher = Dispatchers.Unconfined,
connectionFactory = MockPeerConnectionFactory(),
sdpFactory = SdpFactory.getInstance(),
rtcThreadToken = MockRTCThreadToken(),
),
)
private fun createPeerConnectionTransport(): PeerConnectionTransport {
val observer = mock<PeerConnection.Observer>()
val pc = spy(MockPeerConnection(RTCConfiguration(emptyList()), observer))
val factory = mock<PeerConnectionFactory>()
whenever(factory.createPeerConnection(any<RTCConfiguration>(), any<PeerConnection.Observer>())).thenReturn(pc)
return spy(
PeerConnectionTransport(
config = RTCConfiguration(emptyList()),
pcObserver = observer,
listener = null,
ioDispatcher = Dispatchers.Unconfined,
connectionFactory = factory,
sdpFactory = SdpFactory.getInstance(),
rtcThreadToken = MockRTCThreadToken(),
),
)
}

private fun transport(
subscriberPrimary: Boolean = true,
Expand Down Expand Up @@ -226,6 +234,91 @@ class DualPeerConnectionTransportTest {
assertSame(answer, subscriberPc.localDesc)
}

@Test
fun answerAcceptsOnlyOfferedNackByMidWithReorderedSectionsAndDifferentPayloads() = runTest {
val offer = audioSdp("0", 111, "a=rtcp-fb:111 nack\r\n") + audioSection("1", 112)
val original = stubAudioAnswer(audioSdp("1", 109) + audioSection("0", 110))

val result = transport().answerServerOffer(SessionDescription(SessionDescription.Type.OFFER, offer), 7)

assertTrue(result is Either.Left)
val answer = (result as Either.Left).value
assertSame(answer, subscriberPc.localDesc)
val sections = SdpFactory.getInstance().createSessionDescription(answer.description).getMediaDescriptions(true)
assertFalse(sections[0].toString().contains(" nack"))
assertTrue(sections[1].toString().contains("a=rtcp-fb:110 nack"))
assertFalse(original.description.contains(" nack"))
}

@Test
fun answerWithoutOfferedNackIsAppliedUnchanged() = runTest {
val original = stubAudioAnswer(audioSdp("0", 109))
val offer = SessionDescription(SessionDescription.Type.OFFER, audioSdp("0", 111, "a=rtcp-fb:111 nack pli\r\n"))

val result = transport().answerServerOffer(offer, 7)

assertTrue(result is Either.Left)
assertSame(original, (result as Either.Left).value)
assertSame(original, subscriberPc.localDesc)
}

@Test
fun rejectedNackAnswerFallsBackAndReturnsTheAppliedOriginal() = runTest {
val original = stubAudioAnswer(audioSdp("0", 109))
var rejected = false
val pc = subscriberPc
doAnswer { call ->
val description = call.getArgument<SessionDescription>(1)
if (description.description.contains("a=rtcp-fb:109 nack")) {
rejected = true
call.getArgument<SdpObserver>(0).onSetFailure("NACK rejected")
} else {
call.callRealMethod()
}
null
}.whenever(pc).setLocalDescription(any(), any())
val offer = SessionDescription(SessionDescription.Type.OFFER, audioSdp("0", 111, "a=rtcp-fb:* nack\r\n"))

val result = transport().answerServerOffer(offer, 7)

assertTrue(rejected)
assertTrue(result is Either.Left)
assertSame(original, (result as Either.Left).value)
assertSame(original, subscriberPc.localDesc)
}

@Test
fun failureOfBothAnswersIsReturnedInsteadOfSignalingAnUnappliedAnswer() = runTest {
stubAudioAnswer(audioSdp("0", 109))
val pc = subscriberPc
doAnswer { call ->
call.getArgument<SdpObserver>(0).onSetFailure("All answers rejected")
null
}.whenever(pc).setLocalDescription(any(), any())
val offer = SessionDescription(SessionDescription.Type.OFFER, audioSdp("0", 111, "a=rtcp-fb:111 nack\r\n"))

val result = transport().answerServerOffer(offer, 7)

assertTrue(result is Either.Right)
assertNull(subscriberPc.localDesc)
}

private fun stubAudioAnswer(sdp: String): SessionDescription {
val answer = SessionDescription(SessionDescription.Type.ANSWER, sdp)
val pc = subscriberPc
doAnswer { call ->
call.getArgument<SdpObserver>(0).onCreateSuccess(answer)
null
}.whenever(pc).createAnswer(any(), any())
return answer
}

private fun audioSdp(mid: String, payload: Int, feedback: String = ""): String =
"v=0\r\no=- 1 1 IN IP4 127.0.0.1\r\ns=-\r\nt=0 0\r\n" + audioSection(mid, payload, feedback)

private fun audioSection(mid: String, payload: Int, feedback: String = ""): String =
"m=audio 9 UDP/TLS/RTP/SAVPF $payload\r\na=mid:$mid\r\na=rtpmap:$payload opus/48000/2\r\n$feedback"

@Test
fun answerServerOfferPassesOfferId() = runTest {
val offer = SessionDescription(SessionDescription.Type.OFFER, "remote_offer")
Expand Down
Loading
Loading