diff --git a/demo-app/src/main/kotlin/io/getstream/video/android/ui/call/CallStats.kt b/demo-app/src/main/kotlin/io/getstream/video/android/ui/call/CallStats.kt index 0522464e002..a6cdfe9eba3 100644 --- a/demo-app/src/main/kotlin/io/getstream/video/android/ui/call/CallStats.kt +++ b/demo-app/src/main/kotlin/io/getstream/video/android/ui/call/CallStats.kt @@ -181,6 +181,13 @@ fun CallStats(call: Call) { val publisherCodecLabel = if (publisherVideoCodec.isNotEmpty()) "($publisherVideoCodec)" else "" val subscriberVideoCodec by call.state.stats.subscriber.videoCodec.collectAsStateWithLifecycle() val subscriberCodecLabel = if (subscriberVideoCodec.isNotEmpty()) "($subscriberVideoCodec)" else "" + val publisherAudioBitrate by call.state.stats.publisher.audioBitrateKbps + .collectAsStateWithLifecycle() + val subscriberAudioBitrate by call.state.stats.subscriber.audioBitrateKbps + .collectAsStateWithLifecycle() + val audioCodec by call.state.stats.publisher.audioCodec.collectAsStateWithLifecycle() + val audioTargetBitrate by call.state.stats.publisher.audioTargetBitrateKbps + .collectAsStateWithLifecycle() LatencyOrJitter(title = "Latency", value = latency) Spacer(modifier = Modifier.size(16.dp)) @@ -199,9 +206,19 @@ fun CallStats(call: Call) { value = subscriberResolution, ) Spacer(modifier = Modifier.size(16.dp)) - StatItem(title = "Publish bitrate", value = "$publisherBitrate Kbps") + // These two are the connection's bandwidth estimate, not a transmitted rate — named + // accordingly so they stop being read as "what we are sending". + StatItem(title = "Available outgoing bitrate", value = "$publisherBitrate Kbps") Spacer(modifier = Modifier.size(16.dp)) - StatItem(title = "Receiving bitrate", value = "$subscriberBitrate Kbps") + StatItem(title = "Available incoming bitrate", value = "$subscriberBitrate Kbps") + Spacer(modifier = Modifier.size(16.dp)) + StatItem(title = "Audio target", value = "%.0f Kbps".format(audioTargetBitrate)) + Spacer(modifier = Modifier.size(16.dp)) + StatItem(title = "Audio sent", value = "%.1f Kbps".format(publisherAudioBitrate)) + Spacer(modifier = Modifier.size(16.dp)) + StatItem(title = "Audio received", value = "%.1f Kbps".format(subscriberAudioBitrate)) + Spacer(modifier = Modifier.size(16.dp)) + StatItem(title = "Audio codec", value = audioCodec) } } } diff --git a/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/MenuDefinitions.kt b/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/MenuDefinitions.kt index e3ad36198f1..a692ce4866d 100644 --- a/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/MenuDefinitions.kt +++ b/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/MenuDefinitions.kt @@ -31,6 +31,7 @@ import androidx.compose.material.icons.filled.ClosedCaptionOff import androidx.compose.material.icons.filled.Crop import androidx.compose.material.icons.filled.CropFree import androidx.compose.material.icons.filled.Feedback +import androidx.compose.material.icons.filled.MusicNote import androidx.compose.material.icons.filled.Person import androidx.compose.material.icons.filled.RadioButtonChecked import androidx.compose.material.icons.filled.RawOff @@ -96,6 +97,8 @@ fun defaultStreamMenu( onToggleAudioUsage: () -> Unit = {}, selectedRecordingTypes: Set = emptySet(), onSelectRecordingType: (RecordingType) -> Unit = {}, + isMusicAudioProfile: Boolean = false, + onToggleAudioProfile: () -> Unit = {}, ) = buildList { if (noiseCancellationFeatureEnabled) { add( @@ -175,6 +178,8 @@ fun defaultStreamMenu( onToggleAudioUsage, selectedRecordingTypes, onSelectRecordingType, + isMusicAudioProfile, + onToggleAudioProfile, ), ), ) @@ -358,7 +363,15 @@ fun debugSubmenu( onToggleAudioUsage: () -> Unit, selectedRecordingTypes: Set, onSelectRecordingType: (RecordingType) -> Unit, + isMusicAudioProfile: Boolean = false, + onToggleAudioProfile: () -> Unit = {}, ) = listOf( + ActionMenuItem( + title = if (isMusicAudioProfile) "Music mode: On" else "Music mode: Off", + icon = Icons.Default.MusicNote, + highlight = isMusicAudioProfile, + action = onToggleAudioProfile, + ), DynamicSubMenuItem( title = "List Transcriptions", icon = Icons.AutoMirrored.Filled.ReceiptLong, diff --git a/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/SettingsMenu.kt b/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/SettingsMenu.kt index f385cfd5636..2e2a9f24327 100644 --- a/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/SettingsMenu.kt +++ b/demo-app/src/main/kotlin/io/getstream/video/android/ui/menu/SettingsMenu.kt @@ -74,6 +74,7 @@ import io.getstream.video.android.ui.menu.base.MenuItem import io.getstream.video.android.ui.menu.transcriptions.TranscriptionUiStateManager import io.getstream.video.android.util.filters.SampleAudioFilter import kotlinx.coroutines.launch +import stream.video.sfu.models.AudioBitrateProfile import java.nio.ByteBuffer @OptIn(ExperimentalPermissionsApi::class) @@ -114,6 +115,36 @@ internal fun SettingsMenu( call.speaker.setAudioUsage(newAudioUsage) } + val audioBitrateProfile by call.microphone.audioBitrateProfile.collectAsStateWithLifecycle() + val isMusicAudioProfile = + audioBitrateProfile == AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY + + val onToggleAudioProfile: () -> Unit = { + val next = if (isMusicAudioProfile) { + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_STANDARD_UNSPECIFIED + } else { + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY + } + val turningOn = !isMusicAudioProfile + scope.launch { + call.microphone.setAudioBitrateProfile(next) + .onSuccess { + Toast.makeText( + context, + if (turningOn) "Music mode on" else "Music mode off", + Toast.LENGTH_LONG, + ).show() + } + .onFailure { + Toast.makeText( + context, + "Music mode not changed: ${it.message}", + Toast.LENGTH_LONG, + ).show() + } + } + } + val onToggleAudioFilterClick: () -> Unit = { if (call.audioFilter == null) { call.audioFilter = object : InputAudioFilter { @@ -360,6 +391,8 @@ internal fun SettingsMenu( onToggleAudioUsage = onToggleAudioUsage, selectedRecordingTypes = enabledRecordingTypes, onSelectRecordingType = onSelectRecordingType, + isMusicAudioProfile = isMusicAudioProfile, + onToggleAudioProfile = onToggleAudioProfile, ), ) } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1f5b00ebdbb..f96dd0c7c00 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -49,7 +49,7 @@ tink = "1.9.0" turbine = "0.13.0" itu = "1.7.3" -streamWebRTC = "145.17.0" +streamWebRTC = "145.19.0" streamNoiseCancellation = "3.0.0" streamResult = "1.3.0" streamChat = "6.10.0" diff --git a/stream-video-android-core/api/stream-video-android-core.api b/stream-video-android-core/api/stream-video-android-core.api index a5f5fbee25c..3f2e807f280 100644 --- a/stream-video-android-core/api/stream-video-android-core.api +++ b/stream-video-android-core/api/stream-video-android-core.api @@ -9657,6 +9657,9 @@ public final class io/getstream/video/android/core/ParticipantState$Video : io/g public final class io/getstream/video/android/core/PeerConnectionStats { public fun (Lkotlinx/coroutines/CoroutineScope;)V + public final fun getAudioBitrateKbps ()Lkotlinx/coroutines/flow/StateFlow; + public final fun getAudioCodec ()Lkotlinx/coroutines/flow/StateFlow; + public final fun getAudioTargetBitrateKbps ()Lkotlinx/coroutines/flow/StateFlow; public final fun getBitrateKbps ()Lkotlinx/coroutines/flow/StateFlow; public final fun getJitterInMs ()Lkotlinx/coroutines/flow/StateFlow; public final fun getLatency ()Lkotlinx/coroutines/flow/StateFlow; diff --git a/stream-video-android-core/build.gradle.kts b/stream-video-android-core/build.gradle.kts index a0e7b281dcb..824346223c0 100644 --- a/stream-video-android-core/build.gradle.kts +++ b/stream-video-android-core/build.gradle.kts @@ -93,6 +93,16 @@ android { unitTests { isIncludeAndroidResources = true isReturnDefaultValues = true + + all { + // Robolectric builds a full Android sandbox per SDK level named in @Config, and + // this module's suite spans six of them in one JVM — nothing sets forkEvery here, + // so they accumulate. On CI that runs out of heap while loading an android-all + // jar, and which class reports it depends on execution order, which is why the + // telecom and notification tests kept getting blamed. The Xmx in + // gradle.properties is the daemon's, not the test JVM's, so it never applied. + it.maxHeapSize = "2g" + } } managedDevices { diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/AudioProfileResult.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/AudioProfileResult.kt new file mode 100644 index 00000000000..88e7147e568 --- /dev/null +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/AudioProfileResult.kt @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2014-2026 Stream.io Inc. All rights reserved. + * + * Licensed under the Stream License; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://github.com/GetStream/stream-video-android/blob/main/LICENSE + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.getstream.video.android.core + +import stream.video.sfu.models.AudioBitrateProfile + +/** + * What [MicrophoneManager.setAudioBitrateProfile] managed to change, for logging and for naming + * the stages in a failure. Internal: callers get the profile back on + * [MicrophoneManager.audioBitrateProfile] and a failure when it did not take. + * + * A stage reports true when it matches [profile] *or* when there is nothing for it to move — + * no live capture, no sender, no such hardware. False means a live stage refused, and the reason + * is logged. + * + * @property audioMaxBitrateBps The bitrate now on the live audio sender, or null when the + * SFU-negotiated one stands. + */ +internal data class AudioProfileResult( + val profile: AudioBitrateProfile, + val audioMaxBitrateBps: Int?, + val noiseCancellationApplied: Boolean, + val platformNoiseSuppressorApplied: Boolean, + val platformAcousticEchoCancelerApplied: Boolean, + val softwareAudioProcessingApplied: Boolean, + val audioMaxBitrateApplied: Boolean, + val captureAudioSourceApplied: Boolean, +) { + /** Stages that are still processing audio the previous profile's way. */ + val missedStages: List + get() = buildList { + if (!noiseCancellationApplied) add("noise cancellation") + if (!platformNoiseSuppressorApplied) add("hardware noise suppressor") + if (!platformAcousticEchoCancelerApplied) add("hardware echo canceller") + if (!softwareAudioProcessingApplied) add("software audio processing") + if (!audioMaxBitrateApplied) add("max bitrate") + if (!captureAudioSourceApplied) add("capture audio source") + } + + /** Every stage reached. */ + val complete: Boolean get() = missedStages.isEmpty() +} diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt index b1af162a11f..f8ed71e83df 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt @@ -1040,6 +1040,53 @@ public class Call( notifyNoiseCancellationState(media.isAudioProcessingEnabledIfCreated()) } + // Audio bitrate profile bridges. [MicrophoneManager.setAudioBitrateProfile] is the public + // entry point and can reach neither the media component nor the session from there. + + internal fun setHardwareNoiseSuppressorEnabled(enabled: Boolean): Boolean = + media.setHardwareNoiseSuppressorEnabled(enabled) + + internal fun setHardwareAcousticEchoCancelerEnabled(enabled: Boolean): Boolean = + media.setHardwareAcousticEchoCancelerEnabled(enabled) + + /** Must not be called from the main thread: the module rebuilds AudioRecord. */ + internal fun setCaptureAudioSource(audioSource: Int): Boolean = + media.setCaptureAudioSource(audioSource) + + /** + * Rebuilds the audio source and track so audio-source constraints take effect mid-call. With + * no session the source is built lazily from current constraints, so the change already holds. + */ + internal fun rebuildAudioCapturePipeline(): Boolean = + session.value?.rebuildAudioCapturePipeline() ?: true + + internal fun setAudioMaxBitrate(maxBitrateBps: Int): Boolean = + session.value?.setAudioMaxBitrate(maxBitrateBps) ?: false + + /** Whether an audio sender exists to take a live profile change. */ + internal fun hasLiveAudioSender(): Boolean = + session.value?.hasLiveAudioSender() ?: false + + internal fun audioMaxBitrate(): Int? = session.value?.audioMaxBitrate() + + /** The audio bitrate the SFU negotiated at join, or null when nothing publishes audio. */ + internal fun negotiatedAudioBitrate(): Int? = session.value?.negotiatedAudioBitrate() + + /** The bitrate the SFU offers for [profile], or null when it named none. */ + internal fun audioBitrateFor(profile: stream.video.sfu.models.AudioBitrateProfile): Int? = + session.value?.audioBitrateFor(profile) + + /** Whether a noise-cancellation processor is wired in and can be turned on or off. */ + internal fun isAudioProcessingReachable(): Boolean = media.isAudioProcessingReachable() + + // Absent hardware is not the same as hardware that refused, so these are asked separately. + + internal fun isHardwareNoiseSuppressorSupported(): Boolean = + media.isHardwareNoiseSuppressorSupported() + + internal fun isHardwareAcousticEchoCancelerSupported(): Boolean = + media.isHardwareAcousticEchoCancelerSupported() + fun toggleAudioProcessing(): Boolean { // Reads without building a factory: the gate runs before join, and a factory created // there would capture the pre-join audio bitrate profile. diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallStats.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallStats.kt index a0d7cc8ab6e..c80c172bcbf 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallStats.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallStats.kt @@ -69,6 +69,57 @@ public class PeerConnectionStats(scope: CoroutineScope) { internal val _videoCodec: MutableStateFlow = MutableStateFlow("") val videoCodec: StateFlow = _videoCodec + + internal val _audioBitrateKbps: MutableStateFlow = MutableStateFlow(0F) + + /** + * Audio bitrate actually sent or received, measured from the RTP byte counters between two + * stats polls. + * + * Distinct from [bitrateKbps], which reports the connection's *available* bandwidth estimate + * rather than anything that was transmitted. This one moves when the encoder does. + */ + val audioBitrateKbps: StateFlow = _audioBitrateKbps + + internal val _audioCodec: MutableStateFlow = MutableStateFlow("") + val audioCodec: StateFlow = _audioCodec + + internal val _audioTargetBitrateKbps: MutableStateFlow = MutableStateFlow(0F) + + /** + * The bitrate the audio encoder is aiming for, as reported by `outbound-rtp`. + * + * The requested value, where [audioBitrateKbps] is what actually went out. Comparing the two + * is how you tell "we asked for more" from "we are sending more". + */ + val audioTargetBitrateKbps: StateFlow = _audioTargetBitrateKbps + + /** Byte counter and its timestamp from the previous poll, to derive a rate. */ + internal var lastAudioBytes: Long? = null + internal var lastAudioTimestampUs: Double? = null + + /** + * Derives the audio bitrate from the change in [bytes] since the previous poll. + * + * The first sample only seeds the baseline — a rate needs two points. A counter that went + * backwards means the stream was replaced, so the baseline is reset rather than reported as a + * negative rate. + */ + internal fun updateAudioBitrate(bytes: Long?, timestampUs: Double) { + if (bytes == null) return + val previousBytes = lastAudioBytes + val previousTimestampUs = lastAudioTimestampUs + lastAudioBytes = bytes + lastAudioTimestampUs = timestampUs + + if (previousBytes == null || previousTimestampUs == null) return + val elapsedUs = timestampUs - previousTimestampUs + val deltaBytes = bytes - previousBytes + if (elapsedUs <= 0 || deltaBytes < 0) return + + val bitsPerSecond = deltaBytes * 8.0 * 1_000_000.0 / elapsedUs + _audioBitrateKbps.value = (bitsPerSecond / 1000).toFloat() + } } public data class LocalStats( @@ -193,6 +244,55 @@ public class CallStats(val call: Call, val callScope: CoroutineScope) { subscriber._resolution.value = "$width x $height @ $fps fps" } } + // Audio send/receive rate, derived from the RTP byte counters. Nothing else reports + // what the audio encoder is actually doing — the candidate-pair numbers below are a + // bandwidth estimate, not a transmitted rate. + statGroups["outbound-rtp:audio"]?.firstOrNull()?.let { + publisher.updateAudioBitrate( + bytes = it.members["bytesSent"] as? Long, + timestampUs = it.timestampUs, + ) + (it.members["targetBitrate"] as? Double)?.let { target -> + publisher._audioTargetBitrateKbps.value = (target / 1000).toFloat() + } + } + statGroups["inbound-rtp:audio"]?.firstOrNull()?.let { + subscriber.updateAudioBitrate( + bytes = it.members["bytesReceived"] as? Long, + timestampUs = it.timestampUs, + ) + } + // The publisher and the subscriber report are both fed through here, and each one + // describes only its own direction. Writing both sides from either report leaves + // whichever ran last showing on both, so the codec is resolved through the matching + // RTP statistic's own codecId — which also picks the right entry when a report + // carries more than one audio codec — and only that direction is updated. + val audioRtp = if (isPublisher) { + statGroups["outbound-rtp:audio"] + } else { + statGroups["inbound-rtp:audio"] + }?.firstOrNull() + val audioCodecStat = (audioRtp?.members?.get("codecId") as? String) + ?.let { codecId -> stats.origin.statsMap[codecId] } + ?: statGroups["codec:audio"]?.firstOrNull() + audioCodecStat?.let { + val mimeType = it.members["mimeType"] as? String + val clockRate = it.members["clockRate"] as? Long + val channels = it.members["channels"] as? Long + val fmtp = it.members["sdpFmtpLine"] as? String + val codec = listOfNotNull( + mimeType, + clockRate?.let { rate -> "$rate Hz" }, + channels?.let { count -> if (count > 1) "stereo" else "mono" }, + fmtp, + ).joinToString(" ") + if (isPublisher) { + publisher._audioCodec.value = codec + } else { + subscriber._audioCodec.value = codec + } + } + statGroups["candidate-pair"]?.firstOrNull()?.let { val latency = it.members["currentRoundTripTime"] as? Double val outgoingBitrate = it.members["availableOutgoingBitrate"] as? Double diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/MediaManager.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/MediaManager.kt index 0e154a90299..a48e363502e 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/MediaManager.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/MediaManager.kt @@ -54,12 +54,18 @@ import io.getstream.video.android.core.audio.UsbAudioInputDevice.Companion.isUsb import io.getstream.video.android.core.call.video.FilterVideoProcessor import io.getstream.video.android.core.camera.CameraCharacteristicsValidator import io.getstream.video.android.core.camera.DefaultCameraCharacteristicsValidator +import io.getstream.video.android.core.dispatchers.DispatcherProvider import io.getstream.video.android.core.notifications.internal.telecom.jetpack.TelecomCall import io.getstream.video.android.core.notifications.internal.telecom.jetpack.TelecomCallAction import io.getstream.video.android.core.screenshare.StreamScreenShareService import io.getstream.video.android.core.utils.buildAudioConstraints +import io.getstream.video.android.core.utils.captureAudioSourceFor +import io.getstream.video.android.core.utils.defaultHardwareAudioEffectsEnabled +import io.getstream.video.android.core.utils.defaultSoftwareAudioProcessingEnabled import io.getstream.video.android.core.utils.mapState import io.getstream.video.android.core.utils.safeCall +import io.getstream.video.android.core.utils.safeCallWithDefault +import io.getstream.video.android.core.utils.targetAudioMaxBitrateBps import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.delay @@ -67,6 +73,7 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.launch import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withContext import org.webrtc.AudioSource import org.webrtc.AudioTrack import org.webrtc.Camera2Capturer @@ -605,9 +612,49 @@ class MicrophoneManager( AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_STANDARD_UNSPECIFIED, ) - /** The current audio bitrate profile */ + /** + * The audio bitrate profile currently in force — observe this to drive a music/voice toggle. + * + * Only moves when the profile actually took. Set before joining it always does, since the + * pipeline is then built from it. On a running call it moves only if every reachable stage + * accepted the change; otherwise it keeps reporting the previous profile, so a toggle bound to + * it snaps back rather than claiming a switch the audio did not make, and + * [setAudioBitrateProfile] fails naming the stages that refused. + */ val audioBitrateProfile: StateFlow = _audioBitrateProfile + /** + * The platform (hardware) noise suppressor and echo canceller state this call has asked the + * device for. Derived from [audioBitrateProfile] alone — on, except under MUSIC_HIGH_QUALITY + * or below Android Q. There is no per-stage setter, so the two cannot disagree. + */ + private var hardwareNoiseSuppressorEnabled = + defaultHardwareAudioEffectsEnabled(_audioBitrateProfile.value) + + /** + * Whether WebRTC's own software audio processing — echo cancellation, noise suppression, + * automatic gain control, high-pass filter — is applied to captured audio. A separate stage + * from [hardwareNoiseSuppressorEnabled], which runs in the audio device module. Also derived + * from [audioBitrateProfile] alone. + */ + private var softwareAudioProcessingEnabled = + defaultSoftwareAudioProcessingEnabled(_audioBitrateProfile.value) + + /** The value audio sources are built with; see [softwareAudioProcessingEnabled]. */ + internal val effectiveSoftwareAudioProcessingEnabled: Boolean + get() = softwareAudioProcessingEnabled + + private val _communicationAudioModeEnabled = MutableStateFlow(true) + + /** + * Whether the device is being asked to stay in `AudioManager.MODE_IN_COMMUNICATION`, which is + * where a call puts it by default, rather than `AudioManager.MODE_NORMAL`. + * + * Follows [audioBitrateProfile] — off under MUSIC_HIGH_QUALITY. Not a public knob: + * the profile decides the mode so this state and [audioBitrateProfile] cannot disagree. + */ + internal val communicationAudioModeEnabled: StateFlow = _communicationAudioModeEnabled + // API /** Enable the audio, the rtc engine will automatically inform the SFU */ internal fun enable(fromUser: Boolean = true) { @@ -873,18 +920,31 @@ class MicrophoneManager( // ==================== End USB Audio Input Device Support ==================== /** - * Set the audio bitrate profile. - * This can only be set before joining the call. Once the call is joined, - * changes to the audio bitrate profile will be ignored. + * Sets the audio bitrate profile, before joining or on a running call, so a broadcaster who + * starts playing music can switch to MUSIC_HIGH_QUALITY without rejoining. + * + * Requires HiFi audio enabled in the dashboard settings. The Opus bitrate and channel count + * come from the server at join, so a mid-call switch does not renegotiate them. + * + * **Expect a brief gap in captured audio** on a mid-call switch: the platform rebuilds its + * capture path. The call is not interrupted otherwise. + * + * **Echo cancellation goes off under MUSIC_HIGH_QUALITY**, so treat music as a headphones + * setting. It also costs communication routing and Bluetooth capture — SCO only runs in + * communication mode. * - * @param profile The audio bitrate profile to use - * @return true if the profile was successfully set, false if: - * - The call is already joined and the change was ignored - * - HiFi audio is not enabled in dashboard settings when trying to use MUSIC_HIGH_QUALITY + * Read the profile in force back from [audioBitrateProfile]. + * + * @return success once the profile is in force; failure when HiFi audio is off, the call + * settings could not be fetched, or a running stage refused — the profile is then left where + * it was so the caller can retry. */ - suspend fun setAudioBitrateProfile(profile: AudioBitrateProfile): Result { + suspend fun setAudioBitrateProfile( + profile: AudioBitrateProfile, + ): Result { val connectionState = mediaManager.call.state.connection.value - val isJoined = connectionState is RealtimeConnection.Joined || connectionState is RealtimeConnection.Connected + val isJoined = connectionState is RealtimeConnection.Joined || + connectionState is RealtimeConnection.Connected // get the call settings if they are not present if (mediaManager.call.state.settings.value == null) { @@ -897,7 +957,8 @@ class MicrophoneManager( } } - // Check if HiFi audio is enabled in dashboard + // Check if HiFi audio is enabled in dashboard. This gates a mid-call switch too: the + // profiles mean the same thing whenever they are set, so they answer to the same setting. val hifiAudioEnabled = mediaManager.call.state.settings.value?.audio?.hifiAudioEnabled ?: false if (!hifiAudioEnabled) { logger.w { @@ -909,27 +970,279 @@ class MicrophoneManager( ) } - if (isJoined) { - logger.w { - "[setAudioBitrateProfile] called after call is joined. " + - "Audio bitrate profile can only be set before joining the call. " + - "Ignoring the change." - } + logger.i { "[setAudioBitrateProfile] Setting audio bitrate profile to: $profile" } + val previousProfile = _audioBitrateProfile.value + // The profile decides how the audio device module and the audio sources are built, so + // these have to follow it — the pipeline rebuild below reads them. A switch that does not + // complete puts them back; see [revertProfileDerivedState]. + hardwareNoiseSuppressorEnabled = defaultHardwareAudioEffectsEnabled(profile) + // Whether the source constraints actually change decides whether the pipeline has to be + // rebuilt below, so it has to be read before the new value lands. + val softwareAudioProcessingChanged = + softwareAudioProcessingEnabled != defaultSoftwareAudioProcessingEnabled(profile) + softwareAudioProcessingEnabled = defaultSoftwareAudioProcessingEnabled(profile) + + if (!isJoined) { + // Nothing is capturing or publishing yet, so there is no stage to move and the profile + // is in force by construction. The audio mode has to be requested now: routing enters + // communication mode when it activates, after this returns, and some vendors bind the + // VoIP capture chain to the mode at AudioRecord open. + applyCommunicationAudioModeForProfile(profile) + _audioBitrateProfile.value = profile + return Result.success(Unit) + } + + val result = applyProfileToRunningCall(profile, softwareAudioProcessingChanged) + if (!result.complete) { + // A stage still processing the old way means the audio is not what this profile + // means, so the profile is not published: a toggle stuck on MUSIC while a suppressor + // still eats the music looks exactly like success. Leaving the flow where it was is + // the truth and lets the caller retry. + logger.w { "[setAudioBitrateProfile] $profile only partly applied: $result" } + revertProfileDerivedState(previousProfile) return Result.failure( IllegalStateException( - "Cannot call setAudioBitrateProfile after call has been joined", + "$profile not applied; still on the previous profile: " + + result.missedStages.joinToString(), ), ) } - - logger.i { "[setAudioBitrateProfile] Setting audio bitrate profile to: $profile" } _audioBitrateProfile.value = profile return Result.success(Unit) } + /** + * Puts the pipeline and the state derived from it back to [previousProfile] after a switch that + * did not complete, so nothing disagrees with [audioBitrateProfile] about which profile this + * call is on. + * + * Every stage with a live setter is put back, the bitrate included — it is applied before the + * source rebuild, so it can be the stage that already moved when a later one refused. The + * software audio processing stage is not here: it has no setter, and it runs last in + * [applyProfileToRunningCall] precisely so a refusal cannot leave it moved. + * + * Mode and capture source revert together; `MODE_NORMAL` against `VOICE_COMMUNICATION` (or the + * reverse) is the pair that parks capture on a near-silent path on some vendors. + */ + private suspend fun revertProfileDerivedState(previousProfile: AudioBitrateProfile) { + hardwareNoiseSuppressorEnabled = defaultHardwareAudioEffectsEnabled(previousProfile) + softwareAudioProcessingEnabled = defaultSoftwareAudioProcessingEnabled(previousProfile) + + val call = mediaManager.call + call.setHardwareNoiseSuppressorEnabled(hardwareNoiseSuppressorEnabled) + call.setHardwareAcousticEchoCancelerEnabled(hardwareNoiseSuppressorEnabled) + applyNoiseCancellationFor(previousProfile) + applyCommunicationAudioModeForProfile(previousProfile) + applyCaptureAudioSourceForProfile(previousProfile) + call.setAudioMaxBitrate( + targetAudioMaxBitrateBps( + previousProfile, + serverBitrateBps = call.audioBitrateFor(previousProfile), + negotiatedBitrateBps = call.negotiatedAudioBitrate(), + ), + ) + } + + /** + * What the noise-cancellation processor was doing before this call switched into music, or null + * while not in music. + * + * Remembered rather than derived from the profile: [Call.setAudioProcessingEnabled] is public + * and released, so an app or user may have turned the processor off deliberately, and deriving + * "voice means on" would switch it back on for them on the way out of music. + */ + private var noiseCancellationBeforeMusic: Boolean? = null + + /** + * Moves the noise-cancellation processor to what [profile] needs and reports whether it got + * there. An unreachable processor counts as applied — nothing is processing. + * + * Music turns it off: where one is configured it is the dominant suppressor, and leaving it + * running makes every other stage inaudible. Leaving music restores what ran before. + */ + private fun applyNoiseCancellationFor(profile: AudioBitrateProfile): Boolean { + val call = mediaManager.call + val isMusic = profile == AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY + val reachable = call.isAudioProcessingReachable() + + val wanted = if (isMusic) { + // Captured once, on the way in. A second request for music must not overwrite it with + // the false this stage itself wrote the first time. + if (noiseCancellationBeforeMusic == null) { + noiseCancellationBeforeMusic = call.isAudioProcessingEnabled() + } + false + } else { + // Nothing remembered means this call was never switched into music, so there is + // nothing to put back and no business touching the processor at all. + noiseCancellationBeforeMusic ?: return true + } + + call.setAudioProcessingEnabled(wanted) + if (!isMusic) { + noiseCancellationBeforeMusic = null + } + return !reachable || call.isAudioProcessingEnabled() == wanted + } + + /** + * Moves each stage of the running pipeline onto [profile], reporting what reached the device. + * + * Stages fail independently and for unrelated reasons, so each is applied and reported on its + * own instead of short-circuiting on the first refusal. + */ + private suspend fun applyProfileToRunningCall( + profile: AudioBitrateProfile, + softwareAudioProcessingChanged: Boolean, + ): AudioProfileResult { + val call = mediaManager.call + + // The audio mode selects the vendor capture graph on some devices, so it goes first: a + // new AudioRecord then opens under the right graph rather than being migrated afterwards. + applyCommunicationAudioModeForProfile(profile) + val captureAudioSourceApplied = applyCaptureAudioSourceForProfile(profile) + if (!captureAudioSourceApplied) { + logger.w { + "[setAudioBitrateProfile] capture source did not move; the audio device " + + "module is still on the previous source" + } + } + + val noiseCancellationApplied = applyNoiseCancellationFor(profile) + + // No audio sender means nothing was ever published, so there is no stage to move: the + // requests are remembered and land when the transceiver is created. Note this is not the + // same as being muted — mute only flips the track's enabled flag, the sender survives. + val audioIsLive = call.hasLiveAudioSender() + + // The setters return false for "unsupported" and "refused" alike, so an absent effect is + // read as satisfied: nothing is suppressing, nothing for the profile to fix. Music turns + // both platform effects off, voice turns them back on, from the same profile bit. + val platformNoiseSuppressorApplied = + call.setHardwareNoiseSuppressorEnabled(hardwareNoiseSuppressorEnabled) || + !call.isHardwareNoiseSuppressorSupported() || + !audioIsLive + if (!platformNoiseSuppressorApplied) { + logger.w { + "[setAudioBitrateProfile] the platform did not take the noise suppressor " + + "request; it will be retried when capture restarts" + } + } + + val platformAcousticEchoCancelerApplied = + call.setHardwareAcousticEchoCancelerEnabled(hardwareNoiseSuppressorEnabled) || + !call.isHardwareAcousticEchoCancelerSupported() || + !audioIsLive + if (!platformAcousticEchoCancelerApplied) { + logger.w { + "[setAudioBitrateProfile] the platform did not take the acoustic echo " + + "canceller request; it will be retried when capture restarts" + } + } + + val maxBitrateBps = targetAudioMaxBitrateBps( + profile, + serverBitrateBps = call.audioBitrateFor(profile), + negotiatedBitrateBps = call.negotiatedAudioBitrate(), + ) + val bitrateReachedASender = call.setAudioMaxBitrate(maxBitrateBps) + val audioMaxBitrateApplied = bitrateReachedASender || !audioIsLive + if (!audioMaxBitrateApplied) { + logger.w { + "[setAudioBitrateProfile] requested $maxBitrateBps bps but the live audio " + + "sender did not take it" + } + } + + // Last, and the only stage that is skipped when an earlier one refused. The goog* + // constraints are fixed when the AudioSource is created, so moving them means building a + // new source and track and swapping the live sender — no setter, and nothing + // [revertProfileDerivedState] can undo without a second swap and another audio gap. Since + // the switch is already going to fail, moving them would strand the one stage that cannot + // be put back. Not moving them is also the truthful answer: the source is still on the + // previous profile's constraints, so the stage reports unapplied either way. + val earlierStagesApplied = captureAudioSourceApplied && noiseCancellationApplied && + platformNoiseSuppressorApplied && platformAcousticEchoCancelerApplied && + audioMaxBitrateApplied + val softwareAudioProcessingApplied = + if (!softwareAudioProcessingChanged) { + // Already the value this profile wants; rebuilding would drop audio for nothing. + true + } else if (!earlierStagesApplied) { + logger.w { + "[setAudioBitrateProfile] skipped the audio pipeline rebuild because an " + + "earlier stage refused; the source keeps the previous constraints" + } + false + } else { + val rebuilt = call.rebuildAudioCapturePipeline() + (rebuilt || !audioIsLive).also { applied -> + if (!applied) { + logger.w { + "[setAudioBitrateProfile] the audio pipeline was not rebuilt; the " + + "next source built will use the new constraints" + } + } + } + } + + return AudioProfileResult( + profile = profile, + audioMaxBitrateBps = maxBitrateBps.takeIf { bitrateReachedASender }, + noiseCancellationApplied = noiseCancellationApplied, + platformNoiseSuppressorApplied = platformNoiseSuppressorApplied, + platformAcousticEchoCancelerApplied = platformAcousticEchoCancelerApplied, + softwareAudioProcessingApplied = softwareAudioProcessingApplied, + audioMaxBitrateApplied = audioMaxBitrateApplied, + captureAudioSourceApplied = captureAudioSourceApplied, + ).also { logger.i { "[setAudioBitrateProfile] $it" } } + } + + /** + * MUSIC_HIGH_QUALITY asks for [AudioManager.MODE_NORMAL]; every voice profile asks for + * [AudioManager.MODE_IN_COMMUNICATION]. Recorded even when routing is not running yet so + * [setup] can apply it before AudioSwitch activates. + */ + private fun applyCommunicationAudioModeForProfile(profile: AudioBitrateProfile) { + setCommunicationAudioModeEnabled( + profile != AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ) + } + + /** + * Puts the live AudioRecord on the capture source [profile] asks for. + * + * Off the main thread: the audio device module rebuilds the record and can join the capture + * thread doing it. Same source as the last call is a no-op in the module. + */ + private suspend fun applyCaptureAudioSourceForProfile(profile: AudioBitrateProfile): Boolean = + withContext(DispatcherProvider.IO) { + mediaManager.call.setCaptureAudioSource(captureAudioSourceFor(profile)) + } + + /** + * Applies the audio mode [setAudioBitrateProfile] asked for. Not a public override — the + * profile is the only way to change it. + */ + internal fun setCommunicationAudioModeEnabled(enabled: Boolean): Boolean { + _communicationAudioModeEnabled.value = enabled + var applied = false + ifAudioHandlerInitialized { applied = it.setCommunicationModeEnabled(enabled) } + if (!applied) { + logger.w { + "[setCommunicationAudioModeEnabled] requested $enabled but audio routing is not " + + "managed for this call; the mode was left alone" + } + } + return applied + } + fun cleanup() { ifAudioHandlerInitialized { it.stop() } cleanupUsbDeviceDetection() + // Dropped with the call that captured it, so a value remembered from a call that ended in + // music cannot be put back onto the next one. + noiseCancellationBeforeMusic = null setupCompleted.set(false) } @@ -986,6 +1299,14 @@ class MicrophoneManager( ), ) + // A mode chosen before routing existed was reported but never applied. Hand it to + // the handler that can now hold it, rather than leaving the flow claiming a state + // the device is not in. Set before starting so the handler re-applies it over the + // communication mode it enters when it activates a device. + if (!_communicationAudioModeEnabled.value) { + audioHandler.setCommunicationModeEnabled(false) + } + logger.d { "[setup] Calling start on instance $audioHandler" } audioHandler.start() } else { @@ -1619,12 +1940,60 @@ class MediaManagerImpl( get() = synchronized(mediaLock) { if (_audioSource == null) { _audioSource = call.peerConnectionFactory.makeAudioSource( - buildAudioConstraints { microphone.audioBitrateProfile.value }, + buildAudioConstraints(microphone.effectiveSoftwareAudioProcessingEnabled), ) } _audioSource!! } + /** + * Rebuilds the audio source and track with the current audio constraints and hands the new + * track to [swap] so a live sender can be moved onto it. + * + * The constraints are fixed when a source is created, so this is the only way to change them + * without rejoining. Runs under [mediaLock] so a racing getter cannot observe a half-swapped + * pair, and rolls the new pair back if [swap] reports nothing was replaced — the live source + * is never torn down for a swap that did not happen. + * + * The replaced track and source are disposed here and nowhere else: the sender is handed the + * new track with ownership left behind, so disposal stays in one place. + * + * @param swap moves the published track over; true when it did, or when there is nothing + * published yet and the new pair should simply become current. + */ + internal fun replaceAudioSourceAndTrack(swap: (AudioTrack) -> Boolean): Boolean = + synchronized(mediaLock) { + if (released) return false + val previousTrack = _audioTrack + val previousSource = _audioSource + val newSource = call.peerConnectionFactory.makeAudioSource( + buildAudioConstraints(microphone.effectiveSoftwareAudioProcessingEnabled), + ) + val newTrack = call.peerConnectionFactory.makeAudioTrack( + source = newSource, + trackId = UUID.randomUUID().toString(), + ) + // A fresh track starts enabled, so the state the swap is replacing has to be carried + // over. The outgoing track is the truth and `microphone.isEnabled` is not: `enable` + // and `disable` only move `_status` when `fromUser` is true, so a lifecycle-driven + // pause leaves the flow reading enabled while the track is off, and reading the flow + // would unmute a call the app had muted. The flow is the fallback for the first + // build, where there is no outgoing track to ask. + newTrack.trySetEnabled(previousTrack?.tryEnabled() ?: microphone.isEnabled.value) + + if (!swap(newTrack)) { + newTrack.dispose() + newSource.dispose() + return false + } + + _audioSource = newSource + _audioTrack = newTrack + previousTrack?.dispose() + previousSource?.dispose() + true + } + // for track ids we emulate the browser behaviour of random UUIDs, doing something different would be confusing // Todo : make audioTrack val in next major release and also move it out of MediaManager var audioTrack: AudioTrack @@ -1682,3 +2051,9 @@ class MediaManagerImpl( } fun MediaStreamTrack.trySetEnabled(enabled: Boolean) = safeCall { setEnabled(enabled) } + +/** + * The track's own enabled flag, or null when it cannot answer — a disposed track throws from + * native rather than reporting a value. + */ +internal fun MediaStreamTrack.tryEnabled(): Boolean? = safeCallWithDefault(null) { enabled() } diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/audio/AudioSwitchController.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/audio/AudioSwitchController.kt index 08848cbb377..e455240a1aa 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/audio/AudioSwitchController.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/audio/AudioSwitchController.kt @@ -17,31 +17,54 @@ package io.getstream.video.android.core.audio import android.content.Context +import android.media.AudioManager +import androidx.core.content.getSystemService import com.twilio.audioswitch.AudioDevice import com.twilio.audioswitch.AudioDeviceChangeListener import com.twilio.audioswitch.AudioSwitch +import io.getstream.log.taggedLogger import io.getstream.video.android.core.audio.AudioSwitchHandler.Companion.onAudioFocusChangeListener +import io.getstream.video.android.core.utils.audioModeName internal class AudioSwitchController( private val context: Context, private val preferredDeviceList: List>, private val audioDeviceChangeListener: AudioDeviceChangeListener, ) : AudioHandler { + private val logger by taggedLogger("Audio:SwitchController") + private var audioSwitch: AudioSwitch? = null private var isActivated = false + /** + * The audio mode asked for through [setCommunicationModeEnabled], or null while whatever + * AudioSwitch chose stands. Written from the caller's thread and read from the audio scope + * that drives [selectDevice]. + */ + @Volatile + private var requestedAudioMode: Int? = null + override fun start() { if (audioSwitch != null) return audioSwitch = getAudioSwitch() isActivated = false - audioSwitch?.start(audioDeviceChangeListener) + // Wrapped rather than handed over directly: activating AudioSwitch puts the device back in + // MODE_IN_COMMUNICATION, and route changes arrive here and nowhere else, so this is the + // only place an earlier request can be re-applied over them. + audioSwitch?.start { devices, selected -> + applyRequestedAudioMode() + audioDeviceChangeListener(devices, selected) + } } override fun stop() { audioSwitch?.stop() audioSwitch = null isActivated = false + // AudioSwitch restores the mode it saved before activating, so a request from this session + // must not be re-applied over the mode the next one starts from. + requestedAudioMode = null } fun selectDevice(device: AudioDevice?) { @@ -53,6 +76,48 @@ internal class AudioSwitchController( switch.activate() isActivated = true } + // activate() puts the device in MODE_IN_COMMUNICATION, and a route change can do it again + // later, so a request made earlier is re-applied rather than assumed to still hold. + applyRequestedAudioMode() + } + + /** + * Chooses between [AudioManager.MODE_IN_COMMUNICATION] and [AudioManager.MODE_NORMAL] for the + * running call. + * + * Some vendors pick their VoIP capture chain from the audio mode rather than from the requested + * `MediaRecorder.AudioSource`, and that chain sits below the `AudioEffect` API where the audio + * device module flags and source constraints cannot reach it. Leaving communication mode is the + * only lever on it, at the cost of communication routing and Bluetooth capture — SCO carries + * the headset microphone and only runs in communication mode. + * + * Remembered and re-applied on every route change, because AudioSwitch sets the mode itself + * when it activates a device. + * + * @return true when the mode was applied, false when there is no [AudioManager] to apply it to. + */ + fun setCommunicationModeEnabled(enabled: Boolean): Boolean { + requestedAudioMode = if (enabled) { + AudioManager.MODE_IN_COMMUNICATION + } else { + AudioManager.MODE_NORMAL + } + return applyRequestedAudioMode() + } + + private fun applyRequestedAudioMode(): Boolean { + val mode = requestedAudioMode ?: return false + val audioManager = context.getSystemService() ?: return false + + if (audioManager.mode == mode) return true + + logger.i { + "[applyRequestedAudioMode] ${audioModeName( + audioManager.mode, + )} -> ${audioModeName(mode)}" + } + audioManager.mode = mode + return true } fun getAudioSwitch(): AudioSwitch { diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/audio/AudioSwitchDecorator.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/audio/AudioSwitchDecorator.kt index 898e8ec93e8..02997925276 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/audio/AudioSwitchDecorator.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/audio/AudioSwitchDecorator.kt @@ -64,4 +64,12 @@ internal class AudioSwitchDecorator(private val controller: AudioSwitchControlle val scope = ensureScope() ?: return scope.launch { controller.selectDevice(device) } } + + /** + * Runs inline rather than on the audio scope, unlike the calls above: setting the mode is a + * single cheap call with no device discovery behind it, and the caller is told whether the + * platform took it. + */ + fun setCommunicationModeEnabled(enabled: Boolean): Boolean = + controller.setCommunicationModeEnabled(enabled) } diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/RtcSession.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/RtcSession.kt index 057de0a0198..9d4d71f4121 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/RtcSession.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/RtcSession.kt @@ -566,6 +566,64 @@ public class RtcSession internal constructor( return typed } + /** + * Moves publishing onto a freshly built audio track, so audio-source constraints that are + * fixed at source creation can be changed without rejoining. + * + * No renegotiation: replacing a sender's track does not change the SDP. When nothing is + * published yet the new pair simply becomes current. + * + * @return true when the pipeline was rebuilt. + */ + internal fun rebuildAudioCapturePipeline(): Boolean = + call.mediaManager.replaceAudioSourceAndTrack { newTrack -> + val publisher = publisher.value + // Nothing published yet — no publisher, or a publisher with no audio sender + // (joined muted) — so there is no sender to move and the new pair stands. The + // next publishStream reads mediaManager.audioSource, which is now this pair. + if (publisher == null || !publisher.hasLiveAudioSender()) { + return@replaceAudioSourceAndTrack true + } + + publisher.replaceAudioTrack(newTrack).also { replaced -> + if (replaced) { + // The local participant's track has to point at what is actually being sent. + setLocalTrack( + TrackType.TRACK_TYPE_AUDIO, + AudioTrack( + streamId = buildTrackId(TrackType.TRACK_TYPE_AUDIO), + audio = newTrack, + ), + ) + } + } + } + + /** + * Applies a maximum audio bitrate to the live publisher, with no renegotiation. + * + * @return true when a live audio sender accepted it. + */ + internal fun setAudioMaxBitrate(maxBitrateBps: Int): Boolean = + publisher.value?.setAudioMaxBitrate(maxBitrateBps) ?: false + + /** + * Whether an audio sender exists to take a bitrate ceiling. No publisher is the same as + * no sender: the next transceiver is built from the published profile. + */ + internal fun hasLiveAudioSender(): Boolean = + publisher.value?.hasLiveAudioSender() ?: false + + /** The maximum bitrate on the live audio sender, or null when nothing is publishing audio. */ + internal fun audioMaxBitrate(): Int? = publisher.value?.audioMaxBitrate() + + /** The audio bitrate the SFU negotiated at join, or null when nothing publishes audio. */ + internal fun negotiatedAudioBitrate(): Int? = publisher.value?.negotiatedAudioBitrate() + + /** The bitrate the SFU offers for [profile], or null when it named none. */ + internal fun audioBitrateFor(profile: stream.video.sfu.models.AudioBitrateProfile): Int? = + publisher.value?.audioBitrateFor(profile) + /** * Connection and WebRTC. */ diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/components/CallMediaManager.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/components/CallMediaManager.kt index 8120b7cdc16..cc8cf5f2a02 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/components/CallMediaManager.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/components/CallMediaManager.kt @@ -87,6 +87,14 @@ internal class CallMediaManager( */ private var desiredAudioProcessingEnabled: Boolean = false + /** + * Platform noise-suppressor and echo-canceller states this call asked for, or null while the + * builder defaults stand. Kept alongside [desiredAudioProcessingEnabled] and for the same + * reason: the wanted state must outlive the factory so a recreation cannot silently drop it. + */ + private var desiredHardwareNoiseSuppressorEnabled: Boolean? = null + private var desiredHardwareAcousticEchoCancelerEnabled: Boolean? = null + var peerConnectionFactory: StreamPeerConnectionFactory get() { if (_peerConnectionFactory == null) { @@ -100,6 +108,12 @@ internal class CallMediaManager( webRtcLoggingLevel = clientImpl.loggingLevel.webRtcLoggingLevel, ).also { factory -> factory.setAudioProcessingEnabled(desiredAudioProcessingEnabled) + desiredHardwareNoiseSuppressorEnabled?.let { + factory.setHardwareNoiseSuppressorEnabled(it) + } + desiredHardwareAcousticEchoCancelerEnabled?.let { + factory.setHardwareAcousticEchoCancelerEnabled(it) + } } } return _peerConnectionFactory!! @@ -313,6 +327,14 @@ internal class CallMediaManager( fun isAudioProcessingEnabledIfCreated(): Boolean = _peerConnectionFactory?.isAudioProcessingEnabled() ?: false + /** + * Whether a noise-cancellation processor is wired into this call's native factory at all, so + * "refused to change" can be told from "there is no processor". Without it, a call configured + * with no [org.webrtc.ManagedAudioProcessingFactory] looks like a failure every time. + */ + fun isAudioProcessingReachable(): Boolean = + _peerConnectionFactory?.hasAudioProcessingAttached() ?: false + /** * Whether this call wants audio processing, whether or not a factory exists to run it yet. * @@ -348,6 +370,39 @@ internal class CallMediaManager( return isAudioProcessingEnabled() } + // None of the following builds a factory: like [setAudioProcessingEnabled], one created here + // would capture the pre-join audio bitrate profile and pin it for the rest of the call. The + // wanted state is recorded, and a factory built later picks it up on creation. The setters + // return true when the running capture session accepted the change. + + fun setHardwareNoiseSuppressorEnabled(enabled: Boolean): Boolean { + desiredHardwareNoiseSuppressorEnabled = enabled + return _peerConnectionFactory?.setHardwareNoiseSuppressorEnabled(enabled) ?: false + } + + fun setHardwareAcousticEchoCancelerEnabled(enabled: Boolean): Boolean { + desiredHardwareAcousticEchoCancelerEnabled = enabled + return _peerConnectionFactory?.setHardwareAcousticEchoCancelerEnabled(enabled) ?: false + } + + fun setCaptureAudioSource(audioSource: Int): Boolean = + _peerConnectionFactory?.setCaptureAudioSource(audioSource) ?: false + + fun isHardwareNoiseSuppressorSupported(): Boolean = + _peerConnectionFactory?.isHardwareNoiseSuppressorSupported() ?: false + + fun isHardwareAcousticEchoCancelerSupported(): Boolean = + _peerConnectionFactory?.isHardwareAcousticEchoCancelerSupported() ?: false + + /** Forgets the wanted states, so nothing is re-applied after the call ends. */ + fun resetDesiredHardwareNoiseSuppressor() { + desiredHardwareNoiseSuppressorEnabled = null + } + + fun resetDesiredHardwareAcousticEchoCanceler() { + desiredHardwareAcousticEchoCancelerEnabled = null + } + /** Disables all local capture devices. Used when leaving the call. */ fun disableLocalCapture() { stopScreenSharing() @@ -359,6 +414,8 @@ internal class CallMediaManager( // The wanted state must not outlive the call: a reused Call would otherwise re-apply it // to the factory built for the next session. resetDesiredAudioProcessing() + resetDesiredHardwareNoiseSuppressor() + resetDesiredHardwareAcousticEchoCanceler() mediaManager.cleanup() } } diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/connection/Publisher.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/connection/Publisher.kt index fa6ef27fe61..d46bcb5c9da 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/connection/Publisher.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/connection/Publisher.kt @@ -54,12 +54,14 @@ import org.webrtc.MediaStream import org.webrtc.MediaStreamTrack import org.webrtc.PeerConnection import org.webrtc.RtpParameters +import org.webrtc.RtpSender import org.webrtc.RtpTransceiver import org.webrtc.RtpTransceiver.RtpTransceiverDirection import org.webrtc.RtpTransceiver.RtpTransceiverInit import org.webrtc.SessionDescription import stream.video.sfu.event.VideoLayerSetting import stream.video.sfu.event.VideoSender +import stream.video.sfu.models.AudioBitrateProfile import stream.video.sfu.models.ErrorCode import stream.video.sfu.models.PublishOption import stream.video.sfu.models.TrackInfo @@ -347,6 +349,102 @@ internal class Publisher( } } + /** + * Swaps the track on the live audio sender, without renegotiating. + * + * `takeOwnership = false` is deliberate: [MediaManagerImpl] owns and disposes the audio track, + * and [RtpSender.setTrack] disposes the track it holds only when it owns it, so disposal stays + * in one place and the replaced track stays valid until the caller disposes it. + * + * @return true when a live audio sender was found and accepted the track. + */ + internal fun replaceAudioTrack(newTrack: MediaStreamTrack): Boolean { + val senders = safeCallWithDefault(emptyList()) { + transceiverCache.getByTrackType(TrackType.TRACK_TYPE_AUDIO).mapNotNull { it.sender } + } + if (senders.isEmpty()) { + logger.d { "[replaceAudioTrack] no audio sender to replace the track on" } + return false + } + return senders.any { sender -> + safeCallWithDefault(false) { sender.setTrack(newTrack, false) }.also { replaced -> + logger.d { "[replaceAudioTrack] replaced: $replaced, track: ${newTrack.id()}" } + } + } + } + + /** + * Sets the maximum bitrate on the live audio sender. + * + * Applied through the sender's [RtpParameters], like the video layers, so it takes effect on + * the running encoder with no renegotiation — the audio bitrate is not carried in the SDP on + * our side, it rides entirely on the encoding. + * + * `setParameters` is called for its result rather than through the `parameters` property: + * WebRTC validates the encodings and answers with a boolean, and assigning the property throws + * that answer away, which would report a rejected update as an applied one. + * + * @return true when a live audio sender accepted the new parameters. + */ + internal fun setAudioMaxBitrate(maxBitrateBps: Int): Boolean { + val senders = audioSenders() + if (senders.isEmpty()) { + logger.d { "[setAudioMaxBitrate] no audio sender to apply $maxBitrateBps to" } + return false + } + return senders.any { sender -> + safeCallWithDefault(false) { + val params = sender.parameters ?: return@safeCallWithDefault false + if (params.encodings.isEmpty()) return@safeCallWithDefault false + params.encodings.forEach { it.maxBitrateBps = maxBitrateBps } + sender.setParameters(params).also { accepted -> + logger.d { + "[setAudioMaxBitrate] maxBitrateBps: $maxBitrateBps, accepted: $accepted" + } + } + } + } + } + + /** + * Whether an audio sender exists to take a bitrate ceiling. No sender is not a refused stage: + * [computeTransceiverEncodings] reads the published profile when the transceiver is added, so + * the next publish already has the right ceiling. + */ + internal fun hasLiveAudioSender(): Boolean = audioSenders().isNotEmpty() + + private fun audioSenders(): List = safeCallWithDefault(emptyList()) { + transceiverCache.getByTrackType(TrackType.TRACK_TYPE_AUDIO).mapNotNull { it.sender } + } + + /** + * The bitrate the SFU offers for [profile], or null when it named none. The server sends one + * per profile in `PublishOption.audio_bitrate_profiles`, so a mid-call switch does not have to + * invent a number — this is what a freshly created audio transceiver would be given. + */ + internal fun audioBitrateFor(profile: AudioBitrateProfile): Int? = safeCallWithDefault(null) { + publishOptions.firstOrNull { it.track_type == TrackType.TRACK_TYPE_AUDIO } + ?.audio_bitrate_profiles + ?.firstOrNull { it.profile == profile } + ?.bitrate + ?.takeIf { it > 0 } + } + + /** + * The audio bitrate the SFU negotiated at join for the profile the call joined with, or null + * when it publishes no audio — the value to restore when a switch to music is undone. + */ + internal fun negotiatedAudioBitrate(): Int? = safeCallWithDefault(null) { + publishOptions.firstOrNull { it.track_type == TrackType.TRACK_TYPE_AUDIO }?.bitrate + } + + /** The maximum bitrate currently set on the live audio sender, or null when unknown. */ + internal fun audioMaxBitrate(): Int? = safeCallWithDefault(null) { + transceiverCache.getByTrackType(TrackType.TRACK_TYPE_AUDIO) + .mapNotNull { it.sender?.parameters?.encodings?.firstOrNull()?.maxBitrateBps } + .firstOrNull() + } + @VisibleForTesting public fun newTrackFromSource(trackType: TrackType): MediaStreamTrack { return when (trackType) { diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/connection/StreamPeerConnectionFactory.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/connection/StreamPeerConnectionFactory.kt index 46d28258ea8..fd15f531dc2 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/connection/StreamPeerConnectionFactory.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/connection/StreamPeerConnectionFactory.kt @@ -19,8 +19,11 @@ package io.getstream.video.android.core.call.connection import android.content.Context import android.media.AudioAttributes import android.media.AudioDeviceInfo +import android.media.AudioManager import android.os.Build import androidx.annotation.RequiresApi +import androidx.annotation.VisibleForTesting +import androidx.core.content.getSystemService import io.getstream.log.taggedLogger import io.getstream.video.android.core.MediaManagerImpl import io.getstream.video.android.core.api.SignalServerService @@ -35,6 +38,10 @@ import io.getstream.video.android.core.model.StreamPeerType import io.getstream.video.android.core.model.toPeerType import io.getstream.video.android.core.trace.PeerConnectionTraceKey import io.getstream.video.android.core.trace.Tracer +import io.getstream.video.android.core.utils.audioSourceName +import io.getstream.video.android.core.utils.captureAudioSourceFor +import io.getstream.video.android.core.utils.defaultHardwareAudioEffectsEnabled +import io.getstream.video.android.core.utils.formatAudioCaptureKnobs import io.getstream.video.android.core.utils.safeCallWithDefault import kotlinx.coroutines.CoroutineScope import org.webrtc.AudioSource @@ -213,6 +220,92 @@ public class StreamPeerConnectionFactory( private var adm: JavaAudioDeviceModule? = null + /** + * Platform noise-suppressor state this call asked for at runtime, or null while it has never + * been changed and the builder default stands. + * + * Held here because the effect itself does not survive a capture restart: the audio device + * module attaches it to the current recording session and drops it when recording stops, then + * rebuilds it from the *builder* flag on the next start. Without re-applying, a caller's + * choice would silently revert on the next reconnect or route change. + */ + @Volatile + private var desiredHardwareNoiseSuppressorEnabled: Boolean? = null + + /** + * Same lifetime as [desiredHardwareNoiseSuppressorEnabled]: the built-in AEC is attached + * to the current recording session and rebuilt from the builder flag on the next start. + */ + @Volatile + private var desiredHardwareAcousticEchoCancelerEnabled: Boolean? = null + + /** + * Enables or disables the platform noise suppressor on the live recording session. + * + * Unlike the builder flag this takes effect immediately, but only while audio is being + * captured. Returns false when nothing changed: no module yet, no active capture, or a device + * that is unsupported or ignores the request. The request is remembered either way. + */ + internal fun setHardwareNoiseSuppressorEnabled(enabled: Boolean): Boolean { + desiredHardwareNoiseSuppressorEnabled = enabled + return safeCallWithDefault(false) { + adm?.setNoiseSuppressorEnabled(enabled) ?: false + } + } + + /** + * Whether this device has a platform noise suppressor at all — the difference + * [setHardwareNoiseSuppressorEnabled] cannot report, since it returns false for "unsupported" + * and "refused" alike. Nothing suppressing means the profile is satisfied, not failed. + */ + internal fun isHardwareNoiseSuppressorSupported(): Boolean = safeCallWithDefault(false) { + JavaAudioDeviceModule.isBuiltInNoiseSuppressorSupported() + } + + /** + * Re-applies [desiredHardwareNoiseSuppressorEnabled] to the recording session that has just + * started. No-op when the caller never expressed a preference, so the builder default stands. + */ + @VisibleForTesting + internal fun reapplyHardwareNoiseSuppressor() { + val desired = desiredHardwareNoiseSuppressorEnabled ?: return + val applied = safeCallWithDefault(false) { + adm?.setNoiseSuppressorEnabled(desired) ?: false + } + audioLogger.d { + "[reapplyHardwareNoiseSuppressor] desired: $desired, applied: $applied" + } + } + + /** + * Counterpart of [setHardwareNoiseSuppressorEnabled] for the acoustic echo canceller. The + * request is re-applied when capture restarts, including after [setCaptureAudioSource] rebuilds + * AudioRecord — without that, a source change would resurrect AEC from the builder flag. + */ + internal fun setHardwareAcousticEchoCancelerEnabled(enabled: Boolean): Boolean { + desiredHardwareAcousticEchoCancelerEnabled = enabled + return safeCallWithDefault(false) { + adm?.setAcousticEchoCancelerEnabled(enabled) ?: false + } + } + + /** Whether this device has a platform acoustic echo canceller at all. */ + internal fun isHardwareAcousticEchoCancelerSupported(): Boolean = safeCallWithDefault(false) { + JavaAudioDeviceModule.isBuiltInAcousticEchoCancelerSupported() + } + + /** Re-applies [desiredHardwareAcousticEchoCancelerEnabled] to a session that has just started. */ + @VisibleForTesting + internal fun reapplyHardwareAcousticEchoCanceler() { + val desired = desiredHardwareAcousticEchoCancelerEnabled ?: return + val applied = safeCallWithDefault(false) { + adm?.setAcousticEchoCancelerEnabled(desired) ?: false + } + audioLogger.d { + "[reapplyHardwareAcousticEchoCanceler] desired: $desired, applied: $applied" + } + } + @Volatile private var pendingPreferredInputDevice: AudioDeviceInfo? = null @@ -286,25 +379,66 @@ public class StreamPeerConnectionFactory( .also { factoryCreated = true } } + /** + * Puts the audio device module on [audioSource]. + * + * Android cannot change the source of an open AudioRecord, so this takes effect on the next + * one the module opens — which the audio-mode change accompanying a profile switch triggers. + * Must not be called from the main thread: the module can join the capture thread. + * + * @return true when the module is now on [audioSource], false when no module exists yet or it + * kept the previous source. Reflects the module's requested source, not a live AudioRecord. + */ + internal fun setCaptureAudioSource(audioSource: Int): Boolean { + return safeCallWithDefault(false) { + val module = adm ?: return@safeCallWithDefault false + module.setAudioSource(audioSource) + module.audioSource == audioSource + }.also { applied -> + audioLogger.i { + "[setCaptureAudioSource] requested=${audioSourceName(audioSource)} " + + "applied=$applied ${currentAudioCaptureKnobs()}" + } + } + } + + private fun currentAudioCaptureKnobs(): String { + val profile = audioBitrateProfileProvider?.invoke() ?: audioBitrateProfile + return formatAudioCaptureKnobs( + profile = profile, + audioMode = context.getSystemService()?.mode, + audioSource = safeCallWithDefault(null) { adm?.audioSource }, + hwAec = desiredHardwareAcousticEchoCancelerEnabled + ?: defaultHardwareAudioEffectsEnabled(profile), + hwNs = desiredHardwareNoiseSuppressorEnabled + ?: defaultHardwareAudioEffectsEnabled(profile), + ) + } + private fun initAudioDeviceModule(): JavaAudioDeviceModule? { // Capture the audio bitrate profile when initializing the audio device module audioBitrateProfile = audioBitrateProfileProvider?.invoke() - val isMusicHighQuality = audioBitrateProfile == - stream.video.sfu.models.AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY - val useHardwareAcousticEchoCanceler = if (isMusicHighQuality) { - false - } else { - Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q - } - val useHardwareNoiseSuppressor = if (isMusicHighQuality) { - false - } else { - Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q + val audioSource = captureAudioSourceFor(audioBitrateProfile) + val useHardwareAcousticEchoCanceler = + defaultHardwareAudioEffectsEnabled(audioBitrateProfile) + val useHardwareNoiseSuppressor = + defaultHardwareAudioEffectsEnabled(audioBitrateProfile) + + audioLogger.i { + "[initAudioDeviceModule] " + + formatAudioCaptureKnobs( + profile = audioBitrateProfile, + audioMode = context.getSystemService()?.mode, + audioSource = audioSource, + hwAec = useHardwareAcousticEchoCanceler, + hwNs = useHardwareNoiseSuppressor, + ) } adm = JavaAudioDeviceModule .builder(context) + .setAudioSource(audioSource) .setUseHardwareAcousticEchoCanceler(useHardwareAcousticEchoCanceler) .apply { if (audioUsageProvider.invoke() != defaultAudioUsage) { @@ -352,7 +486,13 @@ public class StreamPeerConnectionFactory( .setAudioRecordStateCallback(object : JavaAudioDeviceModule.AudioRecordStateCallback { override fun onWebRtcAudioRecordStart() { - audioLogger.d { "[onWebRtcAudioRecordStart] no args" } + audioLogger.d { + "[onWebRtcAudioRecordStart] ${currentAudioCaptureKnobs()}" + } + // The platform effects are rebuilt from the builder flags for every recording + // session, so a runtime override has to be re-applied here or it is lost. + reapplyHardwareNoiseSuppressor() + reapplyHardwareAcousticEchoCanceler() } override fun onWebRtcAudioRecordStop() { diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/utils/CallClientUtils.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/utils/CallClientUtils.kt index 68940b9c139..171e8f3c6ee 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/utils/CallClientUtils.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/utils/CallClientUtils.kt @@ -16,6 +16,9 @@ package io.getstream.video.android.core.utils +import android.media.AudioManager +import android.media.MediaRecorder +import android.os.Build import io.getstream.video.android.core.model.IceServer import org.webrtc.MediaConstraints import org.webrtc.PeerConnection @@ -194,14 +197,120 @@ internal val iceRestartConstraints = MediaConstraints().apply { optional.add(MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true")) } +// Builder defaults for [profile], shared by the module builder and the mid-call setters so the +// value reported to callers cannot drift from what the pipeline was actually built with. + +/** Platform (hardware) audio effects: run in the audio device module. */ +@JvmSynthetic +internal fun defaultHardwareAudioEffectsEnabled(profile: AudioBitrateProfile?): Boolean = + profile != AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY && + Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q + +/** + * WebRTC's own software audio processing: a separate stage from + * [defaultHardwareAudioEffectsEnabled], fixed for the lifetime of the audio source it is passed to. + */ +@JvmSynthetic +internal fun defaultSoftwareAudioProcessingEnabled(profile: AudioBitrateProfile?): Boolean = + profile != AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY + +/** + * Capture source for [profile]. MUSIC_HIGH_QUALITY asks for [MediaRecorder.AudioSource.MIC] so the + * platform builds the ordinary record graph instead of the VoIP one; + * [MediaRecorder.AudioSource.VOICE_COMMUNICATION] is WebRTC's default. + */ +@JvmSynthetic +internal fun captureAudioSourceFor(profile: AudioBitrateProfile?): Int = + if (profile == AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY) { + MediaRecorder.AudioSource.MIC + } else { + MediaRecorder.AudioSource.VOICE_COMMUNICATION + } + +/** Readable name for [AudioManager.mode], so logs do not force a lookup of 0 vs 3. */ +@JvmSynthetic +internal fun audioModeName(mode: Int?): String = when (mode) { + AudioManager.MODE_NORMAL -> "MODE_NORMAL" + AudioManager.MODE_RINGTONE -> "MODE_RINGTONE" + AudioManager.MODE_IN_CALL -> "MODE_IN_CALL" + AudioManager.MODE_IN_COMMUNICATION -> "MODE_IN_COMMUNICATION" + else -> if (mode == null) "unset" else "MODE_UNKNOWN($mode)" +} + +/** Readable name for [MediaRecorder.AudioSource], so logs do not force a lookup of 1 vs 7. */ +@JvmSynthetic +internal fun audioSourceName(source: Int?): String = when (source) { + MediaRecorder.AudioSource.DEFAULT -> "DEFAULT" + MediaRecorder.AudioSource.MIC -> "MIC" + MediaRecorder.AudioSource.VOICE_RECOGNITION -> "VOICE_RECOGNITION" + MediaRecorder.AudioSource.VOICE_COMMUNICATION -> "VOICE_COMMUNICATION" + MediaRecorder.AudioSource.UNPROCESSED -> "UNPROCESSED" + else -> if (source == null) "unset" else "AUDIO_SOURCE_UNKNOWN($source)" +} + +/** + * One line for the capture knobs that decide whether music survives the vendor graph. + * Same order everywhere so the three log sites can be grepped as a single story. + */ +@JvmSynthetic +internal fun formatAudioCaptureKnobs( + profile: AudioBitrateProfile?, + audioMode: Int?, + audioSource: Int?, + hwAec: Boolean?, + hwNs: Boolean?, +): String = "profile=${profile ?: "unset"} " + + "audioMode=${audioModeName(audioMode)} " + + "audioSource=${audioSourceName(audioSource)} " + + "hwAec=${hwAec ?: "unset"} " + + "hwNs=${hwNs ?: "unset"}" + +/** Roughly what the SFU asks for on the standard voice profile. */ +internal const val VOICE_MAX_AUDIO_BITRATE_BPS: Int = 64_000 + +/** Roughly what the SFU asks for on the music profile. */ +internal const val MUSIC_MAX_AUDIO_BITRATE_BPS: Int = 128_000 + +/** + * The maximum audio bitrate to put on the publisher for [profile]. + * + * The SFU is not asked again mid-call, so switching profiles has to move the encoder's own ceiling. + * It does not have to guess at the number, though: the server sends one bitrate per profile in + * `PublishOption.audio_bitrate_profiles`, and [serverBitrateBps] is the one for [profile] — the + * same value a freshly created audio transceiver would be given. Only when the server named none + * does this fall back to what was negotiated at join, and then to a constant. + */ +@JvmSynthetic +internal fun targetAudioMaxBitrateBps( + profile: AudioBitrateProfile, + serverBitrateBps: Int?, + negotiatedBitrateBps: Int?, +): Int { + serverBitrateBps?.takeIf { it > 0 }?.let { return it } + return if (profile == AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY) { + maxOf(MUSIC_MAX_AUDIO_BITRATE_BPS, negotiatedBitrateBps ?: 0) + } else { + negotiatedBitrateBps?.takeIf { it > 0 } ?: VOICE_MAX_AUDIO_BITRATE_BPS + } +} + @JvmSynthetic internal fun buildAudioConstraints( audioBitrateProfileProvider: (() -> AudioBitrateProfile)? = null, -): MediaConstraints { +): MediaConstraints = buildAudioConstraints( + defaultSoftwareAudioProcessingEnabled(audioBitrateProfileProvider?.invoke()), +) + +/** + * Builds the audio-source constraints with WebRTC's software audio processing explicitly on or off. + * + * These are fixed when the [org.webrtc.AudioSource] is created, so changing them means building a + * new source and swapping the published track. + */ +@JvmSynthetic +internal fun buildAudioConstraints(softwareAudioProcessingEnabled: Boolean): MediaConstraints { val mediaConstraints = MediaConstraints() - val isMusicHighQuality = audioBitrateProfileProvider?.invoke() == - AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY - val constraintValue = if (isMusicHighQuality) false else true + val constraintValue = softwareAudioProcessingEnabled val items = listOf( MediaConstraints.KeyValuePair( diff --git a/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/AudioProfileResultTest.kt b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/AudioProfileResultTest.kt new file mode 100644 index 00000000000..f4f5cd75cee --- /dev/null +++ b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/AudioProfileResultTest.kt @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2014-2026 Stream.io Inc. All rights reserved. + * + * Licensed under the Stream License; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://github.com/GetStream/stream-video-android/blob/main/LICENSE + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.getstream.video.android.core + +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import stream.video.sfu.models.AudioBitrateProfile + +/** + * Covers [AudioProfileResult.complete], which decides whether `audioBitrateProfile` moves. + * + * Every stage has to hold. A single stage left behind means something is still processing the + * audio the old way, and a toggle sitting on MUSIC while a suppressor eats the music is + * indistinguishable from the bug this feature exists to fix — so the flag is only true when all + * six agree, and each one has to be able to veto on its own. + */ +class AudioProfileResultTest { + + private fun result( + noiseCancellation: Boolean = true, + platformSuppressor: Boolean = true, + platformAec: Boolean = true, + softwareProcessing: Boolean = true, + maxBitrate: Boolean = true, + captureSource: Boolean = true, + ) = AudioProfileResult( + profile = AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + audioMaxBitrateBps = 128_000, + noiseCancellationApplied = noiseCancellation, + platformNoiseSuppressorApplied = platformSuppressor, + platformAcousticEchoCancelerApplied = platformAec, + softwareAudioProcessingApplied = softwareProcessing, + audioMaxBitrateApplied = maxBitrate, + captureAudioSourceApplied = captureSource, + ) + + @Test + fun `all six stages applied is complete`() { + assertTrue(result().complete) + } + + @Test + fun `the noise-cancellation processor alone can veto`() { + assertFalse(result(noiseCancellation = false).complete) + } + + @Test + fun `the platform noise suppressor alone can veto`() { + assertFalse(result(platformSuppressor = false).complete) + } + + @Test + fun `the platform acoustic echo canceller alone can veto`() { + assertFalse(result(platformAec = false).complete) + } + + @Test + fun `the software audio processing stage alone can veto`() { + assertFalse(result(softwareProcessing = false).complete) + } + + @Test + fun `the bitrate stage alone can veto`() { + // The quietest failure of the four: nothing sounds different, the ceiling is just wrong. + assertFalse(result(maxBitrate = false).complete) + } + + @Test + fun `the capture audio source alone can veto`() { + assertFalse(result(captureSource = false).complete) + } +} diff --git a/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/CallAudioProfileBridgeTest.kt b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/CallAudioProfileBridgeTest.kt new file mode 100644 index 00000000000..7261059d436 --- /dev/null +++ b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/CallAudioProfileBridgeTest.kt @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2014-2026 Stream.io Inc. All rights reserved. + * + * Licensed under the Stream License; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://github.com/GetStream/stream-video-android/blob/main/LICENSE + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.getstream.video.android.core + +import android.media.MediaRecorder +import io.getstream.video.android.core.base.IntegrationTestBase +import io.getstream.video.android.core.call.RtcSession +import io.getstream.video.android.core.call.connection.StreamPeerConnectionFactory +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.test.runTest +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import stream.video.sfu.models.AudioBitrateProfile +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * Covers the bridges [MicrophoneManager.setAudioBitrateProfile] reaches the call through. The + * microphone manager cannot see the media component or the session, so every stage of a mid-call + * profile switch travels one of these methods. + * + * What is worth guarding is the answer each gives when its target is absent, because that answer + * is what [AudioProfileResult] reports as a stage. A missing session means nothing is publishing, + * which is a stage that could not be reached — except for the capture-pipeline rebuild, where it + * means the constraints are read fresh at the next build and the change already holds. + */ +@RunWith(RobolectricTestRunner::class) +class CallAudioProfileBridgeTest : IntegrationTestBase(connectCoordinatorWS = false) { + + private fun call(): Call = client.call("default", randomUUID()) + + /** + * Injected outside any `every { }` block on purpose: [injectSession] stubs the session's + * socket state itself, and MockK cannot record one stubbing block inside another. + */ + private fun Call.withSession(): RtcSession = + mockk(relaxed = true).also { injectSession(it) } + + private fun Call.withFactory(): StreamPeerConnectionFactory = + mockk(relaxed = true).also { injectPeerConnectionFactory(it) } + + //region session-backed bridges + + @Test + fun `setAudioMaxBitrate reaches the session`() = runTest { + val call = call() + val session = call.withSession() + every { session.setAudioMaxBitrate(128_000) } returns true + + assertTrue(call.setAudioMaxBitrate(128_000)) + } + + @Test + fun `setAudioMaxBitrate reports false before a session exists`() = runTest { + // Nothing is publishing, so the live setter has nowhere to put the ceiling. + assertFalse(call().setAudioMaxBitrate(128_000)) + } + + @Test + fun `hasLiveAudioSender is false before a session exists`() = runTest { + assertFalse(call().hasLiveAudioSender()) + } + + @Test + fun `hasLiveAudioSender reaches the session`() = runTest { + val call = call() + val session = call.withSession() + every { session.hasLiveAudioSender() } returns true + + assertTrue(call.hasLiveAudioSender()) + } + + @Test + fun `audioMaxBitrate reads through the session`() = runTest { + val call = call() + val session = call.withSession() + every { session.audioMaxBitrate() } returns 128_000 + + assertEquals(128_000, call.audioMaxBitrate()) + } + + @Test + fun `negotiatedAudioBitrate reads through the session`() = runTest { + val call = call() + val session = call.withSession() + every { session.negotiatedAudioBitrate() } returns 64_000 + + assertEquals(64_000, call.negotiatedAudioBitrate()) + } + + @Test + fun `audioBitrateFor reads through the session`() = runTest { + val call = call() + val session = call.withSession() + every { + session.audioBitrateFor(AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY) + } returns 128_000 + + assertEquals( + 128_000, + call.audioBitrateFor(AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY), + ) + } + + @Test + fun `the bitrate readers are null before a session exists`() = runTest { + val call = call() + + assertNull(call.audioMaxBitrate()) + assertNull(call.negotiatedAudioBitrate()) + assertNull( + call.audioBitrateFor(AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY), + ) + } + + @Test + fun `rebuildAudioCapturePipeline reaches the session`() = runTest { + val call = call() + val session = call.withSession() + every { session.rebuildAudioCapturePipeline() } returns true + + assertTrue(call.rebuildAudioCapturePipeline()) + } + + @Test + fun `rebuildAudioCapturePipeline succeeds before a session exists`() = runTest { + // The source has not been built yet and is built from the current constraints when it is, + // so there is nothing to rebuild and the change already holds. Reporting a failed stage + // here would fail a switch that in fact took. + assertTrue(call().rebuildAudioCapturePipeline()) + } + + //endregion + + //region media-backed bridges + + @Test + fun `setCaptureAudioSource reaches the media component`() = runTest { + val call = call() + val factory = call.withFactory() + every { factory.setCaptureAudioSource(MediaRecorder.AudioSource.MIC) } returns true + + assertTrue(call.setCaptureAudioSource(MediaRecorder.AudioSource.MIC)) + verify { factory.setCaptureAudioSource(MediaRecorder.AudioSource.MIC) } + } + + @Test + fun `setCaptureAudioSource reports false before a factory exists`() = runTest { + assertFalse(call().setCaptureAudioSource(MediaRecorder.AudioSource.MIC)) + } + + @Test + fun `setHardwareNoiseSuppressorEnabled reaches the media component`() = runTest { + val call = call() + val factory = call.withFactory() + every { factory.setHardwareNoiseSuppressorEnabled(false) } returns true + + assertTrue(call.setHardwareNoiseSuppressorEnabled(false)) + verify { factory.setHardwareNoiseSuppressorEnabled(false) } + } + + @Test + fun `setHardwareAcousticEchoCancelerEnabled reaches the media component`() = runTest { + val call = call() + val factory = call.withFactory() + every { factory.setHardwareAcousticEchoCancelerEnabled(false) } returns true + + assertTrue(call.setHardwareAcousticEchoCancelerEnabled(false)) + verify { factory.setHardwareAcousticEchoCancelerEnabled(false) } + } + + @Test + fun `isAudioProcessingReachable follows whether a processor is attached`() = runTest { + val call = call() + val factory = call.withFactory() + every { factory.hasAudioProcessingAttached() } returns true + + assertTrue(call.isAudioProcessingReachable()) + } + + @Test + fun `isAudioProcessingReachable is false with no factory built`() = runTest { + // An absent processor is not a stage that failed — nothing is processing, so a profile + // asking for no processing is already satisfied. + assertFalse(call().isAudioProcessingReachable()) + } + + @Test + fun `isHardwareNoiseSuppressorSupported follows the device capability`() = runTest { + val call = call() + val factory = call.withFactory() + every { factory.isHardwareNoiseSuppressorSupported() } returns true + + assertTrue(call.isHardwareNoiseSuppressorSupported()) + } + + @Test + fun `isHardwareAcousticEchoCancelerSupported follows the device capability`() = runTest { + val call = call() + val factory = call.withFactory() + every { factory.isHardwareAcousticEchoCancelerSupported() } returns true + + assertTrue(call.isHardwareAcousticEchoCancelerSupported()) + } + + //endregion +} diff --git a/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/CallStatsTest.kt b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/CallStatsTest.kt new file mode 100644 index 00000000000..5d07b100f5c --- /dev/null +++ b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/CallStatsTest.kt @@ -0,0 +1,471 @@ +/* + * Copyright (c) 2014-2026 Stream.io Inc. All rights reserved. + * + * Licensed under the Stream License; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://github.com/GetStream/stream-video-android/blob/main/LICENSE + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.getstream.video.android.core + +import io.getstream.video.android.core.call.stats.model.RtcStatsReport +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.test.StandardTestDispatcher +import kotlinx.coroutines.test.TestScope +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test +import org.webrtc.RTCStats +import org.webrtc.RTCStatsReport + +/** + * Covers the audio side of [CallStats] — the send/receive rate derived from the RTP byte + * counters, the target bitrate, and the codec. + * + * The rate is the part worth guarding: it is a delta between two polls, so every way the counter + * can misbehave (first sample, a stream replaced underneath it, a clock that did not move) has to + * leave the reported value alone rather than publish a nonsense number. The codec is guarded + * because one report describes one direction, and the publisher and the subscriber are both fed + * through the same method. + */ +class CallStatsTest { + + private val testScope = TestScope(StandardTestDispatcher()) + + private fun callStats(): CallStats { + val call = mockk(relaxed = true) + // updateFromRTCStats reaches through the session for the video track mapping; no session + // means the audio path runs on its own. + every { call.session } returns MutableStateFlow(null) + return CallStats(call, testScope) + } + + private fun stat( + type: String, + id: String = type, + timestampUs: Long = 0, + members: Map, + ) = RTCStats(timestampUs, type, id, members) + + private fun report(vararg stats: RTCStats): RtcStatsReport { + val origin = RTCStatsReport(0, stats.associateBy { it.id }) + return RtcStatsReport(origin, emptyMap()) + } + + private fun outboundAudio( + bytesSent: Long, + timestampUs: Long, + targetBitrate: Double? = null, + codecId: String? = null, + ) = stat( + type = "outbound-rtp", + id = "outbound-audio", + timestampUs = timestampUs, + members = buildMap { + put("kind", "audio") + put("bytesSent", bytesSent) + targetBitrate?.let { put("targetBitrate", it) } + codecId?.let { put("codecId", it) } + }, + ) + + private fun inboundAudio( + bytesReceived: Long, + timestampUs: Long, + codecId: String? = null, + ) = stat( + type = "inbound-rtp", + id = "inbound-audio", + timestampUs = timestampUs, + members = buildMap { + put("kind", "audio") + put("bytesReceived", bytesReceived) + codecId?.let { put("codecId", it) } + }, + ) + + private fun codec( + id: String, + mimeType: String = "audio/opus", + clockRate: Long = 48000, + channels: Long = 1, + fmtp: String = "minptime=10;useinbandfec=1", + ) = stat( + type = "codec", + id = id, + members = mapOf( + "mimeType" to mimeType, + "clockRate" to clockRate, + "channels" to channels, + "sdpFmtpLine" to fmtp, + ), + ) + + //region audio bitrate derivation + + @Test + fun `a missing byte counter leaves the baseline untouched`() { + val stats = callStats() + + stats.publisher.updateAudioBitrate(bytes = null, timestampUs = 1_000_000.0) + + assertEquals(null, stats.publisher.lastAudioBytes) + assertEquals(0F, stats.publisher.audioBitrateKbps.value) + } + + @Test + fun `the first sample only seeds the baseline`() { + val stats = callStats() + + stats.publisher.updateAudioBitrate(bytes = 8_000, timestampUs = 1_000_000.0) + + // A rate needs two points; reporting one would divide by a baseline that does not exist. + assertEquals(8_000L, stats.publisher.lastAudioBytes) + assertEquals(0F, stats.publisher.audioBitrateKbps.value) + } + + @Test + fun `the second sample derives the rate from the delta`() { + val stats = callStats() + + stats.publisher.updateAudioBitrate(bytes = 0, timestampUs = 0.0) + stats.publisher.updateAudioBitrate(bytes = 8_000, timestampUs = 1_000_000.0) + + // 8000 bytes over one second is 64 kbps, the default voice bitrate. + assertEquals(64F, stats.publisher.audioBitrateKbps.value) + } + + @Test + fun `a counter that went backwards reseeds instead of reporting a negative rate`() { + val stats = callStats() + + stats.publisher.updateAudioBitrate(bytes = 0, timestampUs = 0.0) + stats.publisher.updateAudioBitrate(bytes = 16_000, timestampUs = 1_000_000.0) + // The track was replaced, so the counter restarted. + stats.publisher.updateAudioBitrate(bytes = 100, timestampUs = 2_000_000.0) + + // The last good reading stands rather than a negative one, and the new counter becomes + // the baseline for the next poll. + assertEquals(128F, stats.publisher.audioBitrateKbps.value) + assertEquals(100L, stats.publisher.lastAudioBytes) + } + + @Test + fun `a clock that did not move is ignored`() { + val stats = callStats() + + stats.publisher.updateAudioBitrate(bytes = 0, timestampUs = 1_000_000.0) + stats.publisher.updateAudioBitrate(bytes = 8_000, timestampUs = 1_000_000.0) + + assertEquals(0F, stats.publisher.audioBitrateKbps.value) + } + + //endregion + + //region report plumbing + + @Test + fun `a publisher report fills the audio send rate and the target bitrate`() { + val stats = callStats() + + stats.updateFromRTCStats( + report(outboundAudio(bytesSent = 0, timestampUs = 0, targetBitrate = 128_000.0)), + isPublisher = true, + ) + stats.updateFromRTCStats( + report( + outboundAudio( + bytesSent = 16_000, + timestampUs = 1_000_000, + targetBitrate = 128_000.0, + ), + ), + isPublisher = true, + ) + + assertEquals(128F, stats.publisher.audioBitrateKbps.value) + // What the encoder was asked for, against the 128 kbps it actually sent. + assertEquals(128F, stats.publisher.audioTargetBitrateKbps.value) + } + + @Test + fun `a subscriber report fills the audio receive rate`() { + val stats = callStats() + + stats.updateFromRTCStats( + report(inboundAudio(bytesReceived = 0, timestampUs = 0)), + isPublisher = false, + ) + stats.updateFromRTCStats( + report(inboundAudio(bytesReceived = 8_000, timestampUs = 1_000_000)), + isPublisher = false, + ) + + assertEquals(64F, stats.subscriber.audioBitrateKbps.value) + // The receive side has no target of its own to report. + assertEquals(0F, stats.subscriber.audioTargetBitrateKbps.value) + } + + @Test + fun `a null report is ignored`() { + val stats = callStats() + + stats.updateFromRTCStats(null) + + assertEquals(0F, stats.publisher.audioBitrateKbps.value) + } + + @Test + fun `a seeded byte count with no timestamp behind it does not derive a rate`() { + val stats = callStats() + // The two halves of the baseline are separate fields, so a half-written one has to be + // treated as no baseline rather than divided by. + stats.publisher.lastAudioBytes = 8_000 + + stats.publisher.updateAudioBitrate(bytes = 16_000, timestampUs = 1_000_000.0) + + assertEquals(0F, stats.publisher.audioBitrateKbps.value) + } + + @Test + fun `a report carrying no audio at all leaves every audio value alone`() { + val stats = callStats() + + stats.updateFromRTCStats( + report(stat(type = "candidate-pair", members = mapOf("currentRoundTripTime" to 0.05))), + isPublisher = true, + ) + + assertEquals(0F, stats.publisher.audioBitrateKbps.value) + assertEquals(0F, stats.publisher.audioTargetBitrateKbps.value) + assertEquals("", stats.publisher.audioCodec.value) + } + + @Test + fun `byte counters of the wrong type are treated as absent`() { + val stats = callStats() + + // WebRTC reports these as BigInteger on some builds; the cast has to fail into "no + // sample" rather than seed a baseline that later polls subtract from. + stats.updateFromRTCStats( + report( + stat( + type = "outbound-rtp", + id = "outbound-audio", + members = mapOf("kind" to "audio", "bytesSent" to "16000"), + ), + ), + isPublisher = true, + ) + stats.updateFromRTCStats( + report( + stat( + type = "inbound-rtp", + id = "inbound-audio", + members = mapOf("kind" to "audio", "bytesReceived" to 1.5), + ), + ), + isPublisher = false, + ) + + assertNull(stats.publisher.lastAudioBytes) + assertNull(stats.subscriber.lastAudioBytes) + } + + @Test + fun `a target bitrate of the wrong type is skipped`() { + val stats = callStats() + + stats.updateFromRTCStats( + report( + stat( + type = "outbound-rtp", + id = "outbound-audio", + members = mapOf( + "kind" to "audio", + "bytesSent" to 0L, + "targetBitrate" to 128_000, + ), + ), + ), + isPublisher = true, + ) + + assertEquals(0F, stats.publisher.audioTargetBitrateKbps.value) + } + + //endregion + + //region codec direction + + @Test + fun `a publisher report sets the publisher codec only`() { + val stats = callStats() + + stats.updateFromRTCStats( + report( + outboundAudio(bytesSent = 0, timestampUs = 0, codecId = "codec-send"), + codec("codec-send", channels = 2), + ), + isPublisher = true, + ) + + assertEquals( + "audio/opus 48000 Hz stereo minptime=10;useinbandfec=1", + stats.publisher.audioCodec.value, + ) + // Writing both sides from one report is how the subscriber ended up showing the + // publisher's codec, and the other way round on the next poll. + assertEquals("", stats.subscriber.audioCodec.value) + } + + @Test + fun `a subscriber report sets the subscriber codec only`() { + val stats = callStats() + + stats.updateFromRTCStats( + report( + inboundAudio(bytesReceived = 0, timestampUs = 0, codecId = "codec-recv"), + codec("codec-recv"), + ), + isPublisher = false, + ) + + assertEquals( + "audio/opus 48000 Hz mono minptime=10;useinbandfec=1", + stats.subscriber.audioCodec.value, + ) + assertEquals("", stats.publisher.audioCodec.value) + } + + @Test + fun `the codec is resolved through the RTP statistic's own codecId`() { + val stats = callStats() + + stats.updateFromRTCStats( + report( + outboundAudio(bytesSent = 0, timestampUs = 0, codecId = "codec-second"), + // A report can carry more than one audio codec; the first one is not necessarily + // the one this direction is using. + codec("codec-first", mimeType = "audio/PCMU", clockRate = 8000), + codec("codec-second", channels = 2), + ), + isPublisher = true, + ) + + assertEquals( + "audio/opus 48000 Hz stereo minptime=10;useinbandfec=1", + stats.publisher.audioCodec.value, + ) + } + + @Test + fun `a codecId of the wrong type falls back to the codec the report carries`() { + val stats = callStats() + + stats.updateFromRTCStats( + report( + stat( + type = "outbound-rtp", + id = "outbound-audio", + members = mapOf("kind" to "audio", "bytesSent" to 0L, "codecId" to 7L), + ), + codec("codec-only"), + ), + isPublisher = true, + ) + + assertEquals( + "audio/opus 48000 Hz mono minptime=10;useinbandfec=1", + stats.publisher.audioCodec.value, + ) + } + + @Test + fun `a codecId pointing at nothing reports no codec`() { + val stats = callStats() + + // The elvis falls through to codec:audio, and there is none — the alternative is an + // exception on a stats poll. + stats.updateFromRTCStats( + report(outboundAudio(bytesSent = 0, timestampUs = 0, codecId = "missing")), + isPublisher = true, + ) + + assertEquals("", stats.publisher.audioCodec.value) + } + + @Test + fun `codec members of the wrong type drop out of the description`() { + val stats = callStats() + + stats.updateFromRTCStats( + report( + outboundAudio(bytesSent = 0, timestampUs = 0, codecId = "odd-codec"), + stat( + type = "codec", + id = "odd-codec", + members = mapOf( + "mimeType" to 1, + "clockRate" to "48000", + "channels" to "1", + "sdpFmtpLine" to 2, + ), + ), + ), + isPublisher = true, + ) + + // Every member failed its cast, so the description is empty rather than "null null null". + assertEquals("", stats.publisher.audioCodec.value) + } + + @Test + fun `a codec with no channel count reports without a channel word`() { + val stats = callStats() + + stats.updateFromRTCStats( + report( + outboundAudio(bytesSent = 0, timestampUs = 0, codecId = "no-channels"), + stat( + type = "codec", + id = "no-channels", + members = mapOf("mimeType" to "audio/opus", "clockRate" to 48000L), + ), + ), + isPublisher = true, + ) + + assertEquals("audio/opus 48000 Hz", stats.publisher.audioCodec.value) + } + + @Test + fun `a report with no codecId still reports the codec it carries`() { + val stats = callStats() + + stats.updateFromRTCStats( + report( + outboundAudio(bytesSent = 0, timestampUs = 0), + codec("codec-only"), + ), + isPublisher = true, + ) + + assertEquals( + "audio/opus 48000 Hz mono minptime=10;useinbandfec=1", + stats.publisher.audioCodec.value, + ) + } + + //endregion +} diff --git a/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/MediaManagerAudioPipelineTest.kt b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/MediaManagerAudioPipelineTest.kt new file mode 100644 index 00000000000..2a3dbaaba33 --- /dev/null +++ b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/MediaManagerAudioPipelineTest.kt @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2014-2026 Stream.io Inc. All rights reserved. + * + * Licensed under the Stream License; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://github.com/GetStream/stream-video-android/blob/main/LICENSE + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.getstream.video.android.core + +import android.content.Context +import io.getstream.video.android.core.call.connection.StreamPeerConnectionFactory +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.test.StandardTestDispatcher +import kotlinx.coroutines.test.TestScope +import org.junit.Assert.assertFalse +import org.junit.Assert.assertSame +import org.junit.Assert.assertTrue +import org.junit.Test +import org.webrtc.AudioSource +import org.webrtc.AudioTrack +import org.webrtc.EglBase + +/** + * Covers [MediaManagerImpl.replaceAudioSourceAndTrack] — the swap that lets audio-source + * constraints change mid-call. Disposal and rollback are the whole risk here: the audio track is + * handed to a sender that must not own it, so the media manager is the only thing allowed to + * dispose it, exactly once, and only after a swap that actually happened. + */ +class MediaManagerAudioPipelineTest { + + private val testScope = TestScope(StandardTestDispatcher()) + + private val factory = mockk(relaxed = true) + private val call = mockk(relaxed = true) + + private fun mediaManager(): MediaManagerImpl { + every { call.peerConnectionFactory } returns factory + return MediaManagerImpl( + context = mockk(relaxed = true), + call = call, + scope = testScope, + eglBaseContext = mockk(relaxed = true), + ) + } + + private fun stubSourcesAndTracks( + sources: List, + tracks: List, + ) { + every { factory.makeAudioSource(any()) } returnsMany sources + every { factory.makeAudioTrack(any(), any()) } returnsMany tracks + } + + @Test + fun `a successful swap installs the new pair and disposes the replaced one`() { + val firstSource = mockk(relaxed = true) + val secondSource = mockk(relaxed = true) + val firstTrack = mockk(relaxed = true) + val secondTrack = mockk(relaxed = true) + stubSourcesAndTracks( + listOf(firstSource, secondSource), + listOf(firstTrack, secondTrack), + ) + val manager = mediaManager() + + // Force the first pair into existence, the way publishing does. + assertSame(firstTrack, manager.audioTrack) + + var handed: AudioTrack? = null + assertTrue( + manager.replaceAudioSourceAndTrack { + handed = it + true + }, + ) + + assertSame(secondTrack, handed) + assertSame(secondTrack, manager.audioTrack) + assertSame(secondSource, manager.audioSource) + verify { firstTrack.dispose() } + verify { firstSource.dispose() } + // The new pair is live, so disposing it here would hand the sender a dead track. + verify(exactly = 0) { secondTrack.dispose() } + verify(exactly = 0) { secondSource.dispose() } + } + + @Test + fun `a refused swap rolls the new pair back and leaves the live one alone`() { + val firstSource = mockk(relaxed = true) + val secondSource = mockk(relaxed = true) + val firstTrack = mockk(relaxed = true) + val secondTrack = mockk(relaxed = true) + stubSourcesAndTracks( + listOf(firstSource, secondSource), + listOf(firstTrack, secondTrack), + ) + val manager = mediaManager() + assertSame(firstTrack, manager.audioTrack) + + assertFalse(manager.replaceAudioSourceAndTrack { false }) + + // Nothing was replaced, so tearing the live pair down would silence the call. + assertSame(firstTrack, manager.audioTrack) + assertSame(firstSource, manager.audioSource) + verify(exactly = 0) { firstTrack.dispose() } + verify(exactly = 0) { firstSource.dispose() } + verify { secondTrack.dispose() } + verify { secondSource.dispose() } + } + + @Test + fun `the swap carries the microphone mute state onto the new track`() { + val track = mockk(relaxed = true) + stubSourcesAndTracks( + listOf(mockk(relaxed = true)), + listOf(track), + ) + val manager = mediaManager() + + manager.replaceAudioSourceAndTrack { true } + + // The microphone was never enabled, so a fresh track — which starts enabled — has to be + // muted or the swap would unmute the user. + verify { track.setEnabled(false) } + } + + @Test + fun `the swap carries the live track state rather than the microphone flow`() { + val firstTrack = mockk(relaxed = true) + val secondTrack = mockk(relaxed = true) + stubSourcesAndTracks( + listOf(mockk(relaxed = true), mockk(relaxed = true)), + listOf(firstTrack, secondTrack), + ) + val manager = mediaManager() + assertSame(firstTrack, manager.audioTrack) + // MicrophoneManager.enable(fromUser = false) — what a lifecycle-driven resume calls — + // turns the track on without moving _status, so the flow still reads disabled. + every { firstTrack.enabled() } returns true + + manager.replaceAudioSourceAndTrack { true } + + // Reading the flow here would mute a call that is publishing audio. + verify { secondTrack.setEnabled(true) } + verify(exactly = 0) { secondTrack.setEnabled(false) } + } + + @Test + fun `no swap happens after the media manager has been released`() { + stubSourcesAndTracks(listOf(mockk(relaxed = true)), listOf(mockk(relaxed = true))) + val manager = mediaManager() + manager.cleanup() + + var swapCalled = false + assertFalse( + manager.replaceAudioSourceAndTrack { + swapCalled = true + true + }, + ) + + // Rebuilding after teardown would resurrect native objects nothing disposes. + assertFalse(swapCalled) + } +} diff --git a/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/MicrophoneManagerTest.kt b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/MicrophoneManagerTest.kt index 72cd7fd412c..b5cd2af6fc2 100644 --- a/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/MicrophoneManagerTest.kt +++ b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/MicrophoneManagerTest.kt @@ -20,6 +20,7 @@ import android.content.Context import android.media.AudioAttributes import android.media.AudioDeviceInfo import android.media.AudioManager +import android.media.MediaRecorder import android.net.Uri import android.os.Build import android.os.ParcelUuid @@ -379,17 +380,15 @@ class MicrophoneManagerTest { } @Test - fun `setAudioBitrateProfile should fail if call already joined`() = runTest { + fun `setAudioBitrateProfile should fail if hifi disabled`() = runTest { val mediaManager = mockMediaManager( - connection = MutableStateFlow( - RealtimeConnection.Joined(mockk(relaxed = true)), - ), - settings = MutableStateFlow(mockCallSettings(hifiAudioEnabled = true)), + connection = MutableStateFlow(RealtimeConnection.PreJoin), + settings = MutableStateFlow(mockCallSettings(hifiAudioEnabled = false)), ) val microphoneManager = MicrophoneManager(mediaManager, audioUsage, audioUsageProvider) val result = microphoneManager.setAudioBitrateProfile( - AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_HIGH_QUALITY, + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, ) assertTrue(result.isFailure) @@ -400,24 +399,578 @@ class MicrophoneManagerTest { } @Test - fun `setAudioBitrateProfile should fail if hifi disabled`() = runTest { + fun `setAudioBitrateProfile should fail if hifi disabled after joining too`() = runTest { + val call = mockAudioStagesCall() val mediaManager = mockMediaManager( - connection = MutableStateFlow(RealtimeConnection.PreJoin), + connection = MutableStateFlow(joined()), settings = MutableStateFlow(mockCallSettings(hifiAudioEnabled = false)), + call = call, ) val microphoneManager = MicrophoneManager(mediaManager, audioUsage, audioUsageProvider) + // The profiles mean the same thing whenever they are set, so they answer to the same + // dashboard setting; joining is not a way around it. val result = microphoneManager.setAudioBitrateProfile( AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, ) assertTrue(result.isFailure) + verify(exactly = 0) { call.setAudioProcessingEnabled(any()) } + verify(exactly = 0) { call.rebuildAudioCapturePipeline() } + verify(exactly = 0) { call.setAudioMaxBitrate(any()) } + verify(exactly = 0) { call.setCaptureAudioSource(any()) } + verify(exactly = 0) { call.setHardwareAcousticEchoCancelerEnabled(any()) } + } + + @Test + fun `setAudioBitrateProfile before joining moves no stage`() = runTest { + val call = mockAudioStagesCall() + val mediaManager = mockMediaManager( + connection = MutableStateFlow(RealtimeConnection.PreJoin), + settings = MutableStateFlow(mockCallSettings(hifiAudioEnabled = true)), + call = call, + ) + val microphoneManager = MicrophoneManager(mediaManager, audioUsage, audioUsageProvider) + + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ) + + // Nothing is capturing or publishing, so there is nothing to move: the pipeline is built + // from the profile at join and the SFU picks the bitrate. + assertTrue(result.isSuccess) + assertEquals( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + microphoneManager.audioBitrateProfile.value, + ) + verify(exactly = 0) { call.rebuildAudioCapturePipeline() } + verify(exactly = 0) { call.setAudioMaxBitrate(any()) } + // Samsung binds the VoIP capture chain at AudioRecord open from the audio mode. + // The request has to land before join so setup applies MODE_NORMAL before activate(). + assertFalse(microphoneManager.communicationAudioModeEnabled.value) + } + + @Test + @Config(sdk = [Build.VERSION_CODES.Q]) + fun `leaving music puts the noise-cancellation processor back on when it was on`() = runTest { + val call = mockAudioStagesCall(audioProcessingInitiallyEnabled = true) + val microphoneManager = joinedMicrophoneManager(call) + + microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ).getOrThrow() + microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_STANDARD_UNSPECIFIED, + ).getOrThrow() + + // A creator who finishes their song expects their noise cancellation back. + assertTrue(call.isAudioProcessingEnabled()) + } + + @Test + fun `leaving music leaves the noise-cancellation processor off when it was off`() = runTest { + val call = mockAudioStagesCall(audioProcessingInitiallyEnabled = false) + val microphoneManager = joinedMicrophoneManager(call) + + microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ).getOrThrow() + microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_STANDARD_UNSPECIFIED, + ).getOrThrow() + + // Deriving "voice means on" would switch the processor on for someone who had turned it + // off through the already-released Call.setAudioProcessingEnabled. + assertFalse(call.isAudioProcessingEnabled()) + verify(exactly = 0) { call.setAudioProcessingEnabled(true) } + } + + @Test + fun `a repeated music request does not forget what preceded the first one`() = runTest { + val call = mockAudioStagesCall(audioProcessingInitiallyEnabled = true) + val microphoneManager = joinedMicrophoneManager(call) + + repeat(2) { + microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ).getOrThrow() + } + microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_STANDARD_UNSPECIFIED, + ).getOrThrow() + + // The second music request must not remember the false the first one wrote. + assertTrue(call.isAudioProcessingEnabled()) + } + + @Test + fun `a voice profile on a call that never played music leaves the processor alone`() = runTest { + val call = mockAudioStagesCall(audioProcessingInitiallyEnabled = false) + val microphoneManager = joinedMicrophoneManager(call) + + microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_STANDARD_UNSPECIFIED, + ).getOrThrow() + + // Nothing was remembered, so there is nothing to put back and no business writing at all. + verify(exactly = 0) { call.setAudioProcessingEnabled(any()) } + } + + @Test + fun `a partly applied switch does not switch the processor on`() = runTest { + val call = mockAudioStagesCall( + audioProcessingInitiallyEnabled = false, + audioMaxBitrateAccepted = false, + ) + val microphoneManager = joinedMicrophoneManager(call) + + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ) + + // The bitrate stage refused, so the state reverts — to what was there, not to "voice + // means on". + assertTrue(result.isFailure) + assertFalse(call.isAudioProcessingEnabled()) + verify(exactly = 0) { call.setAudioProcessingEnabled(true) } + } + + @Test + fun `setAudioBitrateProfile moves every reachable stage after joining`() = runTest { + val call = mockAudioStagesCall(negotiatedAudioBitrate = 64_000) + val microphoneManager = joinedMicrophoneManager(call) + + // The whole point of the change: before, this returned a failure once joined. + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ) + + assertTrue(result.isSuccess) + assertEquals( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + microphoneManager.audioBitrateProfile.value, + ) + // The noise-cancellation processor is the stage no other control on the class reaches. + verify { call.setAudioProcessingEnabled(false) } + verify { call.setHardwareNoiseSuppressorEnabled(false) } + verify { call.setHardwareAcousticEchoCancelerEnabled(false) } + verify { call.setCaptureAudioSource(MediaRecorder.AudioSource.MIC) } + verify { call.rebuildAudioCapturePipeline() } + verify { call.setAudioMaxBitrate(128_000) } + assertFalse(microphoneManager.communicationAudioModeEnabled.value) + } + + @Test + fun `setAudioBitrateProfile restores the negotiated bitrate when leaving music`() = runTest { + val call = mockAudioStagesCall(negotiatedAudioBitrate = 96_000) + val microphoneManager = joinedMicrophoneManager(call) + + microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ) + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_STANDARD_UNSPECIFIED, + ) + + // What the SFU asked for at join, not a guess at what a voice profile is worth. + assertTrue(result.isSuccess) + verify { call.setAudioMaxBitrate(96_000) } + verify { call.setCaptureAudioSource(MediaRecorder.AudioSource.VOICE_COMMUNICATION) } + assertTrue(microphoneManager.communicationAudioModeEnabled.value) + } + + @Test + fun `setAudioBitrateProfile falls back to the voice bitrate with no negotiated value`() = + runTest { + val call = mockAudioStagesCall(negotiatedAudioBitrate = null) + val microphoneManager = joinedMicrophoneManager(call) + + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_HIGH_QUALITY, + ) + + assertTrue(result.isSuccess) + verify { call.setAudioMaxBitrate(64_000) } + } + + @Test + fun `setAudioBitrateProfile does not rebuild the pipeline when nothing changes`() = runTest { + val call = mockAudioStagesCall() + val microphoneManager = joinedMicrophoneManager(call) + + // Already the default profile's value — rebuilding would drop audio for no reason. + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_HIGH_QUALITY, + ) + + assertTrue(result.isSuccess) + verify(exactly = 0) { call.rebuildAudioCapturePipeline() } + } + + @Test + fun `setAudioBitrateProfile counts an absent noise cancellation processor as applied`() = + runTest { + val call = mockAudioStagesCall(audioProcessingReachable = false) + // Nothing is attached, so nothing is processing: the profile is satisfied. Reporting a + // failed stage would send every customer without a processor chasing a non-problem. + every { call.isAudioProcessingEnabled() } returns false + val microphoneManager = joinedMicrophoneManager(call) + + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_STANDARD_UNSPECIFIED, + ) + + assertTrue(result.isSuccess) + } + + @Test + fun `setAudioBitrateProfile reports a noise cancellation processor that refused`() = runTest { + // The refusal only matters where the stage has work to do, which is on the way out of + // music: the processor was on before it, so leaving music has to put it back. + val call = mockAudioStagesCall( + audioProcessingReachable = true, + audioProcessingInitiallyEnabled = true, + ) + val microphoneManager = joinedMicrophoneManager(call) + microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ).getOrThrow() + // Attached but no longer allowed on this call — setAudioProcessingEnabled(true) is + // refused, so the processor cannot be put back. + every { call.isAudioProcessingEnabled() } returns false + + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_STANDARD_UNSPECIFIED, + ) + + assertTrue(result.isFailure) + assertTrue( + result.exceptionOrNull()?.message.orEmpty().contains("noise cancellation"), + ) + } + + @Test + @Config(sdk = [Build.VERSION_CODES.Q]) + fun `setAudioBitrateProfile names each stage that did not move`() = runTest { + val call = mockAudioStagesCall() + every { call.setHardwareNoiseSuppressorEnabled(any()) } returns false + every { call.rebuildAudioCapturePipeline() } returns false + every { call.setAudioMaxBitrate(any()) } returns false + val microphoneManager = joinedMicrophoneManager(call) + + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ) + + assertTrue(result.isFailure) + val message = result.exceptionOrNull()?.message.orEmpty() + assertTrue(message.contains("hardware noise suppressor")) + assertTrue(message.contains("software audio processing")) + assertTrue(message.contains("max bitrate")) + // The stages that did move are not named as refused. + assertFalse(message.contains("noise cancellation,")) + assertFalse(message.contains("capture audio source")) + // The profile did not take, so the flow a toggle binds to snaps back. assertEquals( AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_STANDARD_UNSPECIFIED, microphoneManager.audioBitrateProfile.value, ) } + @Test + fun `communication audio mode should be on by default`() { + val microphoneManager = + MicrophoneManager(mockMediaManager(), audioUsage, audioUsageProvider) + + assertTrue(microphoneManager.communicationAudioModeEnabled.value) + } + + @Test + fun `setCommunicationAudioModeEnabled should report the requested state when refused`() { + val microphoneManager = + MicrophoneManager(mockMediaManager(), audioUsage, audioUsageProvider) + + // No audio handler: this call does not manage routing, so the mode is left alone. + assertFalse(microphoneManager.setCommunicationAudioModeEnabled(false)) + // Still reported, so the next handler to start picks the choice up. + assertFalse(microphoneManager.communicationAudioModeEnabled.value) + } + + @Test + @Config(sdk = [Build.VERSION_CODES.Q]) + fun `a refused switch puts the profile-derived state back`() = runTest { + // The processor was running before the switch, so putting the state back means putting it + // back on. The case where it was off is covered by + // `a partly applied switch does not switch the processor on`. + val call = mockAudioStagesCall(audioProcessingInitiallyEnabled = true) + every { call.setAudioMaxBitrate(any()) } returns false + val microphoneManager = joinedMicrophoneManager(call) + + microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ) + + // The suppressor is asked for music (off), then put back to the voice value — otherwise + // the value it remembers resurrects music when capture restarts, on a call whose profile + // flow says voice. + verifyOrder { + call.setHardwareNoiseSuppressorEnabled(false) + call.setHardwareNoiseSuppressorEnabled(true) + } + verifyOrder { + call.setHardwareAcousticEchoCancelerEnabled(false) + call.setHardwareAcousticEchoCancelerEnabled(true) + } + // And the noise-cancellation processor goes back on with it. + verifyOrder { + call.setAudioProcessingEnabled(false) + call.setAudioProcessingEnabled(true) + } + // Mode and source follow the published profile, not the attempt that did not complete. + // They have to revert together: MODE_NORMAL against VOICE_COMMUNICATION is silent. + verifyOrder { + call.setCaptureAudioSource(MediaRecorder.AudioSource.MIC) + call.setCaptureAudioSource(MediaRecorder.AudioSource.VOICE_COMMUNICATION) + } + assertTrue(microphoneManager.communicationAudioModeEnabled.value) + } + + @Test + @Config(sdk = [Build.VERSION_CODES.Q]) + fun `a refused switch does not swap the live track at all`() = runTest { + val call = mockAudioStagesCall() + every { call.setAudioMaxBitrate(any()) } returns false + val microphoneManager = joinedMicrophoneManager(call) + + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ) + + // The goog* constraints have no setter, so a swap in either direction is an + // RtpSender.setTrack on a live connection: a gap in captured audio and the publisher's + // disposed-track path. The rebuild runs after every stage that can refuse and is skipped + // when one did, so a failed switch costs no swap going in and needs none coming back. + assertTrue(result.isFailure) + verify(exactly = 0) { call.rebuildAudioCapturePipeline() } + } + + @Test + @Config(sdk = [Build.VERSION_CODES.Q]) + fun `a missing audio sender is not a refused stage`() = runTest { + val call = mockAudioStagesCall() + // Joined muted: the live setters have nowhere to go, exactly as they return before join. + every { call.setHardwareNoiseSuppressorEnabled(any()) } returns false + every { call.setHardwareAcousticEchoCancelerEnabled(any()) } returns false + every { call.rebuildAudioCapturePipeline() } returns false + every { call.setAudioMaxBitrate(any()) } returns false + every { call.hasLiveAudioSender() } returns false + val microphoneManager = joinedMicrophoneManager(call) + + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ) + + // Nothing was ever published, so there is no session to move. The requests are remembered + // and applied when the transceiver is created — same as before joining. + assertTrue(result.isSuccess) + assertEquals( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + microphoneManager.audioBitrateProfile.value, + ) + } + + @Test + fun `the switch uses the bitrate the SFU offers for the profile`() = runTest { + val call = mockAudioStagesCall(negotiatedAudioBitrate = 64_000) + // The server names a bitrate per profile; there is no need to guess at one. + every { + call.audioBitrateFor(AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY) + } returns 160_000 + val microphoneManager = joinedMicrophoneManager(call) + + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ) + + assertTrue(result.isSuccess) + verify { call.setAudioMaxBitrate(160_000) } + } + + @Test + @Config(sdk = [Build.VERSION_CODES.Q]) + fun `the profile flow does not move when a stage refused`() = runTest { + val call = mockAudioStagesCall() + every { call.setAudioMaxBitrate(any()) } returns false + val microphoneManager = joinedMicrophoneManager(call) + + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ) + + assertTrue(result.isFailure) + // A toggle bound to this flow snaps back, rather than claiming a switch the audio did not + // make — reporting MUSIC here is exactly the "looks like it worked" failure to avoid. + assertEquals( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_STANDARD_UNSPECIFIED, + microphoneManager.audioBitrateProfile.value, + ) + } + + @Test + @Config(sdk = [Build.VERSION_CODES.Q]) + fun `the profile flow moves when every reachable stage took the change`() = runTest { + val call = mockAudioStagesCall() + val microphoneManager = joinedMicrophoneManager(call) + + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ) + + assertTrue(result.isSuccess) + assertEquals( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + microphoneManager.audioBitrateProfile.value, + ) + } + + @Test + @Config(sdk = [Build.VERSION_CODES.Q]) + fun `setAudioBitrateProfile reports a capture source that refused`() = runTest { + val call = mockAudioStagesCall() + every { call.setCaptureAudioSource(any()) } returns false + val microphoneManager = joinedMicrophoneManager(call) + + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ) + + assertTrue(result.isFailure) + assertTrue( + result.exceptionOrNull()?.message.orEmpty().contains("capture audio source"), + ) + assertEquals( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_STANDARD_UNSPECIFIED, + microphoneManager.audioBitrateProfile.value, + ) + } + + @Test + @Config(sdk = [Build.VERSION_CODES.Q]) + fun `a device with no platform noise suppressor is not a refused stage`() = runTest { + val call = mockAudioStagesCall(hardwareNoiseSuppressorSupported = false) + // The setter returns false on such a device, exactly as it does when a suppressor refuses. + every { call.setHardwareNoiseSuppressorEnabled(any()) } returns false + val microphoneManager = joinedMicrophoneManager(call) + + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ) + + // Nothing is suppressing, so the profile is satisfied and the switch is not held back. + assertTrue(result.isSuccess) + assertEquals( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + microphoneManager.audioBitrateProfile.value, + ) + } + + @Test + @Config(sdk = [Build.VERSION_CODES.Q]) + fun `a device with no platform acoustic echo canceller is not a refused stage`() = runTest { + val call = mockAudioStagesCall(hardwareAcousticEchoCancelerSupported = false) + every { call.setHardwareAcousticEchoCancelerEnabled(any()) } returns false + val microphoneManager = joinedMicrophoneManager(call) + + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ) + + assertTrue(result.isSuccess) + } + + @Test + @Config(sdk = [Build.VERSION_CODES.Q]) + fun `setAudioBitrateProfile reports an acoustic echo canceller that refused`() = runTest { + val call = mockAudioStagesCall() + every { call.setHardwareAcousticEchoCancelerEnabled(any()) } returns false + val microphoneManager = joinedMicrophoneManager(call) + + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ) + + assertTrue(result.isFailure) + assertTrue( + result.exceptionOrNull()?.message.orEmpty().contains("hardware echo canceller"), + ) + } + + private fun joined() = RealtimeConnection.Joined(mockk(relaxed = true)) + + /** A joined call with HiFi enabled — where a mid-call profile switch actually does something. */ + private fun joinedMicrophoneManager(call: Call): MicrophoneManager = MicrophoneManager( + mockMediaManager( + connection = MutableStateFlow(joined()), + settings = MutableStateFlow(mockCallSettings(hifiAudioEnabled = true)), + call = call, + ), + audioUsage, + audioUsageProvider, + ) + + /** A [Call] with every audio stage present and accepting changes. */ + @Test + @Config(sdk = [Build.VERSION_CODES.Q]) + fun `a refused rebuild puts the bitrate back`() = runTest { + // The bitrate is applied before the rebuild, so it is the stage that can already have + // moved when the rebuild refuses. It has a live setter, so the revert undoes it. + val call = mockAudioStagesCall(negotiatedAudioBitrate = 96_000) + every { call.rebuildAudioCapturePipeline() } returns false + val microphoneManager = joinedMicrophoneManager(call) + + val result = microphoneManager.setAudioBitrateProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ) + + assertTrue(result.isFailure) + verify { call.setAudioMaxBitrate(128_000) } + verify { call.setAudioMaxBitrate(96_000) } + assertEquals( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_STANDARD_UNSPECIFIED, + microphoneManager.audioBitrateProfile.value, + ) + } + + private fun mockAudioStagesCall( + audioProcessingReachable: Boolean = true, + negotiatedAudioBitrate: Int? = 64_000, + hardwareNoiseSuppressorSupported: Boolean = true, + hardwareAcousticEchoCancelerSupported: Boolean = true, + audioProcessingInitiallyEnabled: Boolean = false, + audioMaxBitrateAccepted: Boolean = true, + ): Call { + // Mirrors what was asked for, so a stage that accepts the change reports applied. + var audioProcessingEnabled = audioProcessingInitiallyEnabled + return mockk(relaxed = true) { + every { isAudioProcessingReachable() } returns audioProcessingReachable + every { isHardwareNoiseSuppressorSupported() } returns hardwareNoiseSuppressorSupported + every { isHardwareAcousticEchoCancelerSupported() } returns + hardwareAcousticEchoCancelerSupported + every { setAudioProcessingEnabled(any()) } answers { + audioProcessingEnabled = firstArg() + } + every { isAudioProcessingEnabled() } answers { audioProcessingEnabled } + every { setHardwareNoiseSuppressorEnabled(any()) } returns true + every { setHardwareAcousticEchoCancelerEnabled(any()) } returns true + every { setCaptureAudioSource(any()) } returns true + every { rebuildAudioCapturePipeline() } returns true + every { setAudioMaxBitrate(any()) } returns audioMaxBitrateAccepted + every { hasLiveAudioSender() } returns true + every { negotiatedAudioBitrate() } returns negotiatedAudioBitrate + every { audioBitrateFor(any()) } returns null + } + } + private fun microphoneManagerWithImmediateSetup( mediaManager: MediaManagerImpl = mockMediaManager(), audioTrack: AudioTrack = mockk(relaxed = true), diff --git a/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/audio/AudioSwitchControllerTest.kt b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/audio/AudioSwitchControllerTest.kt index 9e197f6f4e3..d6d0aebfa8e 100644 --- a/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/audio/AudioSwitchControllerTest.kt +++ b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/audio/AudioSwitchControllerTest.kt @@ -17,11 +17,13 @@ package io.getstream.video.android.core.audio import android.content.Context +import android.media.AudioManager import com.twilio.audioswitch.AudioDevice import com.twilio.audioswitch.AudioDeviceChangeListener import com.twilio.audioswitch.AudioSwitch import io.mockk.every import io.mockk.mockk +import io.mockk.slot import io.mockk.spyk import io.mockk.unmockkAll import io.mockk.verify @@ -29,12 +31,15 @@ import kotlinx.coroutines.ExperimentalCoroutinesApi import org.junit.After import org.junit.Before import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue @OptIn(ExperimentalCoroutinesApi::class) class AudioSwitchControllerTest { private val context = mockk(relaxed = true) private val listener = mockk(relaxed = true) + private val audioManager = mockk(relaxed = true) private lateinit var controller: AudioSwitchController private lateinit var audioSwitch: AudioSwitch @@ -43,6 +48,9 @@ class AudioSwitchControllerTest { fun setup() { audioSwitch = mockk(relaxed = true) + every { context.getSystemService(Context.AUDIO_SERVICE) } returns audioManager + every { audioManager.mode } returns AudioManager.MODE_IN_COMMUNICATION + controller = spyk( AudioSwitchController(context, emptyList(), listener), ) @@ -55,11 +63,23 @@ class AudioSwitchControllerTest { unmockkAll() } + /** + * The listener handed to AudioSwitch, which wraps [listener] so the requested audio mode + * survives the route changes AudioSwitch drives on its own. + */ + private fun startedListener(): AudioDeviceChangeListener { + val slot = slot() + verify { audioSwitch.start(capture(slot)) } + return slot.captured + } + @Test fun `start should create and start AudioSwitch`() { controller.start() - verify { audioSwitch.start(listener) } + startedListener().invoke(emptyList(), null) + + verify { listener.invoke(emptyList(), null) } } @Test @@ -68,7 +88,7 @@ class AudioSwitchControllerTest { controller.start() verify(exactly = 1) { controller.getAudioSwitch() } - verify(exactly = 1) { audioSwitch.start(listener) } + verify(exactly = 1) { audioSwitch.start(any()) } } @Test @@ -111,4 +131,87 @@ class AudioSwitchControllerTest { verify(exactly = 0) { audioSwitch.selectDevice(any()) } } + + @Test + fun `setCommunicationModeEnabled false should move the device to MODE_NORMAL`() { + controller.start() + + assertTrue(controller.setCommunicationModeEnabled(false)) + + verify { audioManager.mode = AudioManager.MODE_NORMAL } + } + + @Test + fun `setCommunicationModeEnabled true should move the device to MODE_IN_COMMUNICATION`() { + controller.start() + every { audioManager.mode } returns AudioManager.MODE_NORMAL + + assertTrue(controller.setCommunicationModeEnabled(true)) + + verify { audioManager.mode = AudioManager.MODE_IN_COMMUNICATION } + } + + @Test + fun `setCommunicationModeEnabled should report failure with no AudioManager`() { + every { context.getSystemService(Context.AUDIO_SERVICE) } returns null + controller.start() + + assertFalse(controller.setCommunicationModeEnabled(false)) + } + + @Test + fun `selectDevice should reapply the requested mode over the one activate sets`() { + controller.start() + controller.setCommunicationModeEnabled(false) + // activate() puts the device back in communication mode behind our back. + every { audioManager.mode } returns AudioManager.MODE_IN_COMMUNICATION + + controller.selectDevice(mockk()) + + verify(exactly = 2) { audioManager.mode = AudioManager.MODE_NORMAL } + } + + @Test + fun `selectDevice should leave the mode alone when none was requested`() { + controller.start() + + controller.selectDevice(mockk()) + + verify(exactly = 0) { audioManager.mode = any() } + } + + @Test + fun `a route change should reapply the requested mode over the one AudioSwitch sets`() { + controller.start() + controller.setCommunicationModeEnabled(false) + // Taking audio focus to enumerate devices puts the device back in communication mode + // before the change listener runs. + every { audioManager.mode } returns AudioManager.MODE_IN_COMMUNICATION + + startedListener().invoke(emptyList(), null) + + verify(exactly = 2) { audioManager.mode = AudioManager.MODE_NORMAL } + } + + @Test + fun `a route change should leave the mode alone when none was requested`() { + controller.start() + + startedListener().invoke(emptyList(), null) + + verify(exactly = 0) { audioManager.mode = any() } + } + + @Test + fun `stop should drop the request so the next session does not inherit it`() { + controller.start() + controller.setCommunicationModeEnabled(false) + + controller.stop() + controller.start() + controller.selectDevice(mockk()) + + // Once from the explicit request, and not again for the session that followed it. + verify(exactly = 1) { audioManager.mode = AudioManager.MODE_NORMAL } + } } diff --git a/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/call/components/CallMediaManagerTest.kt b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/call/components/CallMediaManagerTest.kt index cf1f95f699e..0f13733b46b 100644 --- a/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/call/components/CallMediaManagerTest.kt +++ b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/call/components/CallMediaManagerTest.kt @@ -271,4 +271,68 @@ class CallMediaManagerTest { verify { processor.isEnabled = true } } + + @Test + fun `setting the hardware noise suppressor applies it to the existing factory`() { + val factory = mockk(relaxed = true) + every { factory.setHardwareNoiseSuppressorEnabled(false) } returns true + val manager = manager() + manager.peerConnectionFactory = factory + + assertThat(manager.setHardwareNoiseSuppressorEnabled(false)).isTrue() + + verify { factory.setHardwareNoiseSuppressorEnabled(false) } + } + + @Test + fun `setting the hardware acoustic echo canceller applies it to the existing factory`() { + val factory = mockk(relaxed = true) + every { factory.setHardwareAcousticEchoCancelerEnabled(false) } returns true + val manager = manager() + manager.peerConnectionFactory = factory + + assertThat(manager.setHardwareAcousticEchoCancelerEnabled(false)).isTrue() + + verify { factory.setHardwareAcousticEchoCancelerEnabled(false) } + } + + @Test + fun `setting the capture audio source applies it to the existing factory`() { + val factory = mockk(relaxed = true) + every { factory.setCaptureAudioSource(any()) } returns true + val manager = manager() + manager.peerConnectionFactory = factory + + assertThat(manager.setCaptureAudioSource(1)).isTrue() + + verify { factory.setCaptureAudioSource(1) } + } + + @Test + fun `setting the capture audio source does not build a factory for a call without one`() { + val manager = manager() + + assertThat(manager.setCaptureAudioSource(1)).isFalse() + } + + @Test + fun `setting the hardware noise suppressor does not build a factory for a call without one`() { + val manager = manager() + + // Recorded for whichever factory is built later. Building one here would capture the + // pre-join audio bitrate profile, which is what ensureFactoryMatchesAudioProfile exists + // to prevent. + assertThat(manager.setHardwareNoiseSuppressorEnabled(false)).isFalse() + + manager.recreatePeerConnectionFactory() + } + + @Test + fun `setting the hardware acoustic echo canceller does not build a factory for a call without one`() { + val manager = manager() + + assertThat(manager.setHardwareAcousticEchoCancelerEnabled(false)).isFalse() + + manager.recreatePeerConnectionFactory() + } } diff --git a/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/call/connection/PublisherTest.kt b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/call/connection/PublisherTest.kt index 04ea80f2954..5d469675d6f 100644 --- a/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/call/connection/PublisherTest.kt +++ b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/call/connection/PublisherTest.kt @@ -59,6 +59,7 @@ import org.webrtc.SessionDescription import org.webrtc.VideoTrack import stream.video.sfu.event.VideoLayerSetting import stream.video.sfu.event.VideoSender +import stream.video.sfu.models.AudioBitrate import stream.video.sfu.models.AudioBitrateProfile import stream.video.sfu.models.Codec import stream.video.sfu.models.DegradationPreference @@ -841,6 +842,196 @@ class PublisherTest { // Ensure no transceiver was added if transceiver exists for publish option coVerify(exactly = 0) { publisher.addTransceiver(any(), any(), any(), videoPublishOption) } } + + @Test + fun `replaceAudioTrack moves the audio sender over without taking ownership`() = runTest { + val mockSender = mockk(relaxed = true) + every { mockSender.setTrack(any(), any()) } returns true + val mockTransceiver = mockk(relaxed = true) { + every { sender } returns mockSender + } + every { + mockTransceiverCache.getByTrackType(TrackType.TRACK_TYPE_AUDIO) + } returns listOf(mockTransceiver) + val newTrack = mockk(relaxed = true) + + assertTrue(publisher.replaceAudioTrack(newTrack)) + + // takeOwnership must be false: MediaManagerImpl owns and disposes the audio track, and a + // sender that also owned it would dispose the replaced track behind our back. + coVerify { mockSender.setTrack(newTrack, false) } + coVerify(exactly = 0) { mockSender.setTrack(any(), true) } + } + + @Test + fun `replaceAudioTrack reports false when no audio is being published`() = runTest { + every { + mockTransceiverCache.getByTrackType(TrackType.TRACK_TYPE_AUDIO) + } returns emptyList() + + assertFalse(publisher.replaceAudioTrack(mockk(relaxed = true))) + } + + /** A sender whose parameters carry one encoding, which is the audio case. */ + private fun audioSenderWith( + parameters: RtpParameters, + accepts: Boolean, + ): RtpSender = mockk(relaxed = true) { + every { this@mockk.parameters } returns parameters + every { setParameters(any()) } returns accepts + } + + /** + * `RtpParameters.encodings` is a public field, so it cannot be stubbed — the real object is + * built through the same reflection helper the publish-quality tests use. + */ + private fun singleEncodingParameters(): RtpParameters = + buildRtpParams(rid = null, active = true, maxBitrate = 64_000) + + private fun publishingAudioThrough(sender: RtpSender) { + val transceiver = mockk(relaxed = true) { + every { this@mockk.sender } returns sender + } + every { + mockTransceiverCache.getByTrackType(TrackType.TRACK_TYPE_AUDIO) + } returns listOf(transceiver) + } + + @Test + fun `setAudioMaxBitrate applies the ceiling to every encoding`() = runTest { + val params = singleEncodingParameters() + val sender = audioSenderWith(params, accepts = true) + publishingAudioThrough(sender) + + assertTrue(publisher.setAudioMaxBitrate(128_000)) + + assertEquals(128_000, params.encodings.single().maxBitrateBps) + coVerify { sender.setParameters(params) } + } + + @Test + fun `setAudioMaxBitrate reports false when WebRTC rejects the parameters`() = runTest { + val sender = audioSenderWith(singleEncodingParameters(), accepts = false) + publishingAudioThrough(sender) + + // Reporting success here would tell AudioProfileResult the bitrate stage applied while the + // encoder is still on the old ceiling — the half-applied switch the result type exists to + // surface. + assertFalse(publisher.setAudioMaxBitrate(128_000)) + } + + @Test + fun `setAudioMaxBitrate reports false when no audio is being published`() = runTest { + every { + mockTransceiverCache.getByTrackType(TrackType.TRACK_TYPE_AUDIO) + } returns emptyList() + + assertFalse(publisher.setAudioMaxBitrate(128_000)) + } + + @Test + fun `hasLiveAudioSender is false when no audio is being published`() = runTest { + every { + mockTransceiverCache.getByTrackType(TrackType.TRACK_TYPE_AUDIO) + } returns emptyList() + + assertFalse(publisher.hasLiveAudioSender()) + } + + @Test + fun `hasLiveAudioSender is true when an audio sender exists`() = runTest { + publishingAudioThrough(audioSenderWith(singleEncodingParameters(), accepts = true)) + + assertTrue(publisher.hasLiveAudioSender()) + } + + @Test + fun `audioMaxBitrate reads the ceiling off the live sender`() = runTest { + publishingAudioThrough( + audioSenderWith( + buildRtpParams(rid = null, active = true, maxBitrate = 128_000), + accepts = true, + ), + ) + + assertEquals(128_000, publisher.audioMaxBitrate()) + } + + @Test + fun `audioMaxBitrate is null when nothing is publishing audio`() = runTest { + every { + mockTransceiverCache.getByTrackType(TrackType.TRACK_TYPE_AUDIO) + } returns emptyList() + + assertNull(publisher.audioMaxBitrate()) + } + + @Test + fun `negotiatedAudioBitrate reports what the SFU asked for at join`() = runTest { + // The value to restore when a mid-call switch to music is undone, rather than a guess. + assertEquals(128_000, publisher.negotiatedAudioBitrate()) + } + + @Test + fun `negotiatedAudioBitrate is null when the publisher carries no audio option`() = runTest { + val videoOnly = buildPublisher(listOf(videoPublishOption)) + + assertNull(videoOnly.negotiatedAudioBitrate()) + } + + @Test + fun `audioBitrateFor returns the bitrate the SFU offers for the profile`() = runTest { + val withProfiles = buildPublisher( + listOf( + audioPublishOption.copy( + audio_bitrate_profiles = listOf( + AudioBitrate( + profile = AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_STANDARD_UNSPECIFIED, + bitrate = 64_000, + ), + AudioBitrate( + profile = AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + bitrate = 128_000, + ), + ), + ), + ), + ) + + assertEquals( + 128_000, + withProfiles.audioBitrateFor( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ), + ) + } + + @Test + fun `audioBitrateFor is null when the server named none for the profile`() = runTest { + // A zero is the proto default for a field the server left out, not an offer of no + // bitrate — treating it as one would ask the encoder for nothing. + val zeroed = buildPublisher( + listOf( + audioPublishOption.copy( + audio_bitrate_profiles = listOf( + AudioBitrate( + profile = AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + bitrate = 0, + ), + ), + ), + ), + ) + + assertNull( + zeroed.audioBitrateFor(AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY), + ) + assertNull( + zeroed.audioBitrateFor( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_STANDARD_UNSPECIFIED, + ), + ) + } //endregion // change publish quality region @@ -1342,6 +1533,26 @@ class PublisherTest { //endregion // region utils + /** A publisher carrying [publishOptions], for the accessors that read them. */ + private fun buildPublisher(publishOptions: List): Publisher = Publisher( + mediaManager = mockMediaManager, + peerConnectionFactory = mockPeerConnectionFactory, + publishOptions = publishOptions, + coroutineScope = testScope, + type = StreamPeerType.PUBLISHER, + mediaConstraints = MediaConstraints(), + onStreamAdded = null, + onNegotiationNeeded = { _, _ -> }, + onIceCandidate = null, + maxBitRate = 1_500_000, + sfuClient = mockSignalServerService, + sessionId = "session-id", + rejoin = { rejoinInvocations++ }, + tracer = mockk(relaxed = true), + fastReconnect = {}, + transceiverCache = mockTransceiverCache, + ) + private fun buildRtpParams( rid: String?, active: Boolean, diff --git a/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/call/connection/StreamPeerConnectionFactoryTest.kt b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/call/connection/StreamPeerConnectionFactoryTest.kt index 0c3b2aecb45..ea586c28d58 100644 --- a/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/call/connection/StreamPeerConnectionFactoryTest.kt +++ b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/call/connection/StreamPeerConnectionFactoryTest.kt @@ -17,6 +17,7 @@ package io.getstream.video.android.core.call.connection import android.content.Context +import android.media.MediaRecorder import io.getstream.video.android.core.MediaManagerImpl import io.getstream.video.android.core.ParticipantState import io.getstream.video.android.core.api.SignalServerService @@ -41,6 +42,7 @@ import org.webrtc.MediaConstraints import org.webrtc.PeerConnection import org.webrtc.PeerConnection.Observer import org.webrtc.PeerConnectionFactory +import org.webrtc.audio.JavaAudioDeviceModule import stream.video.sfu.models.AudioBitrateProfile import stream.video.sfu.models.PublishOption @@ -360,6 +362,146 @@ class StreamPeerConnectionFactoryTest { ) } + // Platform (hardware) noise suppressor — a live-capture control, unlike the builder flag + + private fun voiceFactory() = factoryWithProfile( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_STANDARD_UNSPECIFIED, + null, + ) + + @Test + fun `the noise suppressor cannot be changed before an audio device module exists`() { + // Nothing is capturing, so the platform effect does not exist to be changed yet. + assertFalse(voiceFactory().setHardwareNoiseSuppressorEnabled(false)) + } + + @Test + fun `setHardwareNoiseSuppressorEnabled reports what the audio device module did`() { + val adm = mockk(relaxed = true) + every { adm.setNoiseSuppressorEnabled(false) } returns true + val callFactory = voiceFactory() + setPrivate(callFactory, "adm", adm) + + assertTrue(callFactory.setHardwareNoiseSuppressorEnabled(false)) + verify { adm.setNoiseSuppressorEnabled(false) } + } + + @Test + fun `a device that ignores the noise suppressor is reported as unchanged`() { + val adm = mockk(relaxed = true) + every { adm.setNoiseSuppressorEnabled(any()) } returns false + val callFactory = voiceFactory() + setPrivate(callFactory, "adm", adm) + + assertFalse( + "Not every device has a platform noise suppressor to turn off", + callFactory.setHardwareNoiseSuppressorEnabled(false), + ) + } + + @Test + fun `the requested noise suppressor state is re-applied when capture restarts`() { + val adm = mockk(relaxed = true) + every { adm.setNoiseSuppressorEnabled(any()) } returns true + val callFactory = voiceFactory() + setPrivate(callFactory, "adm", adm) + + callFactory.setHardwareNoiseSuppressorEnabled(false) + // A new recording session drops the platform effect and rebuilds it from the *builder* + // flag, so without re-applying, the request silently reverts on the next reconnect. + callFactory.reapplyHardwareNoiseSuppressor() + + verify(exactly = 2) { adm.setNoiseSuppressorEnabled(false) } + } + + @Test + fun `the acoustic echo canceller cannot be changed before an audio device module exists`() { + assertFalse(voiceFactory().setHardwareAcousticEchoCancelerEnabled(false)) + } + + @Test + fun `setHardwareAcousticEchoCancelerEnabled reports what the audio device module did`() { + val adm = mockk(relaxed = true) + every { adm.setAcousticEchoCancelerEnabled(false) } returns true + val callFactory = voiceFactory() + setPrivate(callFactory, "adm", adm) + + assertTrue(callFactory.setHardwareAcousticEchoCancelerEnabled(false)) + verify { adm.setAcousticEchoCancelerEnabled(false) } + } + + @Test + fun `the requested acoustic echo canceller state is re-applied when capture restarts`() { + val adm = mockk(relaxed = true) + every { adm.setAcousticEchoCancelerEnabled(any()) } returns true + val callFactory = voiceFactory() + setPrivate(callFactory, "adm", adm) + + callFactory.setHardwareAcousticEchoCancelerEnabled(false) + callFactory.reapplyHardwareAcousticEchoCanceler() + + verify(exactly = 2) { adm.setAcousticEchoCancelerEnabled(false) } + } + + @Test + fun `a device that ignores the acoustic echo canceller is reported as unchanged`() { + val adm = mockk(relaxed = true) + every { adm.setAcousticEchoCancelerEnabled(any()) } returns false + val callFactory = voiceFactory() + setPrivate(callFactory, "adm", adm) + + assertFalse(callFactory.setHardwareAcousticEchoCancelerEnabled(false)) + } + + @Test + fun `nothing is re-applied when the acoustic echo canceller was never changed`() { + val adm = mockk(relaxed = true) + val callFactory = voiceFactory() + setPrivate(callFactory, "adm", adm) + + callFactory.reapplyHardwareAcousticEchoCanceler() + + verify(exactly = 0) { adm.setAcousticEchoCancelerEnabled(any()) } + } + + @Test + fun `setCaptureAudioSource reports what the audio device module did`() { + val adm = mockk(relaxed = true) + every { adm.audioSource } returns MediaRecorder.AudioSource.MIC + val callFactory = voiceFactory() + setPrivate(callFactory, "adm", adm) + + assertTrue(callFactory.setCaptureAudioSource(MediaRecorder.AudioSource.MIC)) + verify { adm.setAudioSource(MediaRecorder.AudioSource.MIC) } + } + + @Test + fun `setCaptureAudioSource reports false when the module restores the previous source`() { + val adm = mockk(relaxed = true) + every { adm.audioSource } returns MediaRecorder.AudioSource.VOICE_COMMUNICATION + val callFactory = voiceFactory() + setPrivate(callFactory, "adm", adm) + + assertFalse(callFactory.setCaptureAudioSource(MediaRecorder.AudioSource.MIC)) + } + + @Test + fun `the capture source cannot be changed before an audio device module exists`() { + assertFalse(voiceFactory().setCaptureAudioSource(MediaRecorder.AudioSource.MIC)) + } + + @Test + fun `nothing is re-applied when the noise suppressor was never changed`() { + val adm = mockk(relaxed = true) + val callFactory = voiceFactory() + setPrivate(callFactory, "adm", adm) + + callFactory.reapplyHardwareNoiseSuppressor() + + // No preference was expressed, so the builder default has to stand untouched. + verify(exactly = 0) { adm.setNoiseSuppressorEnabled(any()) } + } + private fun setPrivate(target: StreamPeerConnectionFactory, name: String, value: Any) { StreamPeerConnectionFactory::class.java.getDeclaredField(name).apply { isAccessible = true diff --git a/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/rtc/RtcSessionAudioProfileTest.kt b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/rtc/RtcSessionAudioProfileTest.kt new file mode 100644 index 00000000000..d6790aa0e4e --- /dev/null +++ b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/rtc/RtcSessionAudioProfileTest.kt @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2014-2026 Stream.io Inc. All rights reserved. + * + * Licensed under the Stream License; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://github.com/GetStream/stream-video-android/blob/main/LICENSE + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.getstream.video.android.core.rtc + +import android.os.PowerManager +import androidx.lifecycle.Lifecycle +import io.getstream.video.android.core.Call +import io.getstream.video.android.core.CallState +import io.getstream.video.android.core.MediaManagerImpl +import io.getstream.video.android.core.StreamVideo +import io.getstream.video.android.core.StreamVideoClient +import io.getstream.video.android.core.analytics.call.observer.SfuAnalytics +import io.getstream.video.android.core.call.RtcSession +import io.getstream.video.android.core.call.components.CallSessionManager +import io.getstream.video.android.core.call.connection.Publisher +import io.mockk.MockKAnnotations +import io.mockk.every +import io.mockk.impl.annotations.MockK +import io.mockk.impl.annotations.RelaxedMockK +import io.mockk.mockk +import io.mockk.unmockkAll +import io.mockk.verify +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.test.StandardTestDispatcher +import kotlinx.coroutines.test.TestScope +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.webrtc.AudioTrack +import stream.video.sfu.models.AudioBitrateProfile + +/** + * Covers the audio side of [RtcSession] — the bridges [io.getstream.video.android.core.Call] calls + * when the audio bitrate profile changes on a running call, and the capture-pipeline rebuild + * behind them. + * + * Every bridge has a no-publisher branch that has to answer without throwing: the profile can be + * set on a session that exists but is not publishing audio yet, and a crash there would take the + * whole switch down rather than reporting one stage as unreachable. + * + * Runs under Robolectric even though nothing here needs Android: the other tests that touch + * [RtcSession] are Robolectric tests, and mixing loaders makes JaCoCo discard this class's + * execution data as a bytecode mismatch — the coverage silently disappears in a full-suite run + * while every test still passes. + */ +@RunWith(RobolectricTestRunner::class) +class RtcSessionAudioProfileTest { + + private val testScope = TestScope(StandardTestDispatcher()) + + @MockK + private lateinit var mockPowerManager: PowerManager + + @RelaxedMockK + private lateinit var mockCall: Call + + @RelaxedMockK + private lateinit var mockMediaManager: MediaManagerImpl + + @RelaxedMockK + private lateinit var mockLifecycle: Lifecycle + + @RelaxedMockK + private lateinit var mockVideoClient: StreamVideoClient + + @RelaxedMockK + private lateinit var mockCallState: CallState + + private lateinit var session: RtcSession + + @Before + fun setUp() { + MockKAnnotations.init(this, relaxUnitFun = true) + // RtcSession refuses to construct without an installed SDK singleton. + val streamVideo = mockk(relaxed = true) + StreamVideo.install(streamVideo) + + every { mockCall.state } returns mockCallState + every { mockCall.scope } returns testScope + every { mockCall.mediaManager } returns mockMediaManager + every { mockCall.peerConnectionFactory } returns mockk(relaxed = true) { + every { makePeerConnection(any(), any(), any(), any()) } returns mockk(relaxed = true) + } + every { mockCallState.ownCapabilities } returns MutableStateFlow(emptyList()) + every { mockCallState.participants } returns MutableStateFlow(emptyList()) + every { mockCallState.remoteParticipants } returns MutableStateFlow(emptyList()) + every { mockCallState.me.value } returns null + + session = RtcSession( + client = streamVideo, + powerManager = mockPowerManager, + call = mockCall, + sessionManager = CallSessionManager(), + sessionId = "session-id", + apiKey = "api-key", + lifecycle = mockLifecycle, + sfuUrl = "https://test-sfu.stream.com", + sfuWsUrl = "wss://test-sfu.stream.com", + sfuToken = "sfu-token", + sfuName = "test-sfu-edge", + remoteIceServers = emptyList(), + clientImpl = mockVideoClient, + coroutineScope = testScope, + sfuConnectionModuleProvider = { mockk(relaxed = true) }, + sfuAnalytics = SfuAnalytics.getFakeSfuAnalytics(), + ) + // A session starts without one; each test opts in. + session.publisher.value = null + } + + @After + fun tearDown() { + StreamVideo.removeClient() + unmockkAll() + } + + private fun publishing(): Publisher = mockk(relaxed = true).also { + session.publisher.value = it + } + + /** Runs the media manager's swap callback with [newTrack], the way a real rebuild would. */ + private fun mediaManagerHandsOver(newTrack: AudioTrack) { + every { mockMediaManager.replaceAudioSourceAndTrack(any()) } answers { + firstArg<(AudioTrack) -> Boolean>().invoke(newTrack) + } + } + + //region rebuildAudioCapturePipeline + + @Test + fun `rebuilding moves the publisher onto the new track`() { + val newTrack = mockk(relaxed = true) + val publisher = publishing() + every { publisher.hasLiveAudioSender() } returns true + every { publisher.replaceAudioTrack(newTrack) } returns true + mediaManagerHandsOver(newTrack) + + assertTrue(session.rebuildAudioCapturePipeline()) + + verify { publisher.replaceAudioTrack(newTrack) } + } + + @Test + fun `rebuilding accepts the new pair when nothing is published yet`() { + mediaManagerHandsOver(mockk(relaxed = true)) + + // There is no sender to move, so the fresh pair simply becomes current — reporting a + // failure here would fail a stage that had nothing to do. + assertTrue(session.rebuildAudioCapturePipeline()) + } + + @Test + fun `rebuilding accepts the new pair when the publisher has no audio sender`() { + val publisher = publishing() + every { publisher.hasLiveAudioSender() } returns false + mediaManagerHandsOver(mockk(relaxed = true)) + + // Joined muted: a publisher exists but nothing is sending audio. The new pair becomes + // current and the next publishStream reads it — same as no publisher at all. + assertTrue(session.rebuildAudioCapturePipeline()) + verify(exactly = 0) { publisher.replaceAudioTrack(any()) } + } + + @Test + fun `rebuilding reports the publisher's refusal`() { + val newTrack = mockk(relaxed = true) + val publisher = publishing() + every { publisher.hasLiveAudioSender() } returns true + every { publisher.replaceAudioTrack(newTrack) } returns false + mediaManagerHandsOver(newTrack) + + assertFalse(session.rebuildAudioCapturePipeline()) + } + + //endregion + + //region bitrate bridges + + @Test + fun `setAudioMaxBitrate goes to the publisher`() { + val publisher = publishing() + every { publisher.setAudioMaxBitrate(128_000) } returns true + + assertTrue(session.setAudioMaxBitrate(128_000)) + } + + @Test + fun `setAudioMaxBitrate reports false with no publisher`() { + assertFalse(session.setAudioMaxBitrate(128_000)) + } + + @Test + fun `hasLiveAudioSender is false with no publisher`() { + assertFalse(session.hasLiveAudioSender()) + } + + @Test + fun `hasLiveAudioSender goes to the publisher`() { + val publisher = publishing() + every { publisher.hasLiveAudioSender() } returns true + + assertTrue(session.hasLiveAudioSender()) + } + + @Test + fun `audioMaxBitrate reads through to the publisher`() { + val publisher = publishing() + every { publisher.audioMaxBitrate() } returns 128_000 + + assertEquals(128_000, session.audioMaxBitrate()) + } + + @Test + fun `audioMaxBitrate is null with no publisher`() { + assertNull(session.audioMaxBitrate()) + } + + @Test + fun `negotiatedAudioBitrate reads through to the publisher`() { + val publisher = publishing() + every { publisher.negotiatedAudioBitrate() } returns 64_000 + + assertEquals(64_000, session.negotiatedAudioBitrate()) + } + + @Test + fun `negotiatedAudioBitrate is null with no publisher`() { + assertNull(session.negotiatedAudioBitrate()) + } + + @Test + fun `audioBitrateFor reads through to the publisher`() { + val publisher = publishing() + every { + publisher.audioBitrateFor(AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY) + } returns 128_000 + + assertEquals( + 128_000, + session.audioBitrateFor( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ), + ) + } + + @Test + fun `audioBitrateFor is null with no publisher`() { + assertNull( + session.audioBitrateFor( + AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY, + ), + ) + } + + //endregion +} diff --git a/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/utils/AudioBitrateTargetTest.kt b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/utils/AudioBitrateTargetTest.kt new file mode 100644 index 00000000000..a92d5c3a104 --- /dev/null +++ b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/utils/AudioBitrateTargetTest.kt @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2014-2026 Stream.io Inc. All rights reserved. + * + * Licensed under the Stream License; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://github.com/GetStream/stream-video-android/blob/main/LICENSE + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.getstream.video.android.core.utils + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import org.webrtc.MediaConstraints +import stream.video.sfu.models.AudioBitrateProfile + +/** + * Covers the two pure helpers a mid-call profile switch leans on: [targetAudioMaxBitrateBps], + * which decides the number the encoder is asked for, and [buildAudioConstraints], which decides + * what the rebuilt audio source is allowed to process. + * + * Three sources feed the bitrate, in order: what the SFU offered for the profile, the constants, + * and what the SFU negotiated at join. That precedence is the whole point, and getting it wrong is + * silent — the switch reports success and the audio sits at the wrong ceiling, which is the + * customer's original complaint wearing a different hat. + */ +class AudioBitrateTargetTest { + + private val music = AudioBitrateProfile.AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY + private val voice = AudioBitrateProfile.AUDIO_BITRATE_PROFILE_VOICE_STANDARD_UNSPECIFIED + + @Test + fun `the server's offer for the profile wins outright`() { + // Whatever the server named for this profile is what a freshly created transceiver would + // have been given, so a mid-call switch has no business inventing a different number. + assertEquals( + 96_000, + targetAudioMaxBitrateBps( + music, + serverBitrateBps = 96_000, + negotiatedBitrateBps = 64_000, + ), + ) + assertEquals( + 48_000, + targetAudioMaxBitrateBps( + voice, + serverBitrateBps = 48_000, + negotiatedBitrateBps = 64_000, + ), + ) + } + + @Test + fun `a zero from the server is the proto default, not an offer`() { + // Wire proto3 gives an absent int32 a zero; asking the encoder for zero would mute it. + assertEquals( + 128_000, + targetAudioMaxBitrateBps(music, serverBitrateBps = 0, negotiatedBitrateBps = null), + ) + assertEquals( + 64_000, + targetAudioMaxBitrateBps(voice, serverBitrateBps = 0, negotiatedBitrateBps = null), + ) + } + + @Test + fun `music never drops below the music constant`() { + assertEquals( + 128_000, + targetAudioMaxBitrateBps(music, serverBitrateBps = null, negotiatedBitrateBps = 64_000), + ) + assertEquals( + 128_000, + targetAudioMaxBitrateBps(music, serverBitrateBps = null, negotiatedBitrateBps = null), + ) + } + + @Test + fun `music keeps a negotiated bitrate that is already higher`() { + // A server that joined the call above the music constant is not talked back down. + assertEquals( + 256_000, + targetAudioMaxBitrateBps( + music, + serverBitrateBps = null, + negotiatedBitrateBps = 256_000, + ), + ) + } + + @Test + fun `voice restores what was negotiated at join`() { + // Leaving music puts back what the SFU asked for, rather than guessing the constant. + assertEquals( + 96_000, + targetAudioMaxBitrateBps(voice, serverBitrateBps = null, negotiatedBitrateBps = 96_000), + ) + } + + @Test + fun `voice falls back to the constant with nothing negotiated`() { + assertEquals( + 64_000, + targetAudioMaxBitrateBps(voice, serverBitrateBps = null, negotiatedBitrateBps = null), + ) + assertEquals( + 64_000, + targetAudioMaxBitrateBps(voice, serverBitrateBps = null, negotiatedBitrateBps = 0), + ) + } + + //region audio-source constraints + + private fun softwareProcessingIn(constraints: MediaConstraints): Boolean = + constraints.optional.single { it.key == "googNoiseSuppression" }.value.toBoolean() + + @Test + fun `the music profile builds constraints with the software processing off`() { + assertFalse(softwareProcessingIn(buildAudioConstraints { music })) + } + + @Test + fun `a voice profile builds constraints with the software processing on`() { + assertTrue(softwareProcessingIn(buildAudioConstraints { voice })) + } + + @Test + fun `no profile provider is treated as a voice profile`() { + // The source can be built before anything has chosen a profile; the default has to be the + // processed one, or every call would start with the music pipeline. + assertTrue(softwareProcessingIn(buildAudioConstraints())) + } + + @Test + fun `all five goog constraints move together`() { + // buildAudioConstraints takes them as one flag on purpose — splitting it (keeping AEC + // under music, say) is a live open question, and this pins today's behaviour. + val off = buildAudioConstraints(softwareAudioProcessingEnabled = false) + val googs = off.optional.filter { it.key.startsWith("goog") } + + assertEquals(5, googs.size) + assertTrue(googs.all { it.value == "false" }) + } + + //endregion +}