patches/expo-audio.patch
Rama main · réplica de lectura del repositorio original.
diff --git a/android/src/main/AndroidManifest.xml b/android/src/main/AndroidManifest.xml
index 36a19f2d9c4a979a940ab681a1fb844ce45f7c75..3509de4747f3ebdda1a014f05fb02f0710563498 100644
--- a/android/src/main/AndroidManifest.xml
+++ b/android/src/main/AndroidManifest.xml
@@ -1,3 +1,8 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" />
+ <!-- Required by ExoPlayer's setWakeMode (see AudioPlayer). Declared here, in
+ the module's own manifest, so it survives `expo prebuild` regenerating
+ the app one. It grants no wake lock by itself: media3 only holds one
+ while actually playing. -->
+ <uses-permission android:name="android.permission.WAKE_LOCK" />
</manifest>
diff --git a/android/src/main/java/expo/modules/audio/AudioModule.kt b/android/src/main/java/expo/modules/audio/AudioModule.kt
index 992239ba7631ce0c97c0b41c5fbf61cc842386fe..4ab89564158370534b36e9ceb73ca61cf3d322b8 100644
--- a/android/src/main/java/expo/modules/audio/AudioModule.kt
+++ b/android/src/main/java/expo/modules/audio/AudioModule.kt
@@ -18,6 +18,7 @@ import androidx.media3.common.C.CONTENT_TYPE_HLS
import androidx.media3.common.C.CONTENT_TYPE_OTHER
import androidx.media3.common.C.CONTENT_TYPE_SS
import androidx.media3.common.MediaItem
+import androidx.media3.common.MediaMetadata
import androidx.media3.common.Player
import androidx.media3.common.util.Util
import androidx.media3.datasource.DataSource
@@ -28,6 +29,7 @@ import androidx.media3.exoplayer.hls.HlsMediaSource
import androidx.media3.exoplayer.smoothstreaming.SsMediaSource
import androidx.media3.exoplayer.source.MediaSource
import androidx.media3.exoplayer.source.ProgressiveMediaSource
+import expo.modules.audio.service.PreloadingBitmapLoader
import expo.modules.interfaces.permissions.Permissions
import expo.modules.kotlin.Promise
import expo.modules.kotlin.exception.Exceptions
@@ -57,6 +59,8 @@ class AudioModule : Module() {
private var shouldPlayInBackground = false
private var audioEnabled = true
private var shouldRouteThroughEarpiece = false
+ /** Whether the earpiece routing below was ever actually turned on. */
+ private var routedThroughEarpiece = false
private var focusAcquired = false
private var interruptionMode: InterruptionMode? = null
private var allowsBackgroundRecording = false
@@ -140,6 +144,17 @@ class AudioModule : Module() {
return playsInSilentMode || audioManager.ringerMode == AudioManager.RINGER_MODE_NORMAL
}
+ /**
+ * `AUDIOFOCUS_GAIN`, not `AUDIOFOCUS_GAIN_TRANSIENT`, when nothing else is
+ * meant to keep playing.
+ *
+ * Transient focus means "lend me the audio for a moment": it is what a
+ * navigation prompt, a notification or a call asks for, and what it tells
+ * whoever was playing is to pause and wait to be given it back. An app
+ * playing an album is not borrowing anything. Upstream asked for transient
+ * focus whatever the mode, so music was announcing itself as an
+ * interruption for as long as it played.
+ */
private fun requestAudioFocus() {
if (focusAcquired || !audioEnabled || interruptionMode == InterruptionMode.MIX_WITH_OTHERS) {
return
@@ -148,14 +163,23 @@ class AudioModule : Module() {
val result = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
val requestType = interruptionMode?.let {
if (it == InterruptionMode.DO_NOT_MIX) {
- AudioManager.AUDIOFOCUS_GAIN_TRANSIENT
+ AudioManager.AUDIOFOCUS_GAIN
} else {
AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK
}
- } ?: AudioManager.AUDIOFOCUS_GAIN_TRANSIENT
+ } ?: AudioManager.AUDIOFOCUS_GAIN
audioFocusRequest = AudioFocusRequest.Builder(requestType).run {
setAudioAttributes(
AudioAttributes.Builder()
+ // Say what this focus is for, the same way the player says what it
+ // plays. Left out, the usage is UNKNOWN, and a car reads the focus
+ // request to decide which of its channels the app belongs to: an
+ // app asking for focus for something it will not name, and only
+ // for a moment, is what a navigation prompt or a call sounds like.
+ // Being put on that channel is what #81 describes from inside the
+ // car: the volume icon is a phone, and the music arrives with the
+ // bandwidth of a phone call.
+ .setUsage(AudioAttributes.USAGE_MEDIA)
.setContentType(AudioAttributes.CONTENT_TYPE_MUSIC)
.build()
)
@@ -169,7 +193,7 @@ class AudioModule : Module() {
} else {
@Suppress("DEPRECATION")
val requestType = if (interruptionMode == InterruptionMode.DO_NOT_MIX) {
- AudioManager.AUDIOFOCUS_GAIN_TRANSIENT
+ AudioManager.AUDIOFOCUS_GAIN
} else {
AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK
}
@@ -363,7 +387,16 @@ class AudioModule : Module() {
bufferDurationMs
)
player.onPlaybackStateChange = { isPlaying ->
- if (!isPlaying && shouldReleaseFocus()) {
+ // Playing again after something that was not a pause: a call that
+ // took the focus for good, a route that changed under us, anything
+ // that left the player sounding with no focus behind it. Asking for
+ // it only in `play` and `replace` meant nothing ever put it back,
+ // and a player holding no focus is one a car will not carry.
+ if (isPlaying) {
+ if (!focusAcquired) {
+ requestAudioFocus()
+ }
+ } else if (shouldReleaseFocus()) {
releaseAudioFocus()
}
}
@@ -376,6 +409,15 @@ class AudioModule : Module() {
player.id
}
+ // Android audio session id of the underlying ExoPlayer. Needed to attach
+ // audio effects (android.media.audiofx) such as an equalizer to this
+ // player's output. Already used internally for the Visualizer.
+ Property("audioSessionId") { player ->
+ runOnMain {
+ player.ref.audioSessionId
+ }
+ }
+
Property("isBuffering") { player ->
runOnMain {
player.ref.playbackState == Player.STATE_BUFFERING
@@ -507,6 +549,27 @@ class AudioModule : Module() {
}
}
+ // Gapless: same source resolution as `replace`, but the track is queued
+ // behind the one playing instead of taking its place, so ExoPlayer
+ // buffers it in advance and joins them with no gap.
+ Function("setNextSource") { player: AudioPlayer, source: AudioSource ->
+ runOnMain {
+ if (player.ref.availableCommands.contains(Player.COMMAND_CHANGE_MEDIA_ITEMS)) {
+ createMediaItem(source)?.let {
+ player.setNextSource(it)
+ }
+ }
+ }
+ }
+
+ Function("clearNextSource") { player: AudioPlayer ->
+ runOnMain {
+ if (player.ref.availableCommands.contains(Player.COMMAND_CHANGE_MEDIA_ITEMS)) {
+ player.clearNextSource()
+ }
+ }
+ }
+
Function("setActiveForLockScreen") { ref: AudioPlayer, active: Boolean, metadata: Metadata?, options: AudioLockScreenOptions? ->
runOnMain {
ref.setActiveForLockScreen(active, metadata, options)
@@ -519,12 +582,26 @@ class AudioModule : Module() {
}
}
+ Function("setLockScreenFavorite") { ref: AudioPlayer, favorite: Boolean?, label: String? ->
+ runOnMain {
+ ref.setLockScreenFavorite(favorite, label)
+ }
+ }
+
Function("clearLockScreenControls") { ref: AudioPlayer ->
runOnMain {
ref.clearLockScreenControls()
}
}
+ // Where the source loaded into this player really starts. See
+ // `AudioPlayer.sessionPositionOffsetMs`. A plain field write, so it does
+ // not need the main thread and must not wait for it: it is set right
+ // beside the `replace()` that installs the re-requested stream.
+ Function("setSessionPositionOffset") { ref: AudioPlayer, offsetMs: Double ->
+ ref.sessionPositionOffsetMs = offsetMs.toLong()
+ }
+
Function("setAudioSamplingEnabled") { player: AudioPlayer, enabled: Boolean ->
runOnMain {
player.setSamplingEnabled(enabled)
@@ -685,7 +762,16 @@ class AudioModule : Module() {
}
playlist.loadInitialPlaylist()
playlist.onPlaybackStateChange = { isPlaying ->
- if (!isPlaying && shouldReleaseFocus()) {
+ // Playing again after something that was not a pause: a call that
+ // took the focus for good, a route that changed under us, anything
+ // that left the player sounding with no focus behind it. Asking for
+ // it only in `play` and `replace` meant nothing ever put it back,
+ // and a player holding no focus is one a car will not carry.
+ if (isPlaying) {
+ if (!focusAcquired) {
+ requestAudioFocus()
+ }
+ } else if (shouldReleaseFocus()) {
releaseAudioFocus()
}
}
@@ -860,13 +946,29 @@ class AudioModule : Module() {
private fun createMediaItem(source: AudioSource?): MediaSource? = source?.uri?.let { uriString ->
val uri = uriString.toUri()
- val mediaItem = when {
+ val baseItem = when {
isRawResource(uri) -> {
val file = getResourceName(uri, uriString)
MediaItem.fromUri(getRawResourceURI(file))
}
else -> MediaItem.fromUri(uri)
}
+ // Named, so that one track is not the next one. See `AudioSource.mediaId`.
+ val identified = source.mediaId?.let { baseItem.buildUpon().setMediaId(it).build() } ?: baseItem
+ // The item's own metadata wins over the tags read out of the stream, which
+ // is the point: a transcode arrives stripped of them, and the cover can be
+ // a file on disk that the stream never had.
+ val mediaItem = source.metadata?.let { meta ->
+ identified.buildUpon().setMediaMetadata(
+ MediaMetadata.Builder()
+ .setTitle(meta.title)
+ .setArtist(meta.artist)
+ .setAlbumTitle(meta.albumTitle)
+ .setArtworkUri(PreloadingBitmapLoader.mask(meta.artworkUrl?.toString()))
+ .setDurationMs(meta.durationMs)
+ .build()
+ ).build()
+ } ?: identified
val preloadedBytes = AudioPreloadManager.get(uriString)
val factory: DataSource.Factory = if (preloadedBytes != null) {
@@ -914,10 +1016,32 @@ class AudioModule : Module() {
}
}
+ /**
+ * Only touches the routing when the earpiece is actually asked for, and only
+ * undoes it once it has been.
+ *
+ * Upstream ran this on every `setAudioModeAsync` call, so an app that never
+ * mentions the earpiece still ended up calling `setSpeakerphoneOn(true)`.
+ * Since Android 12 that is `setCommunicationDevice(built-in speaker)`: the
+ * app claims a device for *call* audio and holds it for as long as it lives.
+ * A music app has no business holding one, and it is the kind of state that
+ * makes a car treat what it is playing as call audio.
+ *
+ * Turning it back off releases the claim instead of forcing the speaker,
+ * which is what "no longer routing through the earpiece" actually means.
+ */
@Suppress("DEPRECATION")
private fun updatePlaySoundThroughEarpiece(playThroughEarpiece: Boolean) {
+ if (!playThroughEarpiece && !routedThroughEarpiece) return
+ routedThroughEarpiece = playThroughEarpiece
audioManager.mode = if (playThroughEarpiece) AudioManager.MODE_IN_COMMUNICATION else AudioManager.MODE_NORMAL
- audioManager.setSpeakerphoneOn(!playThroughEarpiece)
+ if (playThroughEarpiece) {
+ audioManager.setSpeakerphoneOn(false)
+ } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
+ audioManager.clearCommunicationDevice()
+ } else {
+ audioManager.setSpeakerphoneOn(false)
+ }
}
private fun retrieveStreamType(uri: Uri): Int = Util.inferContentType(uri)
@@ -931,7 +1055,7 @@ class AudioModule : Module() {
CONTENT_TYPE_SS -> SsMediaSource.Factory(factory)
CONTENT_TYPE_DASH -> DashMediaSource.Factory(factory)
CONTENT_TYPE_HLS -> HlsMediaSource.Factory(factory)
- CONTENT_TYPE_OTHER -> ProgressiveMediaSource.Factory(factory)
+ CONTENT_TYPE_OTHER -> ProgressiveMediaSource.Factory(factory, seekableExtractors())
else -> throw IllegalStateException("Unsupported type: $type")
}.createMediaSource(mediaItem)
}
diff --git a/android/src/main/java/expo/modules/audio/AudioPlayer.kt b/android/src/main/java/expo/modules/audio/AudioPlayer.kt
index c76a9fd2a53743a691782e0f67d0d71401df58f6..7c9025af6a1645f79f9964b67e040764d8eceb71 100644
--- a/android/src/main/java/expo/modules/audio/AudioPlayer.kt
+++ b/android/src/main/java/expo/modules/audio/AudioPlayer.kt
@@ -8,15 +8,19 @@ import android.util.Log
import androidx.core.content.ContextCompat
import androidx.media3.common.AudioAttributes
import androidx.media3.common.C
+import androidx.media3.common.MediaItem
+import androidx.media3.common.MediaMetadata
import androidx.media3.common.PlaybackException
import androidx.media3.common.PlaybackParameters
import androidx.media3.common.Player
import androidx.media3.common.util.UnstableApi
import androidx.media3.exoplayer.DefaultLoadControl
+import androidx.media3.exoplayer.DefaultRenderersFactory
import androidx.media3.exoplayer.ExoPlayer
import androidx.media3.exoplayer.source.MediaSource
import androidx.media3.session.MediaSession
import expo.modules.audio.service.AudioPlaybackServiceConnection
+import expo.modules.audio.service.PreloadingBitmapLoader
import expo.modules.audio.service.ServiceBindingState
import expo.modules.kotlin.AppContext
import expo.modules.kotlin.exception.Exceptions
@@ -24,6 +28,12 @@ import java.lang.ref.WeakReference
private const val PLAYBACK_STATUS_UPDATE = "playbackStatusUpdate"
private const val AUDIO_SAMPLE_UPDATE = "audioSampleUpdate"
+private const val REMOTE_PREVIOUS = "remotePrevious"
+private const val REMOTE_NEXT = "remoteNext"
+private const val REMOTE_FAVORITE = "remoteFavorite"
+private const val REMOTE_SEEK = "remoteSeek"
+private const val TRACK_TRANSITION = "trackTransition"
+private const val STREAM_METADATA = "streamMetadata"
private const val SEEK_JUMP_INTERVAL_MS: Long = 10_000
@UnstableApi
@@ -36,7 +46,39 @@ class AudioPlayer(
) : BaseAudioPlayer(
player = ExoPlayer.Builder(context)
.setLooper(context.mainLooper)
- .setAudioAttributes(AudioAttributes.DEFAULT, false)
+ // Android does not require devices to ship an ALAC decoder, and Samsung's
+ // codec list does not include one. Keep every platform decoder first, then
+ // let Media3 load the bundled FFmpeg extension only for an unsupported
+ // format. That extension contains ALAC and no other codec.
+ .setRenderersFactory(
+ DefaultRenderersFactory(context).setExtensionRendererMode(
+ DefaultRenderersFactory.EXTENSION_RENDERER_MODE_ON
+ )
+ )
+ // Say what this is. AudioAttributes.DEFAULT leaves the content type as
+ // UNKNOWN, and the system reads it to decide how to treat the audio: which
+ // volume it belongs to, and whether it can go down the low power path
+ // instead of being mixed on the CPU. Music is what we always play.
+ .setAudioAttributes(
+ AudioAttributes.Builder()
+ .setUsage(C.USAGE_MEDIA)
+ .setContentType(C.AUDIO_CONTENT_TYPE_MUSIC)
+ .build(),
+ false
+ )
+ // Pause when the audio output becomes noisy (wired headphones unplugged or a
+ // Bluetooth device disconnects), instead of blasting through the speaker.
+ // expo-audio only handles audio-focus loss, which fires for calls/other apps
+ // but not for a headphone unplug and only inconsistently for Bluetooth — so
+ // without this the app kept playing (or paused only sometimes) on disconnect.
+ .setHandleAudioBecomingNoisy(true)
+ // Keep the CPU and the Wi-Fi radio awake while playing with the screen off.
+ // media3 defaults to WAKE_MODE_NONE: audio already being rendered keeps the
+ // device awake by itself, but fetching the NEXT track does not, so in Doze
+ // playback died at the end of a song and only came back on pause/resume.
+ // WAKE_MODE_NETWORK also takes a WifiLock, which is what stops Wi-Fi from
+ // being turned off in deep sleep. The locks are only held while playing.
+ .setWakeMode(C.WAKE_MODE_NETWORK)
.setSeekForwardIncrementMs(SEEK_JUMP_INTERVAL_MS)
.setSeekBackIncrementMs(SEEK_JUMP_INTERVAL_MS)
.apply {
@@ -64,7 +106,20 @@ class AudioPlayer(
var isActiveForLockScreen = false
internal var metadata: Metadata? = null
internal var lockScreenOptions: AudioLockScreenOptions? = null
- internal var mediaSession: MediaSession = buildBasicMediaSession(context, ref)
+
+ /**
+ * Second of the track the current source begins at, in milliseconds, for
+ * whoever is reading the media session. A stream a server generates on the
+ * fly cannot be seeked, so it is re-requested starting at the second asked
+ * for; the player is handed a new source and counts it from zero, and only
+ * whoever asked for it knows what that zero means. Everything else leaves it
+ * where it is, at zero.
+ */
+ @Volatile
+ var sessionPositionOffsetMs: Long = 0L
+
+ internal var mediaSession: MediaSession =
+ buildBasicMediaSession(context, ref, { sessionPositionOffsetMs }, { emitRemoteSeek(it) })
val serviceConnection = AudioPlaybackServiceConnection(WeakReference(this), appContext)
val isLive: Boolean
@@ -85,11 +140,46 @@ class AudioPlayer(
init {
installPlayerListeners()
+ /**
+ * What the source says it is playing. On an internet radio this is the ICY
+ * metadata the stream sends at the start of every track (ExoPlayer asks for
+ * it on its own and folds it in here, along with ID3 inside the stream), so
+ * it arrives again and again while a single item plays.
+ *
+ * Whether it is worth anything is for JS to decide: on a file this is the
+ * file's own tags, and the app already knows those better than the decoder
+ * does. A listener of its own rather than a hook in `BaseAudioPlayer`,
+ * which has none for this, so the patch stays inside one file.
+ */
+ ref.addListener(object : Player.Listener {
+ override fun onMediaMetadataChanged(mediaMetadata: MediaMetadata) {
+ emit(
+ STREAM_METADATA,
+ mapOf(
+ "title" to mediaMetadata.title?.toString(),
+ "artist" to mediaMetadata.artist?.toString()
+ )
+ )
+ }
+ })
source?.let {
setMediaSource(source)
}
}
+ /**
+ * Gapless: the player moved to the track queued by `setNextSource` on its
+ * own. There is no `didJustFinish` for this (it needs STATE_ENDED, which
+ * never happens with another item waiting), so JS is told here to advance its
+ * index without touching playback.
+ */
+ override fun onMediaItemChanged(mediaItem: MediaItem?, reason: Int) {
+ if (reason == Player.MEDIA_ITEM_TRANSITION_REASON_AUTO) {
+ emit(TRACK_TRANSITION)
+ }
+ super.onMediaItemChanged(mediaItem, reason)
+ }
+
fun setMediaSource(source: MediaSource) {
previousPlaybackState = Player.STATE_IDLE
ref.setMediaSource(source)
@@ -97,6 +187,77 @@ class AudioPlayer(
startUpdating()
}
+ fun emitRemotePrevious() = emit(REMOTE_PREVIOUS)
+ fun emitRemoteNext() = emit(REMOTE_NEXT)
+
+ /** The heart was pressed on the notification, the lock screen or the car. */
+ fun emitRemoteFavorite() = emit(REMOTE_FAVORITE)
+
+ /**
+ * Whether the song playing is a favourite, as JS knows it, for the heart the
+ * session shows; null hides the heart. With its label, which is in the app's
+ * language and the platform reads out.
+ */
+ var lockScreenFavorite: Boolean? = null
+ private set
+ var lockScreenFavoriteLabel: String? = null
+ private set
+
+ fun setLockScreenFavorite(favorite: Boolean?, label: String?) {
+ lockScreenFavorite = favorite
+ lockScreenFavoriteLabel = label
+ if (!isActiveForLockScreen) return
+ val serviceBinder = serviceConnection.playbackServiceBinder
+ if (serviceBinder != null && serviceConnection.bindingState == ServiceBindingState.BOUND) {
+ serviceBinder.service.refreshFavorite(this)
+ }
+ }
+
+ /**
+ * Somebody dragged the progress bar somewhere the source cannot go on its
+ * own (see `playerForMediaSession`). Only whoever queued the track can serve
+ * that, by asking for the stream again, so the second asked for is handed to
+ * JS instead of to the player.
+ */
+ fun emitRemoteSeek(positionMs: Long) = emit(REMOTE_SEEK, mapOf("positionMs" to positionMs))
+
+ /**
+ * Gapless: queues the track that comes after the current one, so ExoPlayer
+ * buffers and joins it by itself instead of the queue (which lives in JS)
+ * only asking for it once the previous one has ended.
+ *
+ * The playlist is kept at two items at most. Whatever already played is
+ * dropped first, so the current item is always index 0 and the queued one
+ * index 1, and a "next" queued earlier is replaced (the queue can change
+ * while a song plays).
+ */
+ fun setNextSource(source: MediaSource) {
+ // Nothing playing: there is nothing to queue BEHIND, and the track would
+ // become the current one without anyone having asked for it.
+ if (ref.mediaItemCount == 0) {
+ return
+ }
+ trimToCurrent()
+ ref.addMediaSource(source)
+ // Its cover too, so the session has it the moment the track starts (#139).
+ appContext?.reactContext?.let {
+ PreloadingBitmapLoader.preload(it, source.mediaItem.mediaMetadata.artworkUri)
+ }
+ }
+
+ /** Drops the queued track, if any, leaving only the one playing. */
+ fun clearNextSource() = trimToCurrent()
+
+ private fun trimToCurrent() {
+ val current = ref.currentMediaItemIndex
+ if (current > 0) {
+ ref.removeMediaItems(0, current)
+ }
+ if (ref.mediaItemCount > 1) {
+ ref.removeMediaItems(1, ref.mediaItemCount)
+ }
+ }
+
fun setActiveForLockScreen(active: Boolean, metadata: Metadata? = null, options: AudioLockScreenOptions? = null) {
if (active) {
this.metadata = metadata
@@ -221,7 +382,8 @@ class AudioPlayer(
internal fun assignBasicMediaSession() {
mediaSession.release()
- mediaSession = buildBasicMediaSession(context, ref)
+ mediaSession =
+ buildBasicMediaSession(context, ref, { sessionPositionOffsetMs }, { emitRemoteSeek(it) })
}
private fun sendAudioSampleUpdate(sample: List<Float>) {
diff --git a/android/src/main/java/expo/modules/audio/AudioRecords.kt b/android/src/main/java/expo/modules/audio/AudioRecords.kt
index 86880d92fd5b6413e8596e1b2b8ae28f2981addd..34044354b228072c1020dfbde62fee049d1521fe 100644
--- a/android/src/main/java/expo/modules/audio/AudioRecords.kt
+++ b/android/src/main/java/expo/modules/audio/AudioRecords.kt
@@ -12,7 +12,27 @@ import expo.modules.kotlin.types.OptimizedRecord
class AudioSource(
@Field val uri: String?,
@Field val headers: Map<String, String>?,
- @Field val name: String? = null
+ @Field val name: String? = null,
+ /**
+ * What this track is, for the media session: the lock screen session is built
+ * on the player, so what a car over Bluetooth (and the system's own controls)
+ * reads is the item's metadata, not the notification's. Left out, all they
+ * get is whatever tags the stream happens to carry — none, once a server
+ * transcodes it.
+ */
+ @Field val metadata: Metadata? = null,
+ /**
+ * What identifies this track to whoever is reading the session.
+ *
+ * `MediaItem.fromUri` leaves the media id as `MediaItem.DEFAULT_MEDIA_ID`,
+ * the empty string, and that is what media3 hands the platform session as
+ * `METADATA_KEY_MEDIA_ID`. Every track this app plays therefore reached a
+ * car, a watch or the Bluetooth stack under the same name, and a head unit
+ * that decides whether to fetch the cover again by looking at that name had
+ * no reason to fetch it a second time. Not the stream URL: the media id goes
+ * out to every connected controller and the URL carries the credentials.
+ */
+ @Field val mediaId: String? = null
) : Record
enum class LoopMode(val value: String) : Enumerable {
@@ -55,7 +75,15 @@ class Metadata(
@Field val title: String?,
@Field val artist: String?,
@Field val albumTitle: String?,
- @Field val artworkUrl: URL?
+ @Field val artworkUrl: URL?,
+ /**
+ * How long the track is, when whoever queued it knows and the stream does
+ * not: a server transcoding on the fly sends no length and no seek table, so
+ * the player has no duration to give and the system's controls drop the
+ * times and the progress bar. media3 falls back to this one whenever the
+ * real duration is unknown, and ignores it otherwise.
+ */
+ @Field val durationMs: Long? = null
) : Record
enum class AndroidOutputFormat(val value: String) : Enumerable {
@@ -110,7 +138,9 @@ enum class AndroidAudioEncoder(val value: String) : Enumerable {
class AudioLockScreenOptions(
@Field val showSeekForward: Boolean,
@Field val showSeekBackward: Boolean,
- @Field val isLiveStream: Boolean? = null
+ @Field val isLiveStream: Boolean? = null,
+ @Field val showSkipPrevious: Boolean = false,
+ @Field val showSkipNext: Boolean = false
) : Record
enum class InterruptionMode(val value: String) : Enumerable {
diff --git a/android/src/main/java/expo/modules/audio/AudioUtils.kt b/android/src/main/java/expo/modules/audio/AudioUtils.kt
index dca91c32a2787a7bc3ecba26f0f3f2ad9cc40d3d..089dcee0b9fe6a369f0809ecdf99a1fa42ea9ef3 100644
--- a/android/src/main/java/expo/modules/audio/AudioUtils.kt
+++ b/android/src/main/java/expo/modules/audio/AudioUtils.kt
@@ -3,6 +3,8 @@ package expo.modules.audio
import android.content.Context
import android.media.AudioDeviceInfo
import android.os.Bundle
+import androidx.media3.common.ForwardingPlayer
+import androidx.media3.common.Player
import androidx.media3.exoplayer.ExoPlayer
import androidx.media3.session.MediaSession
import java.io.File
@@ -31,8 +33,124 @@ fun getMapFromDeviceInfo(deviceInfo: AudioDeviceInfo): Bundle {
return map
}
-fun buildBasicMediaSession(context: Context, player: ExoPlayer): MediaSession {
- return MediaSession.Builder(context, player)
+/**
+ * The player as the media session should see it.
+ *
+ * A server transcoding on the fly answers without a length, and ExoPlayer
+ * files a progressive stream whose length and duration are both unknown under
+ * "live" (`ProgressiveMediaPeriod.setSeekMap`). media3 then refuses to say
+ * where such a stream is: the playback state it publishes carries position -1
+ * and speed 0, so the system's controls, the lock screen and anything reading
+ * the session sit at 0:00 for the whole song while the app's own progress
+ * moves (#135). The second play of the same track was fine, which is the
+ * confusing part of the report: by then Navidrome serves the transcode it
+ * cached, with a length.
+ *
+ * A track whose length we were told is not live, whatever the stream looks
+ * like, and saying so is all it takes to get the position back. That is
+ * exactly the case this covers: `durationMs` is set from the queue for every
+ * song and left out for a radio, which is the thing that really is live and
+ * keeps the player's own answer.
+ *
+ * Nor can the player seek such a source, and it says so: the commands it
+ * offers carry no seek, so the system's controls, the car and the watch drew
+ * a bar nobody could drag while the app's own slider worked. The app is what
+ * makes those seeks work — it asks the server for the stream starting where
+ * you pointed — so the session offers the seek on its behalf
+ * (`onSeek`) whenever the track has a length to seek within. A radio, which
+ * has none, keeps the player's answer and stays undraggable, as it should.
+ *
+ * `positionOffsetMs` is the other half of that: a stream re-requested at
+ * 2:00 is a new source, and the player counts it from zero. The app knows
+ * where it really is and says so here, so the session and the app agree on
+ * one clock instead of drifting apart at every seek. Position, buffer and
+ * duration all move with it, since they are read against each other. It
+ * stays at zero for everything else.
+ */
+private class MediaSessionPlayer(
+ player: Player,
+ private val positionOffsetMs: () -> Long,
+ private val onSeek: ((Long) -> Unit)?,
+) : ForwardingPlayer(player) {
+ /** Does whoever queued this track know how long it is? (a radio does not) */
+ private fun hasKnownDuration(): Boolean = currentMediaItem?.mediaMetadata?.durationMs != null
+
+ override fun isCurrentMediaItemLive(): Boolean =
+ if (hasKnownDuration()) false else super.isCurrentMediaItemLive()
+
+ /** Can the source itself be seeked, or does it have to be asked for again? */
+ private fun playerCanSeek(): Boolean =
+ super.getAvailableCommands().contains(Player.COMMAND_SEEK_IN_CURRENT_MEDIA_ITEM)
+
+ override fun getAvailableCommands(): Player.Commands {
+ val commands = super.getAvailableCommands()
+ if (onSeek == null || !hasKnownDuration()) {
+ return commands
+ }
+ return commands.buildUpon().add(Player.COMMAND_SEEK_IN_CURRENT_MEDIA_ITEM).build()
+ }
+
+ // What the player itself does not know stays unknown: shifting -1, or
+ // `TIME_UNSET`, by the offset would invent an answer.
+ private fun shifted(ms: Long): Long = if (ms < 0) ms else ms + positionOffsetMs()
+
+ override fun getCurrentPosition(): Long = shifted(super.getCurrentPosition())
+
+ override fun getContentPosition(): Long = shifted(super.getContentPosition())
+
+ override fun getBufferedPosition(): Long = shifted(super.getBufferedPosition())
+
+ override fun getContentBufferedPosition(): Long = shifted(super.getContentBufferedPosition())
+
+ // How long the source lasts is on the same clock as where it is, and it has
+ // to be shifted with it: the segment starting at 2:00 of a four minute song
+ // is two minutes long, so leaving it alone put a position of 2:00 on a bar
+ // that ended there. Anything past the middle of a song came out pinned to
+ // the end. A segment with no length of its own says so and the track's own
+ // duration, which the app puts in the item, answers instead.
+ override fun getDuration(): Long = shifted(super.getDuration())
+
+ override fun getContentDuration(): Long = shifted(super.getContentDuration())
+
+ // Somebody scrubbing from the car or the lock screen is pointing at the
+ // clock this reports, so the offset comes back off before the player is
+ // asked. A source that cannot be seeked at all is handed over instead: the
+ // app knows how to get there, by asking the server for the stream starting
+ // at that second, and it is also the one place that knows what the seconds
+ // before the current segment even are.
+ override fun seekTo(positionMs: Long) {
+ val seek = onSeek
+ if (seek != null && !playerCanSeek()) {
+ seek(positionMs)
+ return
+ }
+ super.seekTo(positionMs - positionOffsetMs())
+ }
+
+ override fun seekTo(mediaItemIndex: Int, positionMs: Long) {
+ val seek = onSeek
+ if (seek != null && !playerCanSeek() && mediaItemIndex == currentMediaItemIndex) {
+ seek(positionMs)
+ return
+ }
+ super.seekTo(mediaItemIndex, positionMs - positionOffsetMs())
+ }
+}
+
+/** Wraps a player for a media session. See [MediaSessionPlayer]. */
+fun playerForMediaSession(
+ player: Player,
+ positionOffsetMs: () -> Long = { 0L },
+ onSeek: ((Long) -> Unit)? = null,
+): Player = MediaSessionPlayer(player, positionOffsetMs, onSeek)
+
+fun buildBasicMediaSession(
+ context: Context,
+ player: ExoPlayer,
+ positionOffsetMs: () -> Long = { 0L },
+ onSeek: ((Long) -> Unit)? = null,
+): MediaSession {
+ return MediaSession.Builder(context, playerForMediaSession(player, positionOffsetMs, onSeek))
.setId("ExpoAudioBasicMediaSession_${player.hashCode()}")
.build()
}
diff --git a/android/src/main/java/expo/modules/audio/BaseAudioPlayer.kt b/android/src/main/java/expo/modules/audio/BaseAudioPlayer.kt
index 149e9075c8630b14a69318aa6c37d9ff82b23f05..422d2d18677223f48b0106e44f573659b90160dd 100644
--- a/android/src/main/java/expo/modules/audio/BaseAudioPlayer.kt
+++ b/android/src/main/java/expo/modules/audio/BaseAudioPlayer.kt
@@ -78,7 +78,6 @@ abstract class BaseAudioPlayer(
val listener = object : Player.Listener {
override fun onIsPlayingChanged(isPlaying: Boolean) {
playing = isPlaying
- onPlaybackStateChange?.invoke(isPlaying)
val isTransient = !isPlaying &&
(ref.playbackState == Player.STATE_ENDED || ref.playbackState == Player.STATE_BUFFERING)
@@ -86,9 +85,22 @@ abstract class BaseAudioPlayer(
intendedPlayingState = isPlaying
}
+ // Behind the check and not in front of it. `isPlaying` goes false
+ // while the player buffers, which every seek makes it do, and this
+ // callback is what gives up the audio focus: a stop that lasts as long
+ // as one read from the network was being taken for somebody pressing
+ // pause. Nothing asked for the focus back afterwards, because that only
+ // happens in `play` and `replace`, so what was left was a player that
+ // sounds and holds no focus. On the phone that goes unnoticed, since an
+ // AudioTrack keeps running without it. In a car it does not: Android
+ // Auto only carries the media stream to the head unit while the app
+ // holds the focus, so the car fell silent a moment after every seek
+ // while the app played on. The status update below already knew this
+ // state was not worth reporting.
if (isTransient) {
return
}
+ onPlaybackStateChange?.invoke(isPlaying)
sendStatusUpdate(mapOf("playing" to isPlaying))
}
diff --git a/android/src/main/java/expo/modules/audio/FragmentSeekingExtractor.kt b/android/src/main/java/expo/modules/audio/FragmentSeekingExtractor.kt
new file mode 100644
index 0000000000000000000000000000000000000000..a4e9d208d0fa8f34440bc355de02f81129c91a76
--- /dev/null
+++ b/android/src/main/java/expo/modules/audio/FragmentSeekingExtractor.kt
@@ -0,0 +1,425 @@
+package expo.modules.audio
+
+import android.net.Uri
+import androidx.media3.common.C
+import androidx.media3.common.DataReader
+import androidx.media3.common.Format
+import androidx.media3.common.util.ParsableByteArray
+import androidx.media3.extractor.DefaultExtractorsFactory
+import androidx.media3.extractor.Extractor
+import androidx.media3.extractor.ExtractorInput
+import androidx.media3.extractor.ExtractorOutput
+import androidx.media3.extractor.ExtractorsFactory
+import androidx.media3.extractor.PositionHolder
+import androidx.media3.extractor.SeekMap
+import androidx.media3.extractor.SeekPoint
+import androidx.media3.extractor.SniffFailure
+import androidx.media3.extractor.TrackOutput
+import androidx.media3.extractor.mp4.FragmentedMp4Extractor
+import androidx.media3.extractor.text.SubtitleParser
+
+/**
+ * Extractors that can seek a file carrying no index of its own.
+ *
+ * A server transcoding on the fly writes into a pipe, so what comes out the
+ * other end is a bare run of frames: Navidrome's AAC is raw ADTS, and an MP3
+ * made that way carries a Xing header it never got to go back and fill in.
+ * Downloaded, that is exactly what ends up on disk, and ExoPlayer does not
+ * seek what it cannot index: it answers the seek by starting the track over
+ * from zero, which is what somebody dragging the slider sees (#123).
+ *
+ * Constant bitrate seeking is the answer to that case and to no other: it
+ * works out where a second lives from the length of the file and its
+ * bitrate, and it is only consulted when the container offers nothing
+ * better. A real MP4, a FLAC, or an MP3 with a header worth reading keeps
+ * its own seek table. And these files are transcodes at a bitrate that was
+ * asked for, which is the case the estimate is exact for.
+ *
+ * Deliberately not `setConstantBitrateSeekingAlwaysEnabled`: that one is for
+ * sources whose length is unknown, which is the stream, not the download. A
+ * transcode arriving over HTTP has no length and takes no range requests
+ * either, so guessing a byte offset in it would get us nothing to play. That
+ * one is re-requested from the server at the second we want instead.
+ *
+ * A fragmented M4A without an index is the other file media3 will not seek,
+ * and `FragmentSeekingExtractorsFactory` covers that one (#242).
+ *
+ * Kept out of AudioModule so `pnpm harness:extractor` runs this very factory.
+ */
+internal fun seekableExtractors(): ExtractorsFactory =
+ FragmentSeekingExtractorsFactory(DefaultExtractorsFactory().setConstantBitrateSeekingEnabled(true))
+
+/**
+ * Seeking in a fragmented MP4 that carries no index (#242).
+ *
+ * A fragmented M4A is a run of small `moof` + `mdat` pairs, and the only index
+ * media3 seeks one with is a `sidx` box up front. Plenty of files have none
+ * (anything saved straight from a DASH stream, which is what most rippers do),
+ * and media3 treats every seek into those as a seek to zero: drag the slider
+ * and the song starts over.
+ *
+ * Every fragment opens with the time it starts at (`tfdt`), though, so the
+ * file can be searched. When the fragmented extractor gives up, this reads the
+ * last fragment once to learn how long the song is, hands ExoPlayer a map from
+ * the first fragment to the last, and on each seek homes in on the fragment
+ * holding the second asked for, a few reads at most, before letting the real
+ * extractor carry on from there.
+ */
+internal class FragmentSeekingExtractorsFactory(
+ private val base: ExtractorsFactory
+) : ExtractorsFactory {
+ override fun createExtractors(): Array<Extractor> = wrap(base.createExtractors())
+
+ override fun createExtractors(uri: Uri, responseHeaders: Map<String, List<String>>): Array<Extractor> =
+ wrap(base.createExtractors(uri, responseHeaders))
+
+ override fun setSubtitleParserFactory(factory: SubtitleParser.Factory): ExtractorsFactory {
+ base.setSubtitleParserFactory(factory)
+ return this
+ }
+
+ @Deprecated("Deprecated in media3")
+ override fun experimentalSetTextTrackTranscodingEnabled(enabled: Boolean): ExtractorsFactory {
+ @Suppress("DEPRECATION")
+ base.experimentalSetTextTrackTranscodingEnabled(enabled)
+ return this
+ }
+
+ override fun experimentalSetCodecsToParseWithinGopSampleDependencies(codecs: Int): ExtractorsFactory {
+ base.experimentalSetCodecsToParseWithinGopSampleDependencies(codecs)
+ return this
+ }
+
+ private fun wrap(extractors: Array<Extractor>): Array<Extractor> =
+ Array(extractors.size) { i ->
+ val e = extractors[i]
+ if (e is FragmentedMp4Extractor) FragmentSeekingExtractor(e) else e
+ }
+}
+
+internal class FragmentSeekingExtractor(private val inner: Extractor) : Extractor {
+ private var input: ExtractorInput? = null
+ private lateinit var output: ExtractorOutput
+
+ /** Of the audio track, from its `mdhd`; 0 when it could not be read. */
+ private var timescale = 0L
+ private var declaredDurationUs = C.TIME_UNSET
+ private var firstFragment = C.INDEX_UNSET.toLong()
+
+ /** Set when the map was held back, until the extractor stops for the first time. */
+ private var tailPending = false
+
+ /** Where the extractor was when the last fragment was read, to go back to. */
+ private var resumeAt = C.INDEX_UNSET.toLong()
+ private var readingTail = false
+
+ /** Fragments whose start is known, by position. Our map is live when set. */
+ private var known: MutableList<Fragment>? = null
+
+ private var seekTargetUs = C.TIME_UNSET
+ private var seekHops = 0
+
+ /** No fragment starting before `timeUs` lies at or past `position`: where
+ * the last scan began that only found one starting too late. */
+ private var ceiling: Fragment? = null
+
+ override fun sniff(input: ExtractorInput): Boolean = inner.sniff(input)
+
+ override fun getSniffFailureDetails(): List<SniffFailure> = inner.sniffFailureDetails
+
+ override fun init(output: ExtractorOutput) {
+ this.output = output
+ inner.init(Output(output))
+ }
+
+ override fun read(input: ExtractorInput, seekPosition: PositionHolder): Int {
+ this.input = input
+ if (timescale == 0L && input.position == 0L) timescale = readTimescale(input)
+ if (resumeAt != C.INDEX_UNSET.toLong()) return readTail(input, seekPosition)
+ if (seekTargetUs != C.TIME_UNSET) {
+ val result = homeIn(input, seekPosition)
+ if (result != null) return result
+ }
+ val result = inner.read(input, seekPosition)
+ // The map is sent from inside the read, which carries on to the end of the
+ // first sample: that is the byte to come back to, not the one the map was
+ // sent from, or the rest of the first fragment comes out as noise.
+ if (tailPending) {
+ tailPending = false
+ if (result == Extractor.RESULT_CONTINUE) resumeAt = input.position
+ else output.seekMap(SeekMap.Unseekable(declaredDurationUs))
+ }
+ return result
+ }
+
+ override fun seek(position: Long, timeUs: Long) {
+ inner.seek(position, timeUs)
+ // Zero is the start of the file, not the middle of a fragment: the
+ // extractor reads the header again and needs no help.
+ seekTargetUs = if (known != null && position > 0) timeUs else C.TIME_UNSET
+ seekHops = 0
+ ceiling = null
+ }
+
+ override fun release() = inner.release()
+
+ /**
+ * Out to the last fragment and back. Its start, and the one before it, say
+ * how long the song is; the extractor never sees the trip, since it picks up
+ * again at the very byte it left.
+ */
+ private fun readTail(input: ExtractorInput, seekPosition: PositionHolder): Int {
+ val length = input.length
+ if (!readingTail) {
+ readingTail = true
+ seekPosition.position = maxOf(resumeAt, length - TAIL_BYTES)
+ return Extractor.RESULT_SEEK
+ }
+ val bytes = peekUpTo(input, (length - input.position).coerceAtMost(TAIL_BYTES.toLong()).toInt())
+ val fragments = mutableListOf<Fragment>()
+ var k = 0
+ while (k <= bytes.size - BOX_HEAD) {
+ if (isFragmentAt(bytes, k)) {
+ val t = fragmentTimeUs(bytes, k)
+ if (t != C.TIME_UNSET) fragments.add(Fragment(input.position + k, t))
+ }
+ k++
+ }
+ val last = fragments.lastOrNull()
+ if (last == null || last.timeUs <= 0) {
+ output.seekMap(SeekMap.Unseekable(declaredDurationUs))
+ } else {
+ // The last fragment lasts about as long as the one before it did.
+ val span = if (fragments.size > 1) last.timeUs - fragments[fragments.size - 2].timeUs else 0L
+ val guessUs = last.timeUs + span
+ val declared = declaredDurationUs
+ val durationUs = if (declared != C.TIME_UNSET && declared >= last.timeUs && declared <= guessUs + span) declared else guessUs
+ known = (listOf(Fragment(firstFragment, 0)) + fragments).toMutableList()
+ output.seekMap(FragmentSeekMap(durationUs, firstFragment, last))
+ }
+ seekPosition.position = resumeAt
+ resumeAt = C.INDEX_UNSET.toLong()
+ return Extractor.RESULT_SEEK
+ }
+
+ /**
+ * One step towards the fragment that holds the second asked for: finds the
+ * next fragment from here and either settles on it or jumps again, in
+ * proportion between the closest fragments known on either side. Null once
+ * settled with the input at a fragment the extractor can read.
+ */
+ private fun homeIn(input: ExtractorInput, seekPosition: PositionHolder): Int? {
+ val known = known ?: return null
+ val target = seekTargetUs
+ val scanFrom = input.position
+ val nearby = nextFragments(input)
+ val found = nearby.firstOrNull()
+ for (f in nearby) if (known.none { it.position == f.position }) known.add(f)
+ known.sortBy { it.position }
+ val before = known.lastOrNull { it.timeUs <= target }
+ val after = known.firstOrNull { it.timeUs > target }
+ val settle = when {
+ // Past the last fragment: the last one that starts in time.
+ found == null -> before
+ // The read may have gone past the second asked for: the last fragment of
+ // it that starts in time, if it is close enough.
+ before != null && before.position >= found.position && target - before.timeUs <= CLOSE_ENOUGH_US -> before
+ // Nothing lies between `before` and one that starts too late, so the
+ // second asked for is inside `before`: straight from it to the next, or
+ // both side by side in this read.
+ before != null && after != null &&
+ ((found == after && scanFrom <= before.position + 1) ||
+ (before in nearby && nearby.getOrNull(nearby.indexOf(before) + 1) == after)) -> before
+ seekHops >= MAX_HOPS -> before ?: found
+ else -> null
+ }
+ if (settle != null || found == null) {
+ seekTargetUs = C.TIME_UNSET
+ if (settle == null) return Extractor.RESULT_END_OF_INPUT
+ if (settle.position == input.position) return null
+ seekPosition.position = settle.position
+ return Extractor.RESULT_SEEK
+ }
+ seekHops++
+ if (found.timeUs > target && scanFrom < (ceiling?.position ?: Long.MAX_VALUE)) {
+ ceiling = Fragment(scanFrom, found.timeUs)
+ }
+ val lo = before ?: known.first()
+ val hi = listOfNotNull(after, ceiling).minByOrNull { it.position } ?: known.last()
+ val aimUs = (target - LEAD_US).coerceAtLeast(lo.timeUs)
+ // Bytes are only roughly in proportion to time (a quiet intro takes next to
+ // nothing), so every other jump halves the gap instead: that bounds the
+ // search however wrong the proportion is.
+ val share = when {
+ // A gap one read covers is read whole, from its start.
+ hi.position - lo.position <= SCAN_BYTES -> 0.0
+ seekHops % 2 == 0 -> 0.5
+ hi.timeUs > lo.timeUs -> (aimUs - lo.timeUs).toDouble() / (hi.timeUs - lo.timeUs)
+ else -> 0.0
+ }
+ val next = lo.position + ((hi.position - lo.position) * share).toLong()
+ // Always past `lo`, so the scan finds the fragment after it at worst.
+ seekPosition.position = next.coerceIn(lo.position + 1, maxOf(lo.position + 1, hi.position - 1))
+ return Extractor.RESULT_SEEK
+ }
+
+ /**
+ * The fragments from the input's position on, as many as sit in one read,
+ * the nearest first; empty when there are none. Skips what comes before the
+ * first, so the input may be left further on than it was.
+ */
+ private fun nextFragments(input: ExtractorInput): List<Fragment> {
+ var scanned = 0L
+ while (scanned < MAX_SCAN_BYTES) {
+ input.resetPeekPosition()
+ val bytes = peekUpTo(input, SCAN_BYTES)
+ input.resetPeekPosition()
+ if (bytes.size < BOX_HEAD) return emptyList()
+ val found = mutableListOf<Fragment>()
+ for (k in 0..bytes.size - BOX_HEAD) {
+ if (!isFragmentAt(bytes, k)) continue
+ val t = if (k + int(bytes, k) <= bytes.size) {
+ fragmentTimeUs(bytes, k)
+ } else if (found.isEmpty()) {
+ // The nearest one's `tfdt` may sit past this read: look at all of it.
+ input.advancePeekPosition(k)
+ val moof = peekUpTo(input, int(bytes, k).coerceAtMost(MAX_MOOF_BYTES))
+ input.resetPeekPosition()
+ fragmentTimeUs(moof, 0)
+ } else {
+ C.TIME_UNSET
+ }
+ if (t != C.TIME_UNSET) found.add(Fragment(input.position + k, t))
+ else if (found.isEmpty()) return emptyList()
+ }
+ if (found.isNotEmpty()) return found
+ val step = bytes.size - BOX_HEAD + 1
+ input.skipFully(step)
+ scanned += step
+ }
+ return emptyList()
+ }
+
+ /** Reads what is there, up to `max` bytes, from the peek position on. */
+ private fun peekUpTo(input: ExtractorInput, max: Int): ByteArray {
+ val buf = ByteArray(max)
+ var n = 0
+ while (n < max) {
+ val read = input.peek(buf, n, max - n)
+ if (read == C.RESULT_END_OF_INPUT) break
+ n += read
+ }
+ return if (n == max) buf else buf.copyOf(n)
+ }
+
+ /** The track's timescale from the `mdhd` in the header, or 0. */
+ private fun readTimescale(input: ExtractorInput): Long {
+ input.resetPeekPosition()
+ val bytes = peekUpTo(input, HEADER_BYTES)
+ input.resetPeekPosition()
+ val i = find(bytes, MDHD, 0)
+ if (i < 0 || i + 8 > bytes.size) return 0
+ val v1 = bytes[i + 4].toInt() == 1
+ val at = i + 8 + if (v1) 16 else 8
+ return if (at + 4 <= bytes.size) int(bytes, at).toLong() and 0xffffffffL else 0
+ }
+
+ /** When the fragment at `k` starts, from its `tfdt`. */
+ private fun fragmentTimeUs(b: ByteArray, k: Int): Long {
+ if (timescale <= 0) return C.TIME_UNSET
+ val end = (k + int(b, k)).coerceAtMost(b.size)
+ val i = find(b, TFDT, k, end)
+ if (i < 0 || i + 8 > end) return C.TIME_UNSET
+ val v1 = b[i + 4].toInt() == 1
+ val at = i + 8
+ val units = if (v1) {
+ if (at + 8 > end) return C.TIME_UNSET
+ (int(b, at).toLong() shl 32) or (int(b, at + 4).toLong() and 0xffffffffL)
+ } else {
+ if (at + 4 > end) return C.TIME_UNSET
+ int(b, at).toLong() and 0xffffffffL
+ }
+ return units * 1_000_000L / timescale
+ }
+
+ private inner class Output(private val out: ExtractorOutput) : ExtractorOutput {
+ override fun track(id: Int, type: Int): TrackOutput = out.track(id, type)
+
+ override fun endTracks() = out.endTracks()
+
+ override fun seekMap(seekMap: SeekMap) {
+ val input = input
+ // A `sidx` beats anything worked out here, and without a length (a
+ // stream generated on the fly) there is no last fragment to go and read.
+ if (seekMap.isSeekable || input == null || input.length == C.LENGTH_UNSET.toLong() || timescale <= 0) {
+ out.seekMap(seekMap)
+ return
+ }
+ // Sent the moment the first `moof` header has been read.
+ declaredDurationUs = seekMap.durationUs
+ firstFragment = input.position - 8
+ tailPending = true
+ }
+ }
+
+ private data class Fragment(val position: Long, val timeUs: Long)
+
+ /** Between the first fragment and the last, in proportion. Only a first
+ * guess: `homeIn` takes it from there. */
+ private class FragmentSeekMap(
+ private val durationUs: Long,
+ private val first: Long,
+ private val last: Fragment
+ ) : SeekMap {
+ override fun isSeekable() = true
+
+ override fun getDurationUs() = durationUs
+
+ override fun getSeekPoints(timeUs: Long): SeekMap.SeekPoints {
+ if (timeUs <= 0) return SeekMap.SeekPoints(SeekPoint(timeUs, 0))
+ val share = ((timeUs - LEAD_US).toDouble() / last.timeUs).coerceIn(0.0, 1.0)
+ val position = first + ((last.position - first) * share).toLong()
+ return SeekMap.SeekPoints(SeekPoint(timeUs, position.coerceAtLeast(first + 1)))
+ }
+ }
+
+ private companion object {
+ const val HEADER_BYTES = 64 * 1024
+ const val TAIL_BYTES = 1024 * 1024
+ const val SCAN_BYTES = 64 * 1024
+ const val MAX_SCAN_BYTES = 16L * 1024 * 1024
+ const val MAX_MOOF_BYTES = 256 * 1024
+ const val MAX_HOPS = 12
+ /** A `moof` header and the `mfhd` that always opens it. */
+ const val BOX_HEAD = 16
+ /** Landing this far before the second asked for is as good as on it:
+ * ExoPlayer skips the rest itself. */
+ const val CLOSE_ENOUGH_US = 10_000_000L
+ /** How far ahead of the second asked for each jump aims. */
+ const val LEAD_US = 2_000_000L
+
+ val MDHD = "mdhd".toByteArray()
+ val TFDT = "tfdt".toByteArray()
+
+ fun isFragmentAt(b: ByteArray, k: Int): Boolean =
+ b[k + 4] == 'm'.code.toByte() && b[k + 5] == 'o'.code.toByte() &&
+ b[k + 6] == 'o'.code.toByte() && b[k + 7] == 'f'.code.toByte() &&
+ b[k + 12] == 'm'.code.toByte() && b[k + 13] == 'f'.code.toByte() &&
+ b[k + 14] == 'h'.code.toByte() && b[k + 15] == 'd'.code.toByte() &&
+ int(b, k) >= BOX_HEAD + 8 && int(b, k + 8) == 16
+
+ /** Where the four letters of a box type are, from `from + 4`; -1 if absent. */
+ fun find(b: ByteArray, type: ByteArray, from: Int, to: Int = b.size): Int {
+ var i = from + 4
+ while (i + 4 <= to) {
+ if (b[i] == type[0] && b[i + 1] == type[1] && b[i + 2] == type[2] && b[i + 3] == type[3]) return i
+ i++
+ }
+ return -1
+ }
+
+ fun int(b: ByteArray, k: Int): Int =
+ (b[k].toInt() and 0xff shl 24) or (b[k + 1].toInt() and 0xff shl 16) or
+ (b[k + 2].toInt() and 0xff shl 8) or (b[k + 3].toInt() and 0xff)
+ }
+}
diff --git a/android/src/main/java/expo/modules/audio/service/AudioControlsService.kt b/android/src/main/java/expo/modules/audio/service/AudioControlsService.kt
index 196a5d57f887c2667a88543bd2f6a80044a044f2..f795ace73af25b6c19bf74f3a0664a24fab904c1 100644
--- a/android/src/main/java/expo/modules/audio/service/AudioControlsService.kt
+++ b/android/src/main/java/expo/modules/audio/service/AudioControlsService.kt
@@ -27,6 +27,7 @@ import expo.modules.audio.AudioLockScreenOptions
import expo.modules.audio.AudioPlayer
import expo.modules.audio.Metadata
import expo.modules.audio.getPlaybackServiceErrorMessage
+import expo.modules.audio.playerForMediaSession
import expo.modules.kotlin.AppContext
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
@@ -90,6 +91,9 @@ class AudioControlsService : MediaSessionService() {
ACTION_SEEK_FORWARD -> currentPlayerRef.seekTo(currentPlayerRef.currentPosition + SEEK_INTERVAL_MS)
ACTION_SEEK_BACKWARD -> currentPlayerRef.seekTo(currentPlayerRef.currentPosition - SEEK_INTERVAL_MS)
+ ACTION_SKIP_PREVIOUS -> currentPlayer?.emitRemotePrevious()
+ ACTION_SKIP_NEXT -> currentPlayer?.emitRemoteNext()
+ ACTION_TOGGLE_FAVORITE -> currentPlayer?.emitRemoteFavorite()
}
}
@@ -192,6 +196,18 @@ class AudioControlsService : MediaSessionService() {
val compactViewIndices = mutableListOf<Int>()
var currentIndex = 0
+ if (currentOptions?.showSkipPrevious == true) {
+ builder.addAction(
+ NotificationCompat.Action(
+ androidx.media3.session.R.drawable.media3_icon_previous,
+ "Skip Previous",
+ buildActionPendingIntent(ACTION_SKIP_PREVIOUS)
+ )
+ )
+ compactViewIndices.add(currentIndex)
+ currentIndex++
+ }
+
if (currentOptions?.showSeekBackward == true) {
builder.addAction(
NotificationCompat.Action(
@@ -227,6 +243,33 @@ class AudioControlsService : MediaSessionService() {
)
)
compactViewIndices.add(currentIndex)
+ currentIndex++
+ }
+
+ if (currentOptions?.showSkipNext == true) {
+ builder.addAction(
+ NotificationCompat.Action(
+ androidx.media3.session.R.drawable.media3_icon_next,
+ "Skip Next",
+ buildActionPendingIntent(ACTION_SKIP_NEXT)
+ )
+ )
+ compactViewIndices.add(currentIndex)
+ }
+
+ // Last and out of the compact view, which keeps to the transport.
+ currentPlayer?.lockScreenFavorite?.let { favorite ->
+ builder.addAction(
+ NotificationCompat.Action(
+ if (favorite) {
+ androidx.media3.session.R.drawable.media3_icon_heart_filled
+ } else {
+ androidx.media3.session.R.drawable.media3_icon_heart_unfilled
+ },
+ currentPlayer?.lockScreenFavoriteLabel ?: "Favorite",
+ buildActionPendingIntent(ACTION_TOGGLE_FAVORITE)
+ )
+ )
}
style.setShowActionsInCompactView(*compactViewIndices.toIntArray())
@@ -240,19 +283,41 @@ class AudioControlsService : MediaSessionService() {
val session = mediaSession ?: return
val mediaButtons = mutableListOf<CommandButton>()
+ val showSkipPrevious = currentOptions?.showSkipPrevious == true
+ val showSkipNext = currentOptions?.showSkipNext == true
+ val showSeekBackward = currentOptions?.showSeekBackward == true
+ val showSeekForward = currentOptions?.showSeekForward == true
+
+ // Add skip previous button (primary back slot)
+ if (showSkipPrevious) {
+ mediaButtons.add(
+ CommandButton.Builder(CommandButton.ICON_PREVIOUS)
+ .setDisplayName("Skip Previous")
+ .setEnabled(true)
+ .setSessionCommand(SessionCommand(ACTION_SKIP_PREVIOUS, Bundle.EMPTY))
+ .setSlots(CommandButton.SLOT_BACK, CommandButton.SLOT_OVERFLOW)
+ .build()
+ )
+ }
+
// Add seek backward button if enabled
- if (currentOptions?.showSeekBackward == true) {
+ if (showSeekBackward) {
+ val slot = if (showSkipPrevious) {
+ intArrayOf(CommandButton.SLOT_BACK_SECONDARY, CommandButton.SLOT_OVERFLOW)
+ } else {
+ intArrayOf(CommandButton.SLOT_BACK, CommandButton.SLOT_OVERFLOW)
+ }
mediaButtons.add(
CommandButton.Builder(CommandButton.ICON_SKIP_BACK_10)
.setDisplayName("Seek Backward")
.setEnabled(true)
.setSessionCommand(SessionCommand(ACTION_SEEK_BACKWARD, Bundle.EMPTY))
- .setSlots(CommandButton.SLOT_BACK)
+ .setSlots(*slot)
.build()
)
}
- // Add play/pause button (always present)
+ // Add play/pause button (always present, central slot)
mediaButtons.add(
CommandButton.Builder(if (isPlaying) CommandButton.ICON_PAUSE else CommandButton.ICON_PLAY)
.setDisplayName(if (isPlaying) "Pause" else "Play")
@@ -263,13 +328,47 @@ class AudioControlsService : MediaSessionService() {
)
// Add seek forward button if enabled
- if (currentOptions?.showSeekForward == true) {
+ if (showSeekForward) {
+ val slot = if (showSkipNext) {
+ intArrayOf(CommandButton.SLOT_FORWARD_SECONDARY, CommandButton.SLOT_OVERFLOW)
+ } else {
+ intArrayOf(CommandButton.SLOT_FORWARD, CommandButton.SLOT_OVERFLOW)
+ }
mediaButtons.add(
CommandButton.Builder(CommandButton.ICON_SKIP_FORWARD_10)
.setDisplayName("Seek Forward")
.setEnabled(true)
.setSessionCommand(SessionCommand(ACTION_SEEK_FORWARD, Bundle.EMPTY))
- .setSlots(CommandButton.SLOT_FORWARD)
+ .setSlots(*slot)
+ .build()
+ )
+ }
+
+ // Add skip next button (primary forward slot)
+ if (showSkipNext) {
+ mediaButtons.add(
+ CommandButton.Builder(CommandButton.ICON_NEXT)
+ .setDisplayName("Skip Next")
+ .setEnabled(true)
+ .setSessionCommand(SessionCommand(ACTION_SKIP_NEXT, Bundle.EMPTY))
+ .setSlots(CommandButton.SLOT_FORWARD, CommandButton.SLOT_OVERFLOW)
+ .build()
+ )
+ }
+
+ // The heart, where the platform puts actions of the app's own: beside the
+ // transport controls on the notification and the lock screen, and on the
+ // car's playback screen. Only while JS says the song can be one (not a
+ // radio, say), and drawn as what pressing it would undo.
+ currentPlayer?.lockScreenFavorite?.let { favorite ->
+ mediaButtons.add(
+ CommandButton.Builder(
+ if (favorite) CommandButton.ICON_HEART_FILLED else CommandButton.ICON_HEART_UNFILLED
+ )
+ .setDisplayName(currentPlayer?.lockScreenFavoriteLabel ?: "Favorite")
+ .setEnabled(true)
+ .setSessionCommand(SessionCommand(ACTION_TOGGLE_FAVORITE, Bundle.EMPTY))
+ .setSlots(CommandButton.SLOT_OVERFLOW)
.build()
)
}
@@ -278,6 +377,15 @@ class AudioControlsService : MediaSessionService() {
session.setMediaButtonPreferences(mediaButtons)
}
+ /** The heart changed for this player: the buttons and the notification again. */
+ fun refreshFavorite(player: AudioPlayer) {
+ if (player != currentPlayer) return
+ appContext?.mainQueue?.launch {
+ updateSessionCustomLayout(player.ref.isPlaying)
+ postOrStartForegroundNotificationNow(startInForeground = false)
+ }
+ }
+
private fun postOrStartForegroundNotification(startInForeground: Boolean) {
if (startInForeground) {
postOrStartForegroundNotificationNow(startInForeground = true)
@@ -321,17 +429,36 @@ class AudioControlsService : MediaSessionService() {
}
private fun resolveSessionPlayer(player: AudioPlayer, options: AudioLockScreenOptions?): Player {
+ // The session sees the player through ours first: it answers where a
+ // re-requested stream really is, and that a track we were told the length
+ // of is not live however the stream looks (see `playerForMediaSession`).
+ // Upstream's own wrapper goes on top of it, and both forward.
+ val base = playerForMediaSession(
+ player.ref,
+ { player.sessionPositionOffsetMs },
+ { positionMs -> player.emitRemoteSeek(positionMs) }
+ )
val isLive = options?.isLiveStream ?: player.isLive
- if (!isLive) {
- return player.ref
- }
- return object : ForwardingPlayer(player.ref) {
+ return object : ForwardingPlayer(base) {
+ // The player only offers "next" while gapless has the next track loaded,
+ // and media3 drops a command the player does not offer before
+ // AudioMediaSessionCallback.onPlayerCommandRequest sees it: Garmin's next
+ // worked some of the time (#307). Previous never had the problem, a
+ // single track can always go back to its start. The callback runs all
+ // four in JS, so the player is never asked to.
override fun getAvailableCommands(): Player.Commands {
- return super.getAvailableCommands().buildUpon()
- .remove(Player.COMMAND_SEEK_IN_CURRENT_MEDIA_ITEM)
- .build()
+ val commands = super.getAvailableCommands().buildUpon()
+ .add(Player.COMMAND_SEEK_TO_NEXT)
+ .add(Player.COMMAND_SEEK_TO_NEXT_MEDIA_ITEM)
+ .add(Player.COMMAND_SEEK_TO_PREVIOUS)
+ .add(Player.COMMAND_SEEK_TO_PREVIOUS_MEDIA_ITEM)
+ if (isLive) commands.remove(Player.COMMAND_SEEK_IN_CURRENT_MEDIA_ITEM)
+ return commands.build()
}
+
+ override fun isCommandAvailable(command: Int): Boolean =
+ getAvailableCommands().contains(command)
}
}
@@ -371,7 +498,14 @@ class AudioControlsService : MediaSessionService() {
updateMetadata(metadata)
}
val session = MediaSession.Builder(context, sessionPlayer)
- .setCallback(AudioMediaSessionCallback())
+ .setBitmapLoader(PreloadingBitmapLoader(context))
+ .setCallback(
+ AudioMediaSessionCallback(
+ onSkipPrevious = { currentPlayer?.emitRemotePrevious() },
+ onSkipNext = { currentPlayer?.emitRemoteNext() },
+ onToggleFavorite = { currentPlayer?.emitRemoteFavorite() }
+ )
+ )
.build()
// Replace the basic media session with a session connected to our playback service.
@@ -452,7 +586,14 @@ class AudioControlsService : MediaSessionService() {
updateMetadata(metadata)
}
val session = MediaSession.Builder(context, sessionPlayer)
- .setCallback(AudioMediaSessionCallback())
+ .setBitmapLoader(PreloadingBitmapLoader(context))
+ .setCallback(
+ AudioMediaSessionCallback(
+ onSkipPrevious = { currentPlayer?.emitRemotePrevious() },
+ onSkipNext = { currentPlayer?.emitRemoteNext() },
+ onToggleFavorite = { currentPlayer?.emitRemoteFavorite() }
+ )
+ )
.build()
player.mediaSession.release()
@@ -558,6 +699,9 @@ class AudioControlsService : MediaSessionService() {
const val ACTION_SEEK_FORWARD = "expo.modules.audio.action.SEEK_FORWARD"
const val ACTION_SEEK_BACKWARD = "expo.modules.audio.action.SEEK_BACKWARD"
+ const val ACTION_SKIP_PREVIOUS = "expo.modules.audio.action.SKIP_PREVIOUS"
+ const val ACTION_SKIP_NEXT = "expo.modules.audio.action.SKIP_NEXT"
+ const val ACTION_TOGGLE_FAVORITE = "expo.modules.audio.action.TOGGLE_FAVORITE"
const val SEEK_INTERVAL_MS = 10000L
}
diff --git a/android/src/main/java/expo/modules/audio/service/AudioMediaSessionCallback.kt b/android/src/main/java/expo/modules/audio/service/AudioMediaSessionCallback.kt
index 839aaef12d7699ea1a89037b31849775daa079f4..97cc2d7ea824fa87ac958031a283cc56b8676031 100644
--- a/android/src/main/java/expo/modules/audio/service/AudioMediaSessionCallback.kt
+++ b/android/src/main/java/expo/modules/audio/service/AudioMediaSessionCallback.kt
@@ -1,40 +1,54 @@
package expo.modules.audio.service
+import android.content.Intent
+import android.os.Build
import android.os.Bundle
+import android.view.KeyEvent
import androidx.media3.common.Player
import androidx.media3.session.MediaSession
import androidx.media3.session.SessionCommand
+import androidx.media3.session.SessionError
import androidx.media3.session.SessionResult
import androidx.annotation.OptIn
import androidx.media3.common.util.UnstableApi
import com.google.common.util.concurrent.ListenableFuture
@OptIn(UnstableApi::class)
-class AudioMediaSessionCallback : MediaSession.Callback {
+class AudioMediaSessionCallback(
+ private val onSkipPrevious: () -> Unit = {},
+ private val onSkipNext: () -> Unit = {},
+ /** The heart in the notification, the lock screen and the car. */
+ private val onToggleFavorite: () -> Unit = {}
+) : MediaSession.Callback {
override fun onConnect(
session: MediaSession,
controller: MediaSession.ControllerInfo
): MediaSession.ConnectionResult {
try {
+ val playerCommands = MediaSession.ConnectionResult.DEFAULT_PLAYER_COMMANDS.buildUpon()
+ // Keep seek commands for the seek slider
+ .add(Player.COMMAND_SEEK_IN_CURRENT_MEDIA_ITEM)
+ .add(Player.COMMAND_SEEK_FORWARD)
+ .add(Player.COMMAND_SEEK_BACK)
+ // Only the notification goes without them, which has skip buttons of its own.
+ // Other apps (Garmin, #307) skip through these, see onPlayerCommandRequest.
+ if (session.isMediaNotificationController(controller)) {
+ playerCommands
+ .remove(Player.COMMAND_SEEK_TO_PREVIOUS_MEDIA_ITEM)
+ .remove(Player.COMMAND_SEEK_TO_NEXT_MEDIA_ITEM)
+ .remove(Player.COMMAND_SEEK_TO_PREVIOUS)
+ .remove(Player.COMMAND_SEEK_TO_NEXT)
+ }
// Configure commands - custom layout buttons will be rendered from session
return MediaSession.ConnectionResult.AcceptedResultBuilder(session)
- .setAvailablePlayerCommands(
- MediaSession.ConnectionResult.DEFAULT_PLAYER_COMMANDS.buildUpon()
- // Keep seek commands for the seek slider
- .add(Player.COMMAND_SEEK_IN_CURRENT_MEDIA_ITEM)
- .add(Player.COMMAND_SEEK_FORWARD)
- .add(Player.COMMAND_SEEK_BACK)
- // Remove track navigation commands
- .remove(Player.COMMAND_SEEK_TO_PREVIOUS_MEDIA_ITEM)
- .remove(Player.COMMAND_SEEK_TO_NEXT_MEDIA_ITEM)
- .remove(Player.COMMAND_SEEK_TO_PREVIOUS)
- .remove(Player.COMMAND_SEEK_TO_NEXT)
- .build()
- )
+ .setAvailablePlayerCommands(playerCommands.build())
.setAvailableSessionCommands(
MediaSession.ConnectionResult.DEFAULT_SESSION_COMMANDS.buildUpon()
.add(SessionCommand(AudioControlsService.ACTION_SEEK_BACKWARD, Bundle.EMPTY))
.add(SessionCommand(AudioControlsService.ACTION_SEEK_FORWARD, Bundle.EMPTY))
+ .add(SessionCommand(AudioControlsService.ACTION_SKIP_PREVIOUS, Bundle.EMPTY))
+ .add(SessionCommand(AudioControlsService.ACTION_SKIP_NEXT, Bundle.EMPTY))
+ .add(SessionCommand(AudioControlsService.ACTION_TOGGLE_FAVORITE, Bundle.EMPTY))
.build()
)
.build()
@@ -43,6 +57,54 @@ class AudioMediaSessionCallback : MediaSession.Callback {
}
}
+ /** The real queue lives in JS, so the player must never run these itself. */
+ override fun onPlayerCommandRequest(
+ session: MediaSession,
+ controller: MediaSession.ControllerInfo,
+ playerCommand: Int
+ ): Int {
+ when (playerCommand) {
+ Player.COMMAND_SEEK_TO_NEXT, Player.COMMAND_SEEK_TO_NEXT_MEDIA_ITEM -> onSkipNext()
+ Player.COMMAND_SEEK_TO_PREVIOUS, Player.COMMAND_SEEK_TO_PREVIOUS_MEDIA_ITEM -> onSkipPrevious()
+ else -> return super.onPlayerCommandRequest(session, controller, playerCommand)
+ }
+ return SessionError.ERROR_NOT_SUPPORTED
+ }
+
+ /**
+ * Hardware / AVRCP / wired media buttons send next & previous as the standard
+ * SEEK_TO_NEXT/PREVIOUS player commands, which onConnect removes (the ExoPlayer
+ * holds a single item; the real queue lives in JS). Intercept those keys here
+ * and route them to the same skip callbacks the notification buttons use, so
+ * skipping from Bluetooth headphones etc. works. Only ACTION_DOWN, to avoid
+ * firing twice (down + up).
+ */
+ override fun onMediaButtonEvent(
+ session: MediaSession,
+ controllerInfo: MediaSession.ControllerInfo,
+ intent: Intent
+ ): Boolean {
+ val keyEvent = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
+ intent.getParcelableExtra(Intent.EXTRA_KEY_EVENT, KeyEvent::class.java)
+ } else {
+ @Suppress("DEPRECATION")
+ intent.getParcelableExtra<KeyEvent>(Intent.EXTRA_KEY_EVENT)
+ }
+ if (keyEvent != null && keyEvent.action == KeyEvent.ACTION_DOWN) {
+ when (keyEvent.keyCode) {
+ KeyEvent.KEYCODE_MEDIA_NEXT -> {
+ onSkipNext()
+ return true
+ }
+ KeyEvent.KEYCODE_MEDIA_PREVIOUS -> {
+ onSkipPrevious()
+ return true
+ }
+ }
+ }
+ return super.onMediaButtonEvent(session, controllerInfo, intent)
+ }
+
override fun onCustomCommand(
session: MediaSession,
controller: MediaSession.ControllerInfo,
@@ -56,6 +118,9 @@ class AudioMediaSessionCallback : MediaSession.Callback {
AudioControlsService.ACTION_SEEK_BACKWARD -> {
session.player.seekTo(session.player.currentPosition - AudioControlsService.SEEK_INTERVAL_MS)
}
+ AudioControlsService.ACTION_SKIP_PREVIOUS -> onSkipPrevious()
+ AudioControlsService.ACTION_SKIP_NEXT -> onSkipNext()
+ AudioControlsService.ACTION_TOGGLE_FAVORITE -> onToggleFavorite()
}
return super.onCustomCommand(session, controller, command, args)
}
diff --git a/android/src/main/java/expo/modules/audio/service/MetadataInjectingPlayer.kt b/android/src/main/java/expo/modules/audio/service/MetadataInjectingPlayer.kt
index 18203a364d17dcf3de95d22a20175308e3363889..317382e2b89ec80576a0878675e52636d621f0a8 100644
--- a/android/src/main/java/expo/modules/audio/service/MetadataInjectingPlayer.kt
+++ b/android/src/main/java/expo/modules/audio/service/MetadataInjectingPlayer.kt
@@ -3,10 +3,20 @@ package expo.modules.audio.service
import android.os.Handler
import android.os.Looper
import androidx.core.net.toUri
+import androidx.media3.common.AudioAttributes
+import androidx.media3.common.DeviceInfo
import androidx.media3.common.FlagSet
import androidx.media3.common.ForwardingPlayer
+import androidx.media3.common.MediaItem
import androidx.media3.common.MediaMetadata
+import androidx.media3.common.PlaybackException
+import androidx.media3.common.PlaybackParameters
import androidx.media3.common.Player
+import androidx.media3.common.Timeline
+import androidx.media3.common.TrackSelectionParameters
+import androidx.media3.common.Tracks
+import androidx.media3.common.VideoSize
+import androidx.media3.common.text.CueGroup
import androidx.media3.common.util.UnstableApi
import expo.modules.audio.Metadata
import java.util.IdentityHashMap
@@ -22,6 +32,38 @@ internal class MetadataInjectingPlayer(
private val listeners = IdentityHashMap<Player.Listener, Player.Listener>()
private var injectedMetadata: Metadata? = null
+ /**
+ * The item the injected metadata was handed over for.
+ *
+ * What is injected here describes one track, and it is only ever pushed from
+ * JS. The player, though, moves on by itself: the track queued behind this
+ * one starts, and until JS hears about it and pushes again, everything
+ * reading the session — the notification, the lock screen, the car, a watch —
+ * was being told the title, the album and the cover of the track that had
+ * just ended, over audio that was already the next one. In the foreground
+ * that push lands in the same breath and nobody sees it; minimized, with a
+ * whole album auto-advancing, it is the cover from a neighbouring track that
+ * people report.
+ *
+ * So the injection is tied to what it was said about. Once the player is on
+ * another item, the item's own metadata answers instead, which is no fallback
+ * at all: every source is queued with its title, artist, album and artwork
+ * already on it (see `AudioSource.metadata`), so the session is right from
+ * the instant of the jump, whenever JS gets around to agreeing.
+ */
+ private var injectedFor: MediaItem? = null
+
+ /**
+ * Is what was injected still about the track being played?
+ *
+ * By value and not by identity: whether the player hands back the very
+ * `MediaItem` it was given is its business, and two consecutive tracks are
+ * two different addresses either way. The one case this lets through is a
+ * source queued behind an identical one, which is a song repeating itself —
+ * and the metadata of a song is still the metadata of that song.
+ */
+ private fun injectionApplies(): Boolean = injectedFor == currentMediaItem
+
override fun addListener(listener: Player.Listener) {
val forwardingListener = synchronized(listeners) {
listeners.getOrPut(listener) { MetadataForwardingListener(listener) }
@@ -37,13 +79,16 @@ internal class MetadataInjectingPlayer(
}
override fun getMediaMetadata(): MediaMetadata {
+ if (!injectionApplies()) {
+ return super.getMediaMetadata()
+ }
val metadata = injectedMetadata
return super.getMediaMetadata()
.buildUpon()
.setTitle(metadata?.title)
.setArtist(metadata?.artist)
.setAlbumTitle(metadata?.albumTitle)
- .setArtworkUri(metadata?.artworkUrl?.toString()?.toUri())
+ .setArtworkUri(PreloadingBitmapLoader.mask(metadata?.artworkUrl?.toString()))
.setArtworkData(null, null)
.build()
}
@@ -56,6 +101,9 @@ internal class MetadataInjectingPlayer(
val previousMetadata = mediaMetadata
injectedMetadata = metadata
+ // Said about whatever is playing right now. JS pushes this straight after
+ // installing the source, so that is the track it is describing.
+ injectedFor = currentMediaItem
val newMetadata = mediaMetadata
if (previousMetadata == newMetadata) {
@@ -79,9 +127,28 @@ internal class MetadataInjectingPlayer(
}
}
+ /**
+ * Everything the player says, said again to the listener, with the metadata
+ * swapped for the injected one.
+ *
+ * Every method of `Player.Listener` is a Java `default`, and Kotlin's `by`
+ * delegation writes no forwarder for a member that already has a body: it
+ * inherits the empty default instead. So `: Player.Listener by listener`
+ * compiled down to a listener that answers to the two methods below and
+ * silently swallows the other thirty-odd. The listener being swallowed here
+ * is the media session's own, which is how the session stopped hearing
+ * anything the player did by itself: repeating a track and moving to the one
+ * queued behind it both left the notification's progress bar sitting where
+ * the song had ended. Whatever the app asked for (play, pause, a new source)
+ * still came through, because the service reacts to those on a listener of
+ * its own, which is what made this look like a bug about repeat.
+ *
+ * Hence the long list. Deprecated callbacks are left out: media3 raises the
+ * current ones alongside them.
+ */
private inner class MetadataForwardingListener(
private val listener: Player.Listener
- ) : Player.Listener by listener {
+ ) : Player.Listener {
override fun onMediaMetadataChanged(mediaMetadata: MediaMetadata) {
listener.onMediaMetadataChanged(this@MetadataInjectingPlayer.mediaMetadata)
}
@@ -89,5 +156,92 @@ internal class MetadataInjectingPlayer(
override fun onEvents(player: Player, events: Player.Events) {
listener.onEvents(this@MetadataInjectingPlayer, events)
}
+
+ override fun onTimelineChanged(timeline: Timeline, reason: Int) =
+ listener.onTimelineChanged(timeline, reason)
+
+ override fun onMediaItemTransition(mediaItem: MediaItem?, reason: Int) =
+ listener.onMediaItemTransition(mediaItem, reason)
+
+ override fun onTracksChanged(tracks: Tracks) = listener.onTracksChanged(tracks)
+
+ override fun onPlaylistMetadataChanged(mediaMetadata: MediaMetadata) =
+ listener.onPlaylistMetadataChanged(mediaMetadata)
+
+ override fun onIsLoadingChanged(isLoading: Boolean) = listener.onIsLoadingChanged(isLoading)
+
+ override fun onAvailableCommandsChanged(availableCommands: Player.Commands) =
+ listener.onAvailableCommandsChanged(availableCommands)
+
+ override fun onTrackSelectionParametersChanged(parameters: TrackSelectionParameters) =
+ listener.onTrackSelectionParametersChanged(parameters)
+
+ override fun onPlaybackStateChanged(playbackState: Int) =
+ listener.onPlaybackStateChanged(playbackState)
+
+ override fun onPlayWhenReadyChanged(playWhenReady: Boolean, reason: Int) =
+ listener.onPlayWhenReadyChanged(playWhenReady, reason)
+
+ override fun onPlaybackSuppressionReasonChanged(reason: Int) =
+ listener.onPlaybackSuppressionReasonChanged(reason)
+
+ override fun onIsPlayingChanged(isPlaying: Boolean) = listener.onIsPlayingChanged(isPlaying)
+
+ override fun onRepeatModeChanged(repeatMode: Int) = listener.onRepeatModeChanged(repeatMode)
+
+ override fun onShuffleModeEnabledChanged(shuffleModeEnabled: Boolean) =
+ listener.onShuffleModeEnabledChanged(shuffleModeEnabled)
+
+ override fun onPlayerError(error: PlaybackException) = listener.onPlayerError(error)
+
+ override fun onPlayerErrorChanged(error: PlaybackException?) =
+ listener.onPlayerErrorChanged(error)
+
+ override fun onPositionDiscontinuity(
+ oldPosition: Player.PositionInfo,
+ newPosition: Player.PositionInfo,
+ reason: Int
+ ) = listener.onPositionDiscontinuity(oldPosition, newPosition, reason)
+
+ override fun onPlaybackParametersChanged(playbackParameters: PlaybackParameters) =
+ listener.onPlaybackParametersChanged(playbackParameters)
+
+ override fun onSeekBackIncrementChanged(seekBackIncrementMs: Long) =
+ listener.onSeekBackIncrementChanged(seekBackIncrementMs)
+
+ override fun onSeekForwardIncrementChanged(seekForwardIncrementMs: Long) =
+ listener.onSeekForwardIncrementChanged(seekForwardIncrementMs)
+
+ override fun onMaxSeekToPreviousPositionChanged(maxSeekToPreviousPositionMs: Long) =
+ listener.onMaxSeekToPreviousPositionChanged(maxSeekToPreviousPositionMs)
+
+ override fun onAudioSessionIdChanged(audioSessionId: Int) =
+ listener.onAudioSessionIdChanged(audioSessionId)
+
+ override fun onAudioAttributesChanged(audioAttributes: AudioAttributes) =
+ listener.onAudioAttributesChanged(audioAttributes)
+
+ override fun onVolumeChanged(volume: Float) = listener.onVolumeChanged(volume)
+
+ override fun onSkipSilenceEnabledChanged(skipSilenceEnabled: Boolean) =
+ listener.onSkipSilenceEnabledChanged(skipSilenceEnabled)
+
+ override fun onDeviceInfoChanged(deviceInfo: DeviceInfo) =
+ listener.onDeviceInfoChanged(deviceInfo)
+
+ override fun onDeviceVolumeChanged(volume: Int, muted: Boolean) =
+ listener.onDeviceVolumeChanged(volume, muted)
+
+ override fun onVideoSizeChanged(videoSize: VideoSize) = listener.onVideoSizeChanged(videoSize)
+
+ override fun onSurfaceSizeChanged(width: Int, height: Int) =
+ listener.onSurfaceSizeChanged(width, height)
+
+ override fun onRenderedFirstFrame() = listener.onRenderedFirstFrame()
+
+ override fun onCues(cueGroup: CueGroup) = listener.onCues(cueGroup)
+
+ override fun onMetadata(metadata: androidx.media3.common.Metadata) =
+ listener.onMetadata(metadata)
}
}
diff --git a/android/src/main/java/expo/modules/audio/service/PreloadingBitmapLoader.kt b/android/src/main/java/expo/modules/audio/service/PreloadingBitmapLoader.kt
new file mode 100644
index 0000000000000000000000000000000000000000..5cd423a7887ac9b5748de734ccd13c08fd4c9cd5
--- /dev/null
+++ b/android/src/main/java/expo/modules/audio/service/PreloadingBitmapLoader.kt
@@ -0,0 +1,116 @@
+package expo.modules.audio.service
+
+import android.content.Context
+import android.graphics.Bitmap
+import android.net.Uri
+import android.util.LruCache
+import androidx.media3.common.util.BitmapLoader
+import androidx.media3.common.util.UnstableApi
+import androidx.media3.datasource.DataSourceBitmapLoader
+import com.google.common.util.concurrent.FutureCallback
+import com.google.common.util.concurrent.Futures
+import com.google.common.util.concurrent.ListenableFuture
+import com.google.common.util.concurrent.MoreExecutors
+import java.security.MessageDigest
+
+/**
+ * The media session's artwork loader, able to have a cover ready before the
+ * track it belongs to starts (#139).
+ *
+ * media3 publishes a track change to the platform session straight away, with
+ * no picture if the cover's future is not done yet, and again once it is. The
+ * Bluetooth stack forwards both, but a car told "new track, no cover" does not
+ * always come back for the second one: a Tesla keeps showing nothing. A cover
+ * fetched while the previous track plays is a done future, so the title and
+ * the picture reach the car in the same update.
+ *
+ * It is also the only thing that reads the real address of a cover. A server's
+ * cover URL carries the account's credentials in its query, and the session
+ * hands its metadata to every app allowed to control media: scrobblers,
+ * watches, automation apps. So the session is given an opaque
+ * `resonus-art://<id>` instead (see [mask]), and this loader, in our own
+ * process, is the one that knows which URL each id stands for.
+ */
+@UnstableApi
+internal class PreloadingBitmapLoader(context: Context) : BitmapLoader {
+ private val delegate = loaderFor(context)
+
+ override fun supportsMimeType(mimeType: String): Boolean = delegate.supportsMimeType(mimeType)
+
+ override fun decodeBitmap(data: ByteArray): ListenableFuture<Bitmap> = delegate.decodeBitmap(data)
+
+ override fun loadBitmap(uri: Uri): ListenableFuture<Bitmap> {
+ val real = unmask(uri)
+ ready.get(real.toString())?.let { return Futures.immediateFuture(it) }
+ return load(delegate, real)
+ }
+
+ companion object {
+ private const val MASKED_SCHEME = "resonus-art"
+
+ /** Opaque id to real address. Big enough to outlive the queue around the
+ * track playing, which is all the session ever asks about. */
+ private val masked = LruCache<String, String>(256)
+
+ /**
+ * What the session publishes for [url]: the url itself when it is a local
+ * file or a resource, which tells nobody anything, and an opaque
+ * `resonus-art://<id>` for a network address, which may carry credentials.
+ */
+ fun mask(url: String?): Uri? {
+ if (url.isNullOrEmpty()) return null
+ val uri = Uri.parse(url)
+ if (uri.scheme != "http" && uri.scheme != "https") return uri
+ val id = MessageDigest.getInstance("SHA-256")
+ .digest(url.toByteArray())
+ .take(16)
+ .joinToString("") { "%02x".format(it) }
+ masked.put(id, url)
+ return Uri.parse("$MASKED_SCHEME://$id")
+ }
+
+ /** The address behind a masked uri, or the uri itself if it is not one. */
+ fun unmask(uri: Uri): Uri {
+ if (uri.scheme != MASKED_SCHEME) return uri
+ return masked.get(uri.host ?: return uri)?.let(Uri::parse) ?: uri
+ }
+
+ /** A few: the track playing, the one queued behind it, and some slack. */
+ private val ready = LruCache<String, Bitmap>(4)
+
+ @Volatile private var shared: BitmapLoader? = null
+
+ private fun loaderFor(context: Context): BitmapLoader =
+ shared ?: synchronized(this) {
+ shared ?: DataSourceBitmapLoader.Builder(context.applicationContext)
+ // The session scales it down again to its own limit.
+ .setMaximumOutputDimension(1024)
+ .build()
+ .also { shared = it }
+ }
+
+ private fun load(loader: BitmapLoader, uri: Uri): ListenableFuture<Bitmap> {
+ val key = uri.toString()
+ val future = loader.loadBitmap(uri)
+ Futures.addCallback(
+ future,
+ object : FutureCallback<Bitmap> {
+ override fun onSuccess(result: Bitmap) {
+ ready.put(key, result)
+ }
+
+ override fun onFailure(t: Throwable) {}
+ },
+ MoreExecutors.directExecutor()
+ )
+ return future
+ }
+
+ /** Fetches a cover ahead of its track. Does nothing if it is already here. */
+ fun preload(context: Context, uri: Uri?) {
+ val real = uri?.let(::unmask) ?: return
+ if (ready.get(real.toString()) != null) return
+ load(loaderFor(context), real)
+ }
+ }
+}
diff --git a/build/Audio.types.d.ts b/build/Audio.types.d.ts
index 25604783649fe9b64f6655172f08471a5efcbf45..dde071902d07312f2c0511ac4b34701e94701021 100644
--- a/build/Audio.types.d.ts
+++ b/build/Audio.types.d.ts
@@ -34,6 +34,28 @@ export type AudioSource = string | number | null | {
* Useful for showing track names in a queue or playlist UI.
*/
name?: string;
+ /**
+ * What this track is, for the media session (Bluetooth, Android Auto, the
+ * system's own controls). Their metadata comes from the item, not from
+ * `setActiveForLockScreen`, and the fields given here win over the tags
+ * carried by the stream — which a transcode arrives without. Android reads
+ * it for the session; iOS takes the cover from it, fetched ahead of time.
+ */
+ metadata?: AudioMetadata;
+ /**
+ * What identifies this track to whoever is reading the media session.
+ * Without it every item is `MediaItem.DEFAULT_MEDIA_ID`, the empty string,
+ * and a car cannot tell one track from the next. Never the stream URL: it
+ * goes out to every connected controller. Android only.
+ */
+ mediaId?: string;
+ /**
+ * Whether the asset resolves seeks to the exact frame, which costs a scan of
+ * the file for an index first — on a streamed mp3, a wait before anything
+ * plays. Defaults to `true`. Set it to `false` for short previews that only
+ * need to land near a position. iOS only.
+ */
+ preferPreciseTiming?: boolean;
};
/**
* Options for configuring audio player behavior.
@@ -618,6 +640,15 @@ export type AudioMetadata = {
artist?: string;
albumTitle?: string;
artworkUrl?: string;
+ /**
+ * How long the track is, in milliseconds, for when the source can't say:
+ * a server transcoding on the fly sends no length, so the player has no
+ * duration and the system's controls show neither the times nor the
+ * progress bar. Used only as a fallback, never over a real duration, and
+ * read from the source's `metadata` on Android and from the lock screen
+ * metadata on iOS.
+ */
+ durationMs?: number;
};
/**
* Loop mode for audio playlist playback.
diff --git a/build/AudioConstants.d.ts b/build/AudioConstants.d.ts
index cfd92ad0ff5765254c4e6db974fe44bce34aad06..78fa084cf34a5f56743d636b6501f90971d7898b 100644
--- a/build/AudioConstants.d.ts
+++ b/build/AudioConstants.d.ts
@@ -15,5 +15,9 @@ export type AudioLockScreenOptions = {
* and scrub bar, and disable seek controls.
*/
isLiveStream?: boolean;
+ /** Whether the skip-to-previous-track button should be displayed. Emits `remotePrevious`. */
+ showSkipPrevious?: boolean;
+ /** Whether the skip-to-next-track button should be displayed. Emits `remoteNext`. */
+ showSkipNext?: boolean;
};
//# sourceMappingURL=AudioConstants.d.ts.map
\ No newline at end of file
diff --git a/build/AudioModule.types.d.ts b/build/AudioModule.types.d.ts
index f3ae4ee4044d86032cd5c6dbcfee6f905d1ce99f..42933ee69678b8bdfa28345582dd32e8f1d7efdb 100644
--- a/build/AudioModule.types.d.ts
+++ b/build/AudioModule.types.d.ts
@@ -16,6 +16,30 @@ export declare class NativeAudioModule extends NativeModule {
clearPreloadedSource(source: AudioSource): Promise<void>;
clearAllPreloadedSources(): Promise<void>;
getPreloadedSources(): Promise<string[]>;
+ /**
+ * Starts the cover at `url` downloading — and, when it animates, its clip
+ * encoding — with no track attached: the queue ahead is warmed this way,
+ * and the lock screen reads the result the moment each track starts.
+ * @param url The cover's URL, as metadata carries it.
+ */
+ warmArtwork(url: string): void;
+ /**
+ * Sets the fps animated covers encode at from here on (30 by default).
+ * @param fps The frame rate, 30 or 60.
+ */
+ setArtworkFps(fps: number): void;
+ /**
+ * Hands the lock screen a tall HLS clip from the animated-artwork
+ * endpoint for one static cover — or clears it with a null clip.
+ * @param artworkUrl The static cover URL the clip was fetched for.
+ * @param tallUrl The tall `.m3u8`, or null when there is none.
+ */
+ setAnimatedClipUrls(artworkUrl: string, tallUrl: string | null): void;
+ /**
+ * What the last stream-clip export did (`stage` and one line of
+ * `detail`), for the diagnostics screen.
+ */
+ animatedClipState(): { stage: string; detail: string };
readonly AudioPlayer: typeof AudioPlayer;
readonly AudioRecorder: typeof AudioRecorder;
readonly AudioPlaylist: typeof AudioPlaylist;
@@ -31,6 +55,11 @@ export declare class AudioPlayer extends SharedObject<AudioEvents> {
* Unique identifier for the player object.
*/
id: string;
+ /**
+ * Android audio session id of the underlying player, for attaching audio
+ * effects (such as an equalizer) to this player's output. Android only.
+ */
+ audioSessionId: number;
/**
* Boolean value indicating whether the player is currently playing.
*/
@@ -141,6 +170,14 @@ export declare class AudioPlayer extends SharedObject<AudioEvents> {
* Replaces the current audio source with a new one.
*/
replace(source: AudioSource): void;
+ /**
+ * Queues a source behind the one playing so the player joins them with no
+ * gap, and emits `trackTransition` when it moves on to it. Replaces a
+ * previously queued one.
+ */
+ setNextSource(source: AudioSource): void;
+ /** Drops the source queued by `setNextSource`, if any. */
+ clearNextSource(): void;
/**
* Seeks the playback by the given number of seconds.
* @param seconds The number of seconds to seek by.
@@ -183,6 +220,24 @@ export declare class AudioPlayer extends SharedObject<AudioEvents> {
* This will clear the lock screen's now playing info.
*/
clearLockScreenControls(): void;
+ /**
+ * Second of the track the source loaded into this player begins at, in
+ * milliseconds, for whoever is reading the media session (Android only).
+ *
+ * A stream a server generates on the fly cannot be seeked, so it is
+ * re-requested starting at the second asked for: the player is handed a
+ * new source and counts it from zero, and only whoever asked for it knows
+ * what that zero means. Set it beside that load and back to 0 when a
+ * source starts at the beginning, and the lock screen stops disagreeing
+ * with the app about where the song is.
+ */
+ setSessionPositionOffset(offsetMs: number): void;
+ /**
+ * The heart on the notification, the lock screen and the car (Android
+ * only): whether the song is a favourite, or null to hide it, and what the
+ * button is called. Pressing it emits `remoteFavorite`.
+ */
+ setLockScreenFavorite(favorite: boolean | null, label: string | null): void;
/**
* Remove the player from memory to free up resources.
*/
@@ -222,6 +277,29 @@ export type AudioEvents = {
playbackStatusUpdate(status: AudioStatus): void;
/** Fired when audio sampling is enabled and new sample data is available. */
audioSampleUpdate(data: AudioSample): void;
+ /** Fired when the user taps skip-previous on the lock screen / notification. */
+ remotePrevious(): void;
+ /** Fired when the user taps skip-next on the lock screen / notification. */
+ remoteNext(): void;
+ /** Fired when the heart is pressed on the notification, lock screen or car. */
+ remoteFavorite(): void;
+ /** Fired when the progress bar of the lock screen / notification / car is
+ * dragged on a source the player cannot seek by itself (a stream a server
+ * generates on the fly). Whoever queued the track is expected to get
+ * there its own way. Android only. */
+ remoteSeek(event: {
+ positionMs: number;
+ }): void;
+ /** Fired when the player moves on by itself to the source queued with
+ * `setNextSource` (gapless advance). */
+ trackTransition(): void;
+ /** Fired when the source announces what it is playing: the ICY metadata of
+ * an internet radio (sent again with every track) or tags inside the
+ * stream. Fields are absent when it doesn't send them. Android only. */
+ streamMetadata(info: {
+ title?: string;
+ artist?: string;
+ }): void;
};
export declare class AudioRecorder extends SharedObject<RecordingEvents> {
/**
diff --git a/ios/AnimatedArtwork.swift b/ios/AnimatedArtwork.swift
new file mode 100644
index 0000000000000000000000000000000000000000..2db79308e6395518e094d7f5b3e7806461523a6e
--- /dev/null
+++ b/ios/AnimatedArtwork.swift
@@ -0,0 +1,989 @@
+import AVFoundation
+import CoreGraphics
+import CoreImage
+import CoreMedia
+import CoreVideo
+import CryptoKit
+import Foundation
+import ImageIO
+import MediaPlayer
+import UIKit
+
+#if compiler(>=6.2)
+
+@available(iOS 26.0, tvOS 26.0, *)
+final class AnimatedArtwork {
+ private static let maxSide = 720
+ // 900 frames is 30 fps across the whole 29 seconds, so this cap only bites
+ // on content that could not have been carried at full rate anyway.
+ private static let maxFrames = 900
+ // The frame-rate ceiling, and the system's own limit above it (Apple,
+ // "Providing animated artwork for media items": not above 60 fps). The cap
+ // is 30 by default — half the frames to decode and to encode, for a cover
+ // that reads the same — and Settings rewrites it through
+ // `AudioModule.setArtworkFps`.
+ private static let rateLock = NSLock()
+ private static var rateCap = 30.0
+ static var maxPlaybackRate: Double {
+ get {
+ rateLock.lock()
+ defer { rateLock.unlock() }
+ return rateCap
+ }
+ set {
+ rateLock.lock()
+ defer { rateLock.unlock() }
+ rateCap = newValue
+ }
+ }
+ // The system prefers a clip that is *less than* 30 seconds, so stop at 29.
+ private static let maxDuration = 29.0
+ private static let minFrameDelay = 1.0 / 60.0
+ private static let fallbackDelay = 0.1
+ private static let browserDelayFloor = 0.011
+ private static let frameOptions = [kCGImageSourceShouldCache: false] as CFDictionary
+ private static let blurContext = CIContext()
+ private static let deviceRGB = CGColorSpaceCreateDeviceRGB()
+
+ /// One encode per destination, shared by every instance holding the same
+ /// clip: an evicted and refetched cover joins the task already writing its
+ /// file instead of racing a second encoder at the same path.
+ private static let sharedTaskLock = NSLock()
+ private static var sharedTasks: [String: Task<URL?, Never>] = [:]
+
+ private enum Ratio {
+ case square
+ case tall
+
+ var suffix: String {
+ switch self {
+ case .square:
+ return "1x1"
+ case .tall:
+ return "3x4"
+ }
+ }
+ }
+
+ private struct Sample {
+ let index: Int
+ let time: Double
+ }
+
+ private struct Schedule {
+ let samples: [Sample]
+ let total: Double
+ }
+
+ private let artworkID: String
+ private let ratio: Ratio
+ private let sourceSize: CGSize
+ private let destination: URL
+ private var data: Data?
+ private let lock = NSLock()
+ private var encodeTask: Task<URL?, Never>?
+
+ /// The object the lock screen is handed, built out of this clip. What
+ /// carries it is the file behind it, so the caller publishes it only once
+ /// `isOnDisk` and never before.
+ private(set) var item: MPMediaItemAnimatedArtwork?
+
+ /// The clip is written out: the system opens it the moment it is asked for
+ /// it. Named for the file, not for the encoding, which has a method of its
+ /// own called `isReady`.
+ var isOnDisk: Bool {
+ FileManager.default.fileExists(atPath: destination.path)
+ }
+
+ /// Calls `onReady` from the task that wrote the clip, once it is on disk
+ /// (right away when it already was). Encoding starts here instead of on the
+ /// system's request: that request arrives with the artwork itself, and one
+ /// answered after the wait left the lock screen on the preview frame — the
+ /// clip was ready all along, nothing told the system to ask again.
+ func whenReady(_ onReady: @escaping () -> Void) {
+ // `.userInitiated`, not `.utility`: this runs when the lock screen is
+ // waiting for the clip, often with the phone locked — the one moment
+ // `.utility` gets almost no CPU.
+ Task.detached(priority: .userInitiated) {
+ _ = await self.videoURL()
+ onReady()
+ }
+ }
+
+ /// The clips the system can ask for: the tall one only. It is the key the
+ /// iPhone lock screen draws full screen, and providing only `1x1` left that
+ /// screen with the static artwork — the square variant that used to be
+ /// built beside it is gone now, which halves the encode and leaves the
+ /// system to show its static cover wherever it would have asked for `1x1`.
+ static func make(url: URL, data: Data) -> (square: AnimatedArtwork?, tall: AnimatedArtwork?) {
+ guard let source = CGImageSourceCreateWithData(data as CFData, nil),
+ CGImageSourceGetCount(source) > 1,
+ let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [String: Any]
+ else {
+ return (nil, nil)
+ }
+ let width = (properties[kCGImagePropertyPixelWidth as String] as? NSNumber)?.intValue ?? 0
+ let height = (properties[kCGImagePropertyPixelHeight as String] as? NSNumber)?.intValue ?? 0
+ guard width > 0, height > 0,
+ let frame = CGImageSourceCreateImageAtIndex(source, 0, frameOptions)
+ else {
+ return (nil, nil)
+ }
+ let sourceSize = CGSize(width: width, height: height)
+ let id = identifier(for: url)
+ let tall = variant(id: id, ratio: .tall, sourceSize: sourceSize, frame: frame, url: url, data: data)
+ return (square: nil, tall: tall)
+ }
+
+ private static func variant(
+ id: String,
+ ratio: Ratio,
+ sourceSize: CGSize,
+ frame: CGImage,
+ url: URL,
+ data: Data
+ ) -> AnimatedArtwork? {
+ guard let destination = cacheFile(id: id, ratio: ratio) else {
+ return nil
+ }
+ let artwork = AnimatedArtwork(
+ artworkID: id + "-" + ratio.suffix,
+ ratio: ratio,
+ sourceSize: sourceSize,
+ destination: destination,
+ data: data
+ )
+ let preview = artwork.previewFrame(frame)
+ artwork.item = MPMediaItemAnimatedArtwork(
+ artworkID: artwork.artworkID,
+ previewImageRequestHandler: { _ in preview },
+ videoAssetFileURLRequestHandler: { _ in await artwork.videoURL() }
+ )
+ return artwork
+ }
+
+ private init(artworkID: String, ratio: Ratio, sourceSize: CGSize, destination: URL, data: Data) {
+ self.artworkID = artworkID
+ self.ratio = ratio
+ self.sourceSize = sourceSize
+ self.destination = destination
+ self.data = data
+ }
+
+ private var canvas: CGSize {
+ let longest = max(Int(sourceSize.width), Int(sourceSize.height))
+ let side = Self.evenSide(longest: longest)
+ switch ratio {
+ case .square:
+ return CGSize(width: side, height: side)
+ case .tall:
+ // A multiple of 6 keeps both edges even and makes 3:4 exact, which the
+ // system checks the same way it checks the preview.
+ let width = max(6, side - (side % 6))
+ return CGSize(width: width, height: width * 4 / 3)
+ }
+ }
+
+ /// The preview is built with the variant's aspect ratio from the first
+ /// frame: a preview that does not match the key makes the system drop the
+ /// animated artwork and fall back to the static one. It is composed the
+ /// same way the clip's frames are — same sized backdrop, same feather
+ /// mask — so the two cannot drift apart.
+ private func previewFrame(_ frame: CGImage) -> UIImage {
+ let size = canvas
+ let format = UIGraphicsImageRendererFormat()
+ format.scale = 1
+ // Not opaque: the composition leans on alpha while it builds, and the
+ // backdrop underneath makes the result opaque anyway.
+ format.opaque = false
+ var backdrop: CGImage? = nil
+ var mask: CGImage? = nil
+ if ratio == .tall {
+ backdrop = Self.blurred(frame).flatMap { Self.sized($0, to: size) }
+ mask = Self.featherMask(size: Self.artRect(in: size).size)
+ }
+ return UIGraphicsImageRenderer(size: size, format: format).image { renderer in
+ let context = renderer.cgContext
+ // UIKit contexts are flipped, so a CGImage drawn straight comes out
+ // upside down — that is what the lock screen showed while the video
+ // was still encoding.
+ context.saveGState()
+ context.translateBy(x: 0, y: size.height)
+ context.scaleBy(x: 1, y: -1)
+ draw(frame, in: context, canvas: size, backdrop: backdrop, mask: mask)
+ context.restoreGState()
+ }
+ }
+
+ /// The square the cover sits in on the tall canvas, centred on it.
+ private static func artRect(in canvas: CGSize) -> CGRect {
+ let side = min(canvas.width, canvas.height)
+ return CGRect(x: 0, y: (canvas.height - side) / 2, width: side, height: side)
+ }
+
+ private func draw(
+ _ frame: CGImage,
+ in context: CGContext,
+ canvas: CGSize,
+ backdrop: CGImage?,
+ mask: CGImage?
+ ) {
+ let bounds = CGRect(origin: .zero, size: canvas)
+ context.interpolationQuality = .high
+ guard ratio == .tall, let backdrop else {
+ context.setFillColor(red: 0, green: 0, blue: 0, alpha: 1)
+ context.fill(bounds)
+ context.draw(frame, in: bounds)
+ return
+ }
+ let art = Self.artRect(in: canvas)
+ guard let mask else {
+ // Only when the mask could not be built: the feather is the point of
+ // the composition, but a cover without it still reads.
+ context.setFillColor(red: 0, green: 0, blue: 0, alpha: 1)
+ context.fill(bounds)
+ context.draw(backdrop, in: bounds)
+ context.draw(frame, in: art)
+ return
+ }
+ // Composed in this one context with nothing allocated per frame: the
+ // cover in the art square, its alpha trimmed by the mask, the backdrop
+ // slipped underneath — which lands on backdrop * (1 - mask) + cover *
+ // mask, the feathered edges melting into the blur. The backdrop is
+ // already sized to the canvas, so every draw here is a blit.
+ context.clear(bounds)
+ context.draw(frame, in: art)
+ context.setBlendMode(.destinationIn)
+ context.draw(mask, in: art)
+ context.setBlendMode(.destinationOver)
+ context.draw(backdrop, in: bounds)
+ context.setBlendMode(.normal)
+ }
+
+ /// The backdrop at canvas size, aspect-filled: scaled once per clip here,
+ /// rather than a high-quality scale per frame of the encode.
+ private static func sized(_ image: CGImage, to canvas: CGSize) -> CGImage? {
+ let width = Int(canvas.width)
+ let height = Int(canvas.height)
+ guard width > 0, height > 0,
+ let context = CGContext(
+ data: nil,
+ width: width,
+ height: height,
+ bitsPerComponent: 8,
+ bytesPerRow: 0,
+ space: deviceRGB,
+ bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
+ )
+ else {
+ return nil
+ }
+ context.interpolationQuality = .high
+ context.setFillColor(red: 0, green: 0, blue: 0, alpha: 1)
+ context.fill(CGRect(x: 0, y: 0, width: canvas.width, height: canvas.height))
+ let source = CGSize(width: image.width, height: image.height)
+ context.draw(image, in: aspectFill(source, in: CGRect(origin: .zero, size: canvas)))
+ return context.makeImage()
+ }
+
+ /// The feather itself: white, transparent at both edges and solid through
+ /// the middle — the same gradient `drawFeathered` used to build on every
+ /// single frame, now built once per clip.
+ private static func featherMask(size: CGSize) -> CGImage? {
+ let width = Int(size.width)
+ let height = Int(size.height)
+ guard width > 4, height > 4,
+ let context = CGContext(
+ data: nil,
+ width: width,
+ height: height,
+ bitsPerComponent: 8,
+ bytesPerRow: 0,
+ space: deviceRGB,
+ bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
+ )
+ else {
+ return nil
+ }
+ let colors = [
+ UIColor.white.withAlphaComponent(0).cgColor,
+ UIColor.white.cgColor,
+ UIColor.white.cgColor,
+ UIColor.white.withAlphaComponent(0).cgColor,
+ ] as CFArray
+ let locations: [CGFloat] = [0, 0.16, 0.84, 1]
+ guard let gradient = CGGradient(colorsSpace: deviceRGB, colors: colors, locations: locations) else {
+ return nil
+ }
+ context.drawLinearGradient(
+ gradient,
+ start: CGPoint(x: 0, y: 0),
+ end: CGPoint(x: 0, y: size.height),
+ options: []
+ )
+ return context.makeImage()
+ }
+
+ private static func blurred(_ frame: CGImage) -> CGImage? {
+ let input = CIImage(cgImage: frame)
+ let extent = input.extent
+ guard extent.width > 1, extent.height > 1, !extent.isInfinite, !extent.isNull else {
+ return downscaleBlur(frame)
+ }
+ guard let filter = CIFilter(name: "CIGaussianBlur") else {
+ return downscaleBlur(frame)
+ }
+ // A gaussian blur samples outside the picture, and into nothing: the outer
+ // band of the backdrop comes out see-through, which is exactly where the
+ // cover's own edge was still readable. The clamp repeats the border pixel
+ // instead, so the backdrop stays solid all the way out and the radius alone
+ // decides how soft it reads.
+ let clamped = input.applyingFilter(
+ "CIAffineClamp",
+ parameters: ["inputTransform": NSValue(cgAffineTransform: .identity)]
+ )
+ filter.setValue(clamped, forKey: kCIInputImageKey)
+ filter.setValue(96.0, forKey: kCIInputRadiusKey)
+ guard let output = filter.outputImage?.cropped(to: extent),
+ let image = blurContext.createCGImage(output, from: extent)
+ else {
+ return downscaleBlur(frame)
+ }
+ return image
+ }
+
+ /// Blur without CoreImage: shrink the frame to a few pixels and scale it
+ /// back up, so the bands around the cover always come out soft even when
+ /// the gaussian filter refuses the image.
+ private static func downscaleBlur(_ frame: CGImage) -> CGImage? {
+ let width = 16
+ let height = max(1, Int((CGFloat(width) * CGFloat(frame.height) / CGFloat(max(1, frame.width))).rounded()))
+ guard let context = CGContext(
+ data: nil,
+ width: width,
+ height: height,
+ bitsPerComponent: 8,
+ bytesPerRow: 0,
+ space: CGColorSpaceCreateDeviceRGB(),
+ bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
+ ) else {
+ return nil
+ }
+ context.interpolationQuality = .high
+ context.draw(frame, in: CGRect(x: 0, y: 0, width: CGFloat(width), height: CGFloat(height)))
+ return context.makeImage()
+ }
+
+ private static func aspectFill(_ size: CGSize, in bounds: CGRect) -> CGRect {
+ guard size.width > 0, size.height > 0 else {
+ return bounds
+ }
+ let scale = max(bounds.width / size.width, bounds.height / size.height)
+ let width = size.width * scale
+ let height = size.height * scale
+ return CGRect(x: bounds.midX - width / 2, y: bounds.midY - height / 2, width: width, height: height)
+ }
+
+ private func videoURL() async -> URL? {
+ if FileManager.default.fileExists(atPath: destination.path) {
+ return destination
+ }
+ return await encodeTaskValue().value
+ }
+
+ private func encodeTaskValue() -> Task<URL?, Never> {
+ lock.lock()
+ if let encodeTask {
+ lock.unlock()
+ return encodeTask
+ }
+ lock.unlock()
+ // One task per destination across instances: a second instance holding
+ // the same clip joins the encode already running rather than starting a
+ // rival writer at the same path.
+ Self.sharedTaskLock.lock()
+ if let shared = Self.sharedTasks[destination.path] {
+ Self.sharedTaskLock.unlock()
+ lock.lock()
+ encodeTask = shared
+ lock.unlock()
+ return shared
+ }
+ // `.userInitiated`: the lock screen is waiting on this file, often with
+ // the phone locked, where `.utility` would crawl.
+ let created = Task.detached(priority: .userInitiated) {
+ await self.encode()
+ }
+ Self.sharedTasks[destination.path] = created
+ Self.sharedTaskLock.unlock()
+ lock.lock()
+ encodeTask = created
+ lock.unlock()
+ let path = destination.path
+ Task {
+ _ = await created.value
+ Self.sharedTaskLock.lock()
+ Self.sharedTasks.removeValue(forKey: path)
+ Self.sharedTaskLock.unlock()
+ }
+ return created
+ }
+
+ private func encode() async -> URL? {
+ defer {
+ lock.lock()
+ data = nil
+ lock.unlock()
+ }
+ if FileManager.default.fileExists(atPath: destination.path) {
+ return destination
+ }
+ lock.lock()
+ let data = self.data
+ lock.unlock()
+ guard let data, let source = CGImageSourceCreateWithData(data as CFData, nil) else {
+ return nil
+ }
+ guard let schedule = Self.schedule(source: source, count: CGImageSourceGetCount(source)) else {
+ return nil
+ }
+ return await write(source: source, schedule: schedule)
+ }
+
+ private func write(source: CGImageSource, schedule: Schedule) async -> URL? {
+ let fileManager = FileManager.default
+ guard !fileManager.fileExists(atPath: destination.path) else {
+ return destination
+ }
+ let directory = destination.deletingLastPathComponent()
+ try? fileManager.createDirectory(at: directory, withIntermediateDirectories: true)
+ let temporary = directory.appendingPathComponent("tmp-" + UUID().uuidString + ".mp4")
+ defer {
+ if fileManager.fileExists(atPath: temporary.path) {
+ try? fileManager.removeItem(at: temporary)
+ }
+ }
+
+ let canvas = self.canvas
+ let width = Int(canvas.width)
+ let height = Int(canvas.height)
+ // What the rate controller is told to expect: without it the encoder
+ // budgets for a frame rate that has nothing to do with the samples being
+ // appended, and spends the bits it has on the wrong thing.
+ let outputRate = Double(schedule.samples.count) / max(schedule.total, 0.001)
+ let expectedFrameRate = Int(min(Self.maxPlaybackRate, max(1, outputRate.rounded())))
+ let settings: [String: Any] = [
+ AVVideoCodecKey: AVVideoCodecType.h264,
+ AVVideoWidthKey: width,
+ AVVideoHeightKey: height,
+ AVVideoCompressionPropertiesKey: [
+ AVVideoAverageBitRateKey: width * height * 2,
+ AVVideoMaxKeyFrameIntervalKey: 30,
+ AVVideoExpectedSourceFrameRateKey: expectedFrameRate,
+ AVVideoAllowFrameReorderingKey: false,
+ ] as [String: Any],
+ ]
+ guard let writer = try? AVAssetWriter(outputURL: temporary, fileType: .mp4) else {
+ return nil
+ }
+ let input = AVAssetWriterInput(mediaType: .video, outputSettings: settings)
+ input.expectsMediaDataInRealTime = false
+ guard writer.canAdd(input) else {
+ return nil
+ }
+ writer.add(input)
+ let adaptor = AVAssetWriterInputPixelBufferAdaptor(
+ assetWriterInput: input,
+ sourcePixelBufferAttributes: [
+ kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
+ kCVPixelBufferWidthKey as String: width,
+ kCVPixelBufferHeightKey as String: height,
+ kCVPixelBufferCGImageCompatibilityKey as String: true,
+ kCVPixelBufferCGBitmapContextCompatibilityKey as String: true,
+ ]
+ )
+ guard writer.startWriting() else {
+ return nil
+ }
+ writer.startSession(atSourceTime: .zero)
+ defer {
+ if writer.status == .writing {
+ writer.cancelWriting()
+ }
+ }
+
+ // Both composed once per clip: the blurred backdrop scaled to the canvas
+ // (every frame then draws it as a blit, with no aspect-fill math and no
+ // high-quality scale per frame), and the feather mask that trims the
+ // cover's top and bottom edges.
+ let backdrop: CGImage?
+ let mask: CGImage?
+ if ratio == .tall,
+ let first = CGImageSourceCreateImageAtIndex(source, 0, Self.frameOptions),
+ let blurred = Self.blurred(first) {
+ backdrop = Self.sized(blurred, to: canvas)
+ mask = Self.featherMask(size: Self.artRect(in: canvas).size)
+ } else {
+ backdrop = nil
+ mask = nil
+ }
+
+ for sample in schedule.samples {
+ guard let frame = CGImageSourceCreateImageAtIndex(source, sample.index, Self.frameOptions),
+ let buffer = pixelBuffer(adaptor: adaptor, frame: frame, canvas: canvas, backdrop: backdrop, mask: mask),
+ await isReady(input, writer: writer),
+ adaptor.append(buffer, withPresentationTime: CMTime(seconds: sample.time, preferredTimescale: 600))
+ else {
+ return nil
+ }
+ }
+
+ guard let lastIndex = schedule.samples.last?.index,
+ let lastFrame = CGImageSourceCreateImageAtIndex(source, lastIndex, Self.frameOptions),
+ let lastBuffer = pixelBuffer(adaptor: adaptor, frame: lastFrame, canvas: canvas, backdrop: backdrop, mask: mask),
+ await isReady(input, writer: writer)
+ else {
+ return nil
+ }
+ guard adaptor.append(lastBuffer, withPresentationTime: CMTime(seconds: schedule.total, preferredTimescale: 600)) else {
+ return nil
+ }
+
+ input.markAsFinished()
+ await withCheckedContinuation { (continuation: CheckedContinuation<Void, Never>) in
+ writer.finishWriting {
+ continuation.resume()
+ }
+ }
+ guard writer.status == .completed else {
+ return nil
+ }
+
+ do {
+ try fileManager.moveItem(at: temporary, to: destination)
+ } catch {
+ return fileManager.fileExists(atPath: destination.path) ? destination : nil
+ }
+ return destination
+ }
+
+ private func isReady(_ input: AVAssetWriterInput, writer: AVAssetWriter) async -> Bool {
+ while !input.isReadyForMoreMediaData {
+ if writer.status != .writing {
+ return false
+ }
+ try? await Task.sleep(nanoseconds: 4_000_000)
+ }
+ return true
+ }
+
+ private func pixelBuffer(
+ adaptor: AVAssetWriterInputPixelBufferAdaptor,
+ frame: CGImage,
+ canvas: CGSize,
+ backdrop: CGImage?,
+ mask: CGImage?
+ ) -> CVPixelBuffer? {
+ guard let pool = adaptor.pixelBufferPool else {
+ return nil
+ }
+ var created: CVPixelBuffer?
+ guard CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool, &created) == kCVReturnSuccess,
+ let buffer = created
+ else {
+ return nil
+ }
+ CVPixelBufferLockBaseAddress(buffer, [])
+ defer { CVPixelBufferUnlockBaseAddress(buffer, []) }
+ guard let context = CGContext(
+ data: CVPixelBufferGetBaseAddress(buffer),
+ width: Int(canvas.width),
+ height: Int(canvas.height),
+ bitsPerComponent: 8,
+ bytesPerRow: CVPixelBufferGetBytesPerRow(buffer),
+ space: Self.deviceRGB,
+ bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue
+ ) else {
+ return nil
+ }
+ draw(frame, in: context, canvas: canvas, backdrop: backdrop, mask: mask)
+ return buffer
+ }
+
+ private static func schedule(source: CGImageSource, count: Int) -> Schedule? {
+ let delays = (0..<count).map { frameDelay(at: $0, source: source) }
+ let span = delays.reduce(0, +)
+ // Frames are dropped for one of two reasons and no other: the clip runs
+ // faster than the system will show it, or there are more frames than the
+ // encoder should carry. Counting alone was a third reason, and it was the
+ // one that showed: a cover long enough to fill the 29 seconds at 30 fps is
+ // 870 frames, which came out as every third one, so the lock screen showed
+ // 10 fps of a clip the app beside it was playing at 30. Length is not a
+ // reason to drop frames, only to cut the tail, which the clamp below does.
+ var step = 1
+ if span > 0 {
+ step = max(step, Int(((Double(count) / span) / maxPlaybackRate).rounded(.up)))
+ }
+ step = max(step, Int((Double(count) / Double(maxFrames)).rounded(.up)))
+ var samples: [Sample] = []
+ var time = 0.0
+ var index = 0
+ while index < count {
+ var delay = 0.0
+ var cursor = index
+ let end = min(index + step, count)
+ while cursor < end {
+ delay += delays[cursor]
+ cursor += 1
+ }
+ if time + delay > maxDuration {
+ delay = maxDuration - time
+ }
+ guard delay > 0 else {
+ break
+ }
+ samples.append(Sample(index: index, time: time))
+ time += delay
+ if time >= maxDuration {
+ break
+ }
+ index += step
+ }
+ guard !samples.isEmpty else {
+ return nil
+ }
+ return Schedule(samples: samples, total: time)
+ }
+
+ private static func frameDelay(at index: Int, source: CGImageSource) -> Double {
+ guard let properties = CGImageSourceCopyPropertiesAtIndex(source, index, nil) as? [String: Any] else {
+ return fallbackDelay
+ }
+ // The per-frame delays live in a format sub-dictionary whose key is
+ // "{WebP}"/"{GIF}"/"{PNG}", not "WebP": looking for the unbraced name
+ // never matched, so every frame fell back to 100ms and the cover crawled
+ // at ten frames per second. Any sub-dictionary carrying a delay wins.
+ for value in properties.values {
+ guard let dictionary = value as? [String: Any] else {
+ continue
+ }
+ let unclamped = (dictionary["UnclampedDelayTime"] as? NSNumber)?.doubleValue ?? 0
+ let clamped = (dictionary["DelayTime"] as? NSNumber)?.doubleValue ?? 0
+ let delay = unclamped > 0 ? unclamped : clamped
+ guard delay > 0 else {
+ continue
+ }
+ return delay < browserDelayFloor ? fallbackDelay : max(delay, minFrameDelay)
+ }
+ return fallbackDelay
+ }
+
+ private static func evenSide(longest: Int) -> Int {
+ let capped = min(maxSide, longest)
+ return max(2, capped - (capped % 2))
+ }
+
+ private static func identifier(for url: URL) -> String {
+ SHA256.hash(data: Data(url.absoluteString.utf8))
+ .map { String(format: "%02x", $0) }
+ .joined()
+ }
+
+ private static func cacheFile(id: String, ratio: Ratio) -> URL? {
+ guard let base = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first else {
+ return nil
+ }
+ return base
+ .appendingPathComponent("animated-artwork", isDirectory: true)
+ .appendingPathComponent(id + "-" + ratio.suffix + ".mp4")
+ }
+}
+
+/// Tall HLS clip from the animated-artwork endpoint: downloaded once for
+/// offline playback — the system only plays local video — then handed out
+/// like an encoded clip. One download per artwork URL, shared by whoever
+/// asks while it runs, and reused from disk after. Nil when the stream
+/// cannot be read: the caller falls back to the static artwork, which is
+/// always there.
+@available(iOS 26.0, tvOS 26.0, *)
+final class StreamArtworkClip {
+ private static let lock = NSLock()
+ private static var inflight: [String: Task<MPMediaItemAnimatedArtwork?, Never>] = [:]
+
+ /// What the last export did, for the diagnostics screen: the stage and one
+ /// line of detail. Written from background tasks, read from anywhere.
+ private static var lastStage = "idle"
+ private static var lastDetail = ""
+
+ static func state() -> [String: String] {
+ lock.lock()
+ defer { lock.unlock() }
+ return ["stage": lastStage, "detail": lastDetail]
+ }
+
+ private static func setStage(_ stage: String, _ detail: String = "") {
+ lock.lock()
+ lastStage = stage
+ lastDetail = detail
+ lock.unlock()
+ }
+
+ private static func identifier(for url: URL) -> String {
+ SHA256.hash(data: Data(url.absoluteString.utf8))
+ .map { String(format: "%02x", $0) }
+ .joined()
+ }
+
+ private static func destination(id: String) -> URL? {
+ guard let base = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first else {
+ return nil
+ }
+ return base
+ .appendingPathComponent("animated-artwork", isDirectory: true)
+ .appendingPathComponent(id + "-stream-3x4.mp4")
+ }
+
+ /// A cover stream is one file behind two playlists: the master lists the
+ /// variants, the media one the file. Anything else (segments per scene) is
+ /// a real stream, not a cover clip, and is left alone.
+ private static let widestStream = 1100
+
+ private static func playlistLines(_ text: String) -> [String] {
+ text.components(separatedBy: .newlines)
+ .map { $0.trimmingCharacters(in: .whitespaces) }
+ .filter { !$0.isEmpty }
+ }
+
+ private static func streamAttributes(_ tag: String) -> [String: String] {
+ var out: [String: String] = [:]
+ guard let regex = try? NSRegularExpression(pattern: "([A-Z0-9-]+)=(\"[^\"]*\"|[^,]*)") else {
+ return out
+ }
+ let range = NSRange(tag.startIndex..., in: tag)
+ for match in regex.matches(in: tag, range: range) {
+ if let key = Range(match.range(at: 1), in: tag),
+ let value = Range(match.range(at: 2), in: tag) {
+ out[String(tag[key])] = String(tag[value]).trimmingCharacters(in: CharacterSet(charactersIn: "\""))
+ }
+ }
+ return out
+ }
+
+ private static func pickStream(master: String, base: URL) -> URL? {
+ let lines = playlistLines(master)
+ var best: URL?
+ var bestHEVC = false
+ var bestArea = 0
+ var bestRate = 0
+ var index = 0
+ while index + 1 < lines.count {
+ defer { index += 1 }
+ let tag = lines[index]
+ let uri = lines[index + 1]
+ guard tag.hasPrefix("#EXT-X-STREAM-INF:"), !uri.hasPrefix("#") else { continue }
+ let attributes = streamAttributes(tag)
+ let size = (attributes["RESOLUTION"] ?? "").split(separator: "x").compactMap { Int($0) }
+ guard size.count == 2, size[0] <= widestStream else { continue }
+ let codecs = attributes["CODECS"] ?? ""
+ let hevc = codecs.hasPrefix("hvc1") || codecs.hasPrefix("hev1")
+ let area = size[0] * size[1]
+ let rate = Int(attributes["AVERAGE-BANDWIDTH"] ?? attributes["BANDWIDTH"] ?? "") ?? 0
+ let better = best == nil || (hevc && !bestHEVC) ||
+ (hevc == bestHEVC && (area > bestArea || (area == bestArea && rate < bestRate)))
+ if !better { continue }
+ guard let url = URL(string: uri, relativeTo: base)?.absoluteURL else { continue }
+ best = url
+ bestHEVC = hevc
+ bestArea = area
+ bestRate = rate
+ }
+ return best
+ }
+
+ private static func singleFile(media: String, base: URL) -> URL? {
+ var file: String?
+ for line in playlistLines(media) {
+ let uri: String?
+ if line.hasPrefix("#EXT-X-MAP:") {
+ uri = streamAttributes(line)["URI"]
+ } else if line.hasPrefix("#") {
+ uri = nil
+ } else {
+ uri = line
+ }
+ guard let uri else { continue }
+ if let file, file != uri { return nil }
+ file = uri
+ }
+ guard let file else { return nil }
+ return URL(string: file, relativeTo: base)?.absoluteURL
+ }
+
+ private static func downloadText(_ url: URL) async -> String? {
+ guard let (data, response) = try? await URLSession.shared.data(from: url),
+ (response as? HTTPURLResponse)?.statusCode == 200,
+ let text = String(data: data, encoding: .utf8) else {
+ return nil
+ }
+ return text
+ }
+
+ private static func downloadFile(_ url: URL, to destination: URL) async -> Bool {
+ guard let (data, response) = try? await URLSession.shared.data(from: url),
+ (response as? HTTPURLResponse)?.statusCode == 200,
+ !data.isEmpty else {
+ return false
+ }
+ do {
+ try? FileManager.default.removeItem(at: destination)
+ try data.write(to: destination, options: .atomic)
+ return true
+ } catch {
+ return false
+ }
+ }
+
+ static func item(m3u8: URL) -> Task<MPMediaItemAnimatedArtwork?, Never> {
+ let id = identifier(for: m3u8)
+ lock.lock()
+ if let running = inflight[id] {
+ lock.unlock()
+ return running
+ }
+ let task = Task<MPMediaItemAnimatedArtwork?, Never> {
+ defer {
+ lock.lock()
+ inflight.removeValue(forKey: id)
+ lock.unlock()
+ }
+ return await exportWithTimeout(m3u8: m3u8, id: id)
+ }
+ inflight[id] = task
+ lock.unlock()
+ return task
+ }
+
+ /// The export, bounded: a hanging HLS load must turn into a diagnosed
+ /// timeout, not a task that never answers and a lock screen that never
+ /// animates.
+ private static func exportWithTimeout(m3u8: URL, id: String) async -> MPMediaItemAnimatedArtwork? {
+ await withTaskGroup(
+ of: MPMediaItemAnimatedArtwork?.self,
+ returning: MPMediaItemAnimatedArtwork?.self
+ ) { group in
+ group.addTask { await Self.export(m3u8: m3u8, id: id) }
+ group.addTask {
+ try? await Task.sleep(nanoseconds: 45_000_000_000)
+ // Cancelled because the export already answered: leave its stage.
+ if Task.isCancelled { return nil }
+ Self.setStage("timeout", m3u8.host ?? "")
+ return nil
+ }
+ for await result in group {
+ group.cancelAll()
+ return result
+ }
+ return nil
+ }
+ }
+
+ private static func export(m3u8: URL, id: String) async -> MPMediaItemAnimatedArtwork? {
+ guard let destination = destination(id: id) else {
+ setStage("failed", "no-cache-dir")
+ return nil
+ }
+ if !FileManager.default.fileExists(atPath: destination.path) {
+ setStage("downloading", m3u8.host ?? "")
+ try? FileManager.default.createDirectory(
+ at: destination.deletingLastPathComponent(),
+ withIntermediateDirectories: true
+ )
+ // The download keeps running after the app backgrounds: without time
+ // the system suspends it mid-file and the clip never lands.
+ var backgroundTask: UIBackgroundTaskIdentifier = .invalid
+ backgroundTask = UIApplication.shared.beginBackgroundTask(withName: "stream-clip-download") {}
+ defer {
+ if backgroundTask != .invalid {
+ UIApplication.shared.endBackgroundTask(backgroundTask)
+ }
+ }
+ guard let master = await downloadText(m3u8) else {
+ setStage("failed", "master")
+ return nil
+ }
+ guard let stream = pickStream(master: master, base: m3u8) else {
+ setStage("failed", "stream")
+ return nil
+ }
+ guard let media = await downloadText(stream) else {
+ setStage("failed", "media")
+ return nil
+ }
+ guard let file = singleFile(media: media, base: stream),
+ await downloadFile(file, to: destination) else {
+ setStage("failed", "file")
+ return nil
+ }
+ } else {
+ setStage("cached", "")
+ }
+ // The preview is cut from the downloaded clip itself: local, fast, and
+ // the exact frames the lock screen plays, so the two cannot drift apart.
+ // It is composed to the tall shape exactly: a still more than a hair off
+ // makes the system drop the animated artwork.
+ guard let preview = previewComposed(file: destination) else {
+ setStage("failed", "preview")
+ return nil
+ }
+ setStage("ready", "")
+ return MPMediaItemAnimatedArtwork(
+ artworkID: id + "-stream-3x4",
+ previewImageRequestHandler: { _ in preview },
+ videoAssetFileURLRequestHandler: { _ in destination }
+ )
+ }
+
+ private static func previewComposed(file: URL) -> UIImage? {
+ let generator = AVAssetImageGenerator(asset: AVURLAsset(url: file))
+ generator.appliesPreferredTrackTransform = true
+ guard let frame = try? generator.copyCGImage(
+ at: CMTime(seconds: 0.5, preferredTimescale: 600),
+ actualTime: nil
+ ) else {
+ return nil
+ }
+ let image = UIImage(cgImage: frame)
+ guard image.size.width > 0, image.size.height > 0 else {
+ return nil
+ }
+ let size = CGSize(width: 900, height: 1200)
+ let format = UIGraphicsImageRendererFormat.preferred()
+ format.scale = 1
+ format.opaque = true
+ return UIGraphicsImageRenderer(size: size, format: format).image { _ in
+ let scale = max(size.width / image.size.width, size.height / image.size.height)
+ let drawn = CGSize(width: image.size.width * scale, height: image.size.height * scale)
+ image.draw(in: CGRect(
+ x: (size.width - drawn.width) / 2,
+ y: (size.height - drawn.height) / 2,
+ width: drawn.width,
+ height: drawn.height
+ ))
+ }
+ }
+
+ private static func previewFirstFrame(file: URL) -> UIImage? {
+ let generator = AVAssetImageGenerator(asset: AVURLAsset(url: file))
+ generator.appliesPreferredTrackTransform = true
+ generator.maximumSize = CGSize(width: 768, height: 1024)
+ guard let image = try? generator.copyCGImage(
+ at: CMTime(seconds: 0.5, preferredTimescale: 600),
+ actualTime: nil
+ ) else {
+ return nil
+ }
+ return UIImage(cgImage: image)
+ }
+}
+
+#endif
diff --git a/ios/ArtworkPreloader.swift b/ios/ArtworkPreloader.swift
new file mode 100644
index 0000000000000000000000000000000000000000..324f3800712d70577a341b40ed49666bd6bf993e
--- /dev/null
+++ b/ios/ArtworkPreloader.swift
@@ -0,0 +1,103 @@
+import Foundation
+
+/// The lock screen's cover, fetched ahead of the track it belongs to: a
+/// download, and an encode behind it when the cover is animated, both better
+/// spent while the previous track still plays. `AudioPlayer.setNextSource`
+/// warms the track queued behind the current one — the same idea as Android's
+/// `PreloadingBitmapLoader` — and `MediaController.loadArtworkFromURL` asks
+/// here first, so a cover is fetched once for every caller that wants it, and
+/// the second ask of one has nothing left to wait for.
+final class ArtworkPreloader {
+ static let shared = ArtworkPreloader()
+
+ /// A cover, fetched: its bytes, and the clips cut from them for the lock
+ /// screen (nil when the cover does not animate, or the system is too old
+ /// to play an animated one).
+ struct Ready {
+ let data: Data
+ let square: AnyObject?
+ let tall: AnyObject?
+ }
+
+ private let lock = NSLock()
+ /// What was fetched, last write last, bounded by `limit`: a cover's bytes
+ /// run to a few megabytes when it animates, so only the handful that a skip
+ /// or a media reset asks for again is worth keeping.
+ private var stored: [URL: Ready] = [:]
+ private var order: [URL] = []
+ private var waiters: [URL: [(Ready?) -> Void]] = [:]
+ private var inflight: Set<URL> = []
+ private let delivery = DispatchQueue(label: "expo.modules.audio.artwork-preloader")
+ private let limit = 6
+
+ /// Nothing wants an answer: the point is only to have the cover later.
+ func warm(_ url: URL?) {
+ guard let url else { return }
+ fetch(url) { _ in }
+ }
+
+ /// The cover for `url`: here already, or on its way, or the fetch this call
+ /// starts. `completion` runs off the main thread, as the URL session's queue
+ /// did before it — the caller hops after it.
+ func fetch(_ url: URL, completion: @escaping (Ready?) -> Void) {
+ lock.lock()
+ if let ready = stored[url] {
+ lock.unlock()
+ delivery.async { completion(ready) }
+ return
+ }
+ waiters[url, default: []].append(completion)
+ if inflight.contains(url) {
+ lock.unlock()
+ return
+ }
+ inflight.insert(url)
+ lock.unlock()
+ start(url)
+ }
+
+ private func start(_ url: URL) {
+ URLSession.shared.dataTask(with: url) { [weak self] data, _, error in
+ guard let self else { return }
+ guard let data, error == nil else {
+ self.done(url, nil)
+ return
+ }
+ var square: AnyObject?
+ var tall: AnyObject?
+#if compiler(>=6.2)
+ if #available(iOS 26.0, tvOS 26.0, *) {
+ let clips = AnimatedArtwork.make(url: url, data: data)
+ square = clips.square
+ tall = clips.tall
+ // The encode starts now, while the track these clips belong to is
+ // still the next one: the file it writes is the lock screen's own
+ // cache, so a clip that lands before the move is there the moment
+ // the system asks for it.
+ (square as? AnimatedArtwork)?.whenReady {}
+ (tall as? AnimatedArtwork)?.whenReady {}
+ }
+#endif
+ self.done(url, Ready(data: data, square: square, tall: tall))
+ }.resume()
+ }
+
+ private func done(_ url: URL, _ ready: Ready?) {
+ lock.lock()
+ inflight.remove(url)
+ let pending = waiters.removeValue(forKey: url) ?? []
+ if let ready {
+ stored[url] = ready
+ order.removeAll { $0 == url }
+ order.append(url)
+ while order.count > limit, let oldest = order.first {
+ order.removeFirst()
+ stored.removeValue(forKey: oldest)
+ }
+ }
+ lock.unlock()
+ delivery.async {
+ pending.forEach { $0(ready) }
+ }
+ }
+}
diff --git a/ios/AudioModule.swift b/ios/AudioModule.swift
index d26560627c2dfa97370440ad28304617f3f9f0f7..31a715beb88474b4029784074070a0c11d8ffc01 100644
--- a/ios/AudioModule.swift
+++ b/ios/AudioModule.swift
@@ -32,6 +32,49 @@ public class AudioModule: Module {
try setAudioMode(mode: mode)
}
+ /// The lock screen's cover for `url`, fetched and its clip encoded with no
+ /// track attached to it: how the queue ahead is warmed, covers of tracks
+ /// the player has not been pointed at yet. (`setNextSource` warms the one
+ /// queued behind the current, `replaceCurrentSource` the one loading.)
+ Function("warmArtwork") { (urlString: String?) in
+ guard let urlString, let url = URL(string: urlString) else {
+ return
+ }
+ ArtworkPreloader.shared.warm(url)
+ }
+
+ /// The fps the covers encode at from here on - the app's Settings menu
+ /// writes it (30 by default, 60 on request).
+ Function("setArtworkFps") { (fps: Double) in
+ if #available(iOS 26.0, tvOS 26.0, *) {
+ AnimatedArtwork.maxPlaybackRate = fps
+ }
+ }
+
+ /// Tall HLS clip from the animated-artwork endpoint for one static cover:
+ /// published to the lock screen under it, or cleared with a null clip.
+ /// The static cover URL keys it, so a clip arriving late still lands on
+ /// the track it was fetched for — and never on the next one.
+ Function("setAnimatedClipUrls") { (artworkUrlString: String?, tallUrlString: String?) in
+ guard let artworkUrlString, let artworkUrl = URL(string: artworkUrlString) else {
+ return
+ }
+ let tallUrl = tallUrlString.flatMap { URL(string: $0) }
+ MediaController.shared.setStreamClip(tallUrl: tallUrl, for: artworkUrl)
+ }
+
+ /// What the last stream-clip export did, for the diagnostics screen.
+ Function("animatedClipState") { () -> [String: String] in
+#if compiler(>=6.2)
+ if #available(iOS 26.0, tvOS 26.0, *) {
+ return StreamArtworkClip.state()
+ }
+ return [:]
+#else
+ return [:]
+#endif
+ }
+
AsyncFunction("setIsAudioActiveAsync") { (isActive: Bool) in
try setIsAudioActive(isActive)
}
@@ -241,6 +284,18 @@ public class AudioModule: Module {
}
}
+ // Gapless: same source resolution as `replace`, but the track is queued
+ // behind the one playing instead of taking its place, so AVQueuePlayer
+ // buffers it in advance and joins them with no gap. The move arrives as
+ // `trackTransition`.
+ Function("setNextSource") { (player: AudioPlayer, source: AudioSource) in
+ player.setNextSource(source)
+ }
+
+ Function("clearNextSource") { (player: AudioPlayer) in
+ player.clearNextSource()
+ }
+
Function("pause") { player in
player.ref.pause()
if !player.keepAudioSessionActive {
diff --git a/ios/AudioPlayer.swift b/ios/AudioPlayer.swift
index 074294321e872bb53c0b2d41044dac3c612f14f0..10a9ec0ce086f81072bdbc6ded1aadae41fc687d 100644
--- a/ios/AudioPlayer.swift
+++ b/ios/AudioPlayer.swift
@@ -4,6 +4,7 @@ import Combine
private enum AudioConstants {
static let playbackStatus = "playbackStatusUpdate"
static let audioSample = "audioSampleUpdate"
+ static let trackTransition = "trackTransition"
}
public class AudioPlayer: SharedRef<AVPlayer>, Playable {
@@ -25,17 +26,40 @@ public class AudioPlayer: SharedRef<AVPlayer>, Playable {
var samplingEnabled = false
var keepAudioSessionActive = false
+ /**
+ * Gapless: the track queued behind the current one by `setNextSource`, if
+ * any. It is in the player's queue already, this is only the handle needed
+ * to tell an advance to it apart from a source JS installed itself.
+ */
+ private var gaplessItem: AVPlayerItem?
+ /** The same track as the source it was queued from, so a reset of the media
+ * services can rebuild what is playing after the player moved to it. */
+ private var gaplessSource: AudioSource?
+
+ /**
+ * What the player does when the current item runs out: move on to the next
+ * one in its queue when there is one waiting (a loop, or the track gapless
+ * queued behind), and pause when there is not — which is what makes the end
+ * of a track reach JS as `didJustFinish`.
+ */
+ private var endAction: AVPlayer.ActionAtItemEnd {
+ isLooping || gaplessItem != nil ? .advance : .pause
+ }
+
var isLooping = false {
didSet {
guard isLooping != oldValue else {
return
}
if isLooping {
- ref.actionAtItemEnd = .advance
+ // The loop is the repeat of the song playing, so what gapless had
+ // queued behind it is not what comes next any more — and the loop's
+ // own copy cannot be added while another item waits behind.
+ clearNextSource()
enqueueNextLoopItem()
} else {
removeQueuedLoopItems()
- ref.actionAtItemEnd = .pause
+ ref.actionAtItemEnd = endAction
}
updateStatus(with: [:])
}
@@ -56,7 +80,21 @@ public class AudioPlayer: SharedRef<AVPlayer>, Playable {
var duration: Double {
let seconds = ref.currentItem?.duration.seconds ?? 0.0
- return seconds.isNaN ? 0.0 : seconds
+ if seconds.isFinite && seconds > 0 { return seconds }
+ // AVPlayer plays a stream it cannot time (nothing in the container or the
+ // response to compute a length from) and then reports 0. The app reads 0
+ // as "this stream has no random access" and seeks it by re-requesting with
+ // `timeOffset` — a parameter the server IGNORES on a file it serves
+ // directly (it is only ever given to ffmpeg), so playback restarts at zero
+ // while the position keeps counting from the seek: the time then runs
+ // ahead of the sound by every seek, and reaches the end while the audio
+ // still has that much left. The caller's length stands in, the same way it
+ // already does on the lock screen (`MediaController.applyPlaybackInfo`),
+ // so a playable stream always reports the length it has.
+ if let ms = metadata?.durationMs, ms > 0 {
+ return ms / 1000.0
+ }
+ return 0.0
}
var currentTime: Double {
@@ -100,7 +138,9 @@ public class AudioPlayer: SharedRef<AVPlayer>, Playable {
}
func play(at rate: Float) {
- ref.actionAtItemEnd = isLooping ? .advance : .pause
+ // Whatever was queued behind the current item stays queued: a pause and
+ // resume in the middle of a song must not take the advance away from it.
+ ref.actionAtItemEnd = endAction
if isLooping {
enqueueNextLoopItem()
}
@@ -176,14 +216,19 @@ public class AudioPlayer: SharedRef<AVPlayer>, Playable {
}
}
+ // Exact landing by default: AVPlayer's fast seek (±infinity) may stop
+ // anywhere, and on a container with no sample index — mp3, ogg/opus, ADTS —
+ // "anywhere" is seconds away from what was asked, so the audio plays from a
+ // different place than the position reported for it. Skip and the lock
+ // screen's own command already seek with `.zero`.
func seekTo(seconds: Double, toleranceMillisBefore: Double? = nil, toleranceMillisAfter: Double? = nil) async {
let time = CMTime(seconds: seconds, preferredTimescale: CMTimeScale(NSEC_PER_SEC))
let toleranceBefore = toleranceMillisBefore.map {
CMTime(seconds: $0 / 1000.0, preferredTimescale: CMTimeScale(NSEC_PER_SEC))
- } ?? CMTime.positiveInfinity
+ } ?? CMTime.zero
let toleranceAfter = toleranceMillisAfter.map {
CMTime(seconds: $0 / 1000.0, preferredTimescale: CMTimeScale(NSEC_PER_SEC))
- } ?? CMTime.positiveInfinity
+ } ?? CMTime.zero
await withCheckedContinuation { (continuation: CheckedContinuation<Void, Never>) in
ref.seek(to: time, toleranceBefore: toleranceBefore, toleranceAfter: toleranceAfter) { [weak self] _ in
@@ -224,10 +269,26 @@ public class AudioPlayer: SharedRef<AVPlayer>, Playable {
.store(in: &cancellables)
ref.publisher(for: \.currentItem)
- .sink { [weak self] _ in
+ .sink { [weak self] item in
guard let self else {
return
}
+ // Gapless: the player moved on its own to the track queued by
+ // `setNextSource`, and it is already playing it. Nothing is loaded
+ // here — there is nothing to load — this is the announcement JS needs
+ // to follow the queue, tell the lock screen what is on now and queue
+ // the track after this one. A source JS installed itself never reads
+ // as a move of this kind: the handle is dropped before the item is
+ // swapped (`replacePlayerItem`).
+ if let queued = self.gaplessItem, item === queued {
+ // What is loaded from here on: a reset of the media services has to
+ // rebuild this track, not the one it left.
+ self.source = self.gaplessSource
+ self.gaplessItem = nil
+ self.gaplessSource = nil
+ self.ref.actionAtItemEnd = self.endAction
+ self.emit(event: AudioConstants.trackTransition)
+ }
if self.isLooping {
self.enqueueNextLoopItem()
self.addPlaybackEndNotification()
@@ -262,6 +323,10 @@ public class AudioPlayer: SharedRef<AVPlayer>, Playable {
func replaceCurrentSource(source: AudioSource) {
self.source = source
+ // Its cover too: the lock screen asks the moment a source lands, and a
+ // track loaded by hand never passed through the warm `setNextSource` does
+ // — a tap in a list, a crossfade's reserve player, a resume from pause.
+ ArtworkPreloader.shared.warm(source.metadata?.artworkUrl)
let wasPlaying = ref.timeControlStatus == .playing
let wasSamplingEnabled = samplingEnabled
removeQueuedLoopItems()
@@ -308,6 +373,7 @@ public class AudioPlayer: SharedRef<AVPlayer>, Playable {
}
private func replacePlayer(with source: AudioSource, restoreSampling: Bool) {
+ MediaController.shared.removeSession(for: ref)
teardownPlayer()
ref = AudioUtils.createAVPlayer(from: source)
@@ -319,6 +385,7 @@ public class AudioPlayer: SharedRef<AVPlayer>, Playable {
}
private func teardownPlayer() {
+ clearNextSource()
removeQueuedLoopItems()
if samplingEnabled {
uninstallTap()
@@ -346,7 +413,7 @@ public class AudioPlayer: SharedRef<AVPlayer>, Playable {
private func restorePlaybackState(time: Double, shouldResume: Bool, rate: Float) {
let cmTime = CMTime(seconds: time, preferredTimescale: CMTimeScale(NSEC_PER_SEC))
- ref.seek(to: cmTime) { [weak self] _ in
+ ref.seek(to: cmTime, toleranceBefore: .zero, toleranceAfter: .zero) { [weak self] _ in
guard let self, shouldResume else { return }
self.play(at: rate)
}
@@ -405,6 +472,11 @@ public class AudioPlayer: SharedRef<AVPlayer>, Playable {
}
private func replacePlayerItem(with item: AVPlayerItem?) {
+ // Whatever was queued behind the old item goes with it: this one is the
+ // track now, and JS knows it (it called `replace`, and forgets its own
+ // memo beside that call).
+ gaplessItem = nil
+ gaplessSource = nil
if let queuePlayer = ref as? AVQueuePlayer {
queuePlayer.removeAllItems()
if let item {
@@ -429,12 +501,67 @@ public class AudioPlayer: SharedRef<AVPlayer>, Playable {
queuePlayer.insert(nextItem, after: currentItem)
}
+ /**
+ * Gapless: queues the track that comes after the current one, so the player
+ * buffers it and joins the two by itself instead of the queue (which lives
+ * in JS) asking for it only once the previous one has ended.
+ *
+ * One track waits behind the current one at most: what was queued before is
+ * dropped first, so a queue reordered while a song plays ends up with the
+ * right track waiting. The loop is not touched by any of this — it is the
+ * repeat of the current song, not a track of its own (see `clearNextSource`).
+ */
+ func setNextSource(_ source: AudioSource) {
+ // Nothing playing: there is nothing to queue BEHIND, and the track would
+ // become the current one without anyone having asked for it.
+ guard let queue = ref as? AVQueuePlayer, let current = queue.currentItem else {
+ return
+ }
+ // Whatever is behind the current item goes first, loop copies included:
+ // what comes next is this, and only this.
+ clearNextSource()
+ removeQueuedLoopItems()
+ guard let item = AudioUtils.createAVPlayerItem(from: source) else {
+ return
+ }
+ // What the item under it says: the rate is applied to the item the player
+ // knows about, and this one is about to become it.
+ item.audioTimePitchAlgorithm = current.audioTimePitchAlgorithm
+ queue.insert(item, after: current)
+ gaplessItem = item
+ gaplessSource = source
+ ref.actionAtItemEnd = endAction
+ // Its cover too, so the lock screen has it the moment the track starts:
+ // a download, and an encode behind it when the cover animates, both
+ // better spent now — while the track it belongs to is still the NEXT
+ // one — than in the first seconds of the track itself. `MediaController`
+ // reads what was fetched from the preloader, the same idea as Android's
+ // `PreloadingBitmapLoader` (#139).
+ ArtworkPreloader.shared.warm(source.metadata?.artworkUrl)
+ }
+
+ /** Drops the track queued behind the current one, if any. */
+ func clearNextSource() {
+ if let item = gaplessItem {
+ gaplessItem = nil
+ gaplessSource = nil
+ (ref as? AVQueuePlayer)?.remove(item)
+ }
+ ref.actionAtItemEnd = endAction
+ }
+
+ /**
+ * Drops what is queued behind the current item, except the track gapless
+ * put there: those are the copies a loop makes of the song playing, which
+ * exist only while it is on.
+ */
private func removeQueuedLoopItems() {
guard let queuePlayer = ref as? AVQueuePlayer else {
return
}
- for item in queuePlayer.items() where item != queuePlayer.currentItem {
+ let queued = queuePlayer.items()
+ for item in queued where item != queuePlayer.currentItem && item !== gaplessItem {
queuePlayer.remove(item)
}
}
@@ -459,7 +586,21 @@ public class AudioPlayer: SharedRef<AVPlayer>, Playable {
return
}
+ // Already moved on: what finished is not what is playing any more, and
+ // the player never stopped over it — a lap of a loop, or the track
+ // gapless queued behind, which announces the move itself.
+ if finishedItem !== queuePlayer.currentItem {
+ return
+ }
+
if !self.isLooping {
+ // Gapless: a track is waiting behind this one and the player is
+ // advancing to it by itself. Nothing ended as far as JS is concerned,
+ // the next song is already starting — so the end is not reported
+ // here, the move is (see `setupPublisher`).
+ if self.gaplessItem != nil, self.ref.actionAtItemEnd == .advance {
+ return
+ }
let currentTime = finishedItem.duration.seconds
self.updateStatus(with: [
"playing": false,
@@ -511,6 +652,7 @@ public class AudioPlayer: SharedRef<AVPlayer>, Playable {
MediaController.shared.setActivePlayer(nil)
}
+ MediaController.shared.removeSession(for: ref)
teardownPlayer()
audioProcessor?.invalidate()
diff --git a/ios/AudioRecords.swift b/ios/AudioRecords.swift
index 33db27c3b8fb40fa9db6659a0787c6b2b503d1ab..a5c41356f93d3d4a45bf14020af1cb1943731751 100644
--- a/ios/AudioRecords.swift
+++ b/ios/AudioRecords.swift
@@ -64,12 +64,17 @@ struct Metadata: Record {
@Field var artist: String?
@Field var albumTitle: String?
@Field var artworkUrl: URL?
+ // The track's length from the caller, for a stream that can't say (a server
+ // transcoding on the fly sends no length). Fallback only.
+ @Field var durationMs: Double?
}
struct LockScreenOptions: Record {
@Field var showSeekForward: Bool = false
@Field var showSeekBackward: Bool = false
@Field var isLiveStream: Bool? = false
+ @Field var showSkipPrevious: Bool = false
+ @Field var showSkipNext: Bool = false
}
enum BitRateStrategy: String, Enumerable {
diff --git a/ios/AudioSource.swift b/ios/AudioSource.swift
index 0c8c2fede4b079f6d71217bafe07b5de55438821..98b021e42f27fac73d103835779d65ec983c11b1 100644
--- a/ios/AudioSource.swift
+++ b/ios/AudioSource.swift
@@ -10,4 +10,13 @@ struct AudioSource: Record {
@Field
var name: String?
+
+ @Field
+ var preferPreciseTiming: Bool?
+
+ @Field
+ // The cover travels with the track so it can be fetched while the previous
+ // one is still playing (`setNextSource` warms it, `ArtworkPreloader` keeps
+ // it) — the lock screen then has it the moment this track starts.
+ var metadata: Metadata?
}
diff --git a/ios/AudioUtils.swift b/ios/AudioUtils.swift
index 79c7c0b256581a768c42a0291bf84bdff4c4b142..13bbad46de02ac1b42a05877e510f9055769a2a5 100644
--- a/ios/AudioUtils.swift
+++ b/ios/AudioUtils.swift
@@ -84,9 +84,23 @@ struct AudioUtils {
url
}
- var options: [String: Any]?
+ // AVPlayer without this flag seeks by "approximate random access by time":
+ // on a container with no sample index (mp3, ogg, adts — everything that is
+ // not mp4/m4a) it lands NEAR the requested position while the clock keeps
+ // counting from the requested one, so a dragged position leaves the bar and
+ // the sound a whole seek apart until the track is restarted. The flag makes
+ // AVFoundation resolve the exact frame instead, and a container that
+ // already carries an index (m4a/mp4) reports the same precise timing
+ // anyway, so it costs nothing there.
+ //
+ // `preferPreciseTiming: false` opts a source out of all that: a preview only
+ // has to land *near* where it jumped, and on a streamed mp3 the exact seek
+ // means scanning the file for an index before the first second plays.
+ var options: [String: Any] = [
+ AVURLAssetPreferPreciseDurationAndTimingKey: source.preferPreciseTiming ?? true
+ ]
if let headers = source.headers {
- options = ["AVURLAssetHTTPHeaderFieldsKey": headers]
+ options["AVURLAssetHTTPHeaderFieldsKey"] = headers
}
let asset = AVURLAsset(url: finalUrl, options: options)
diff --git a/ios/MediaController.swift b/ios/MediaController.swift
index e977f577cd2e16b194b56537ad76b423f40d2496..a3ea966cf083987ff78169e5c3edde173b9f6377 100644
--- a/ios/MediaController.swift
+++ b/ios/MediaController.swift
@@ -6,11 +6,76 @@ class MediaController {
static let shared = MediaController()
private var activePlayer: AudioPlayer?
- private var remoteCommandCenter = MPRemoteCommandCenter.shared()
- private var nowPlayingInfoCenter = MPNowPlayingInfoCenter.default()
+ private var activeSession: MPNowPlayingSession?
+ private var sessions: [ObjectIdentifier: MPNowPlayingSession] = [:]
+ private var remoteCommandTargets: [ObjectIdentifier: [Any]] = [:]
+
+ private var remoteCommandCenter: MPRemoteCommandCenter? {
+ activeSession?.remoteCommandCenter
+ }
+
+ private var nowPlayingInfoCenter: MPNowPlayingInfoCenter? {
+ activeSession?.nowPlayingInfoCenter
+ }
+
+ private func session(for player: AudioPlayer) -> MPNowPlayingSession {
+ let key = ObjectIdentifier(player.ref)
+ if let session = sessions[key] {
+ return session
+ }
+
+ let session = MPNowPlayingSession(players: [player.ref])
+ session.automaticallyPublishesNowPlayingInfo = false
+ sessions[key] = session
+ return session
+ }
+
+ private func activateSession(for player: AudioPlayer, completion: @escaping (Bool) -> Void) {
+ let session = session(for: player)
+ activeSession = session
+ session.becomeActiveIfPossible { [weak self] isActive in
+ self?.performOnMain {
+#if DEBUG
+ if !isActive {
+ print("Now Playing session activation failed for player \(player.id)")
+ }
+#endif
+ completion(isActive)
+ }
+ }
+ }
+
+ func removeSession(for player: AVPlayer) {
+ performOnMain {
+ let key = ObjectIdentifier(player)
+ guard let session = self.sessions.removeValue(forKey: key) else { return }
+ self.disableRemoteCommands(for: key, remoteCommandCenter: session.remoteCommandCenter)
+ if self.activeSession === session {
+ self.activeSession = nil
+ }
+ }
+ }
+
+ private func registerCommandTarget(
+ for playerKey: ObjectIdentifier,
+ command: MPRemoteCommand,
+ handler: @escaping (MPRemoteCommandEvent) -> MPRemoteCommandHandlerStatus
+ ) {
+ remoteCommandTargets[playerKey]?.append(command.addTarget(handler: handler))
+ }
private var currentArtworkUrl: URL?
private var cachedArtwork: MPMediaItemArtwork?
+ private var cachedAnimatedSquareArtwork: AnyObject?
+ private var cachedAnimatedTallArtwork: AnyObject?
+ /// Tall HLS clip from the animated-artwork endpoint, keyed by the static
+ /// cover it was fetched for. Set per track from JS; a late arrival lands
+ /// only while the key still matches, so it never leaks onto the next one.
+ /// Deliberately kept out of `resetArtworkState`: the reset runs on every
+ /// artwork change, and wiping here would drop a clip fetched just before
+ /// the metadata carrying it — the key mismatch already invalidates those.
+ private var streamTallClip: AnyObject?
+ private var streamTallArtworkUrl: URL?
private var artworkLoadToken: UUID?
private var currentArtworkItemIdentifier: ObjectIdentifier?
private var isLiveStream = false
@@ -39,8 +104,11 @@ class MediaController {
isLiveStream = options?.isLiveStream ?? player?.isLive ?? false
if let player {
- enableRemoteCommands(options: options)
- updateNowPlayingInfoOnMain(for: player)
+ activateSession(for: player) { [weak self] isActive in
+ guard let self, isActive, self.activePlayer?.id == player.id, self.activeSession === self.session(for: player) else { return }
+ self.enableRemoteCommands(options: options)
+ self.updateNowPlayingInfoOnMain(for: player)
+ }
} else {
disableRemoteCommands()
clearNowPlayingInfoOnMain()
@@ -51,6 +119,15 @@ class MediaController {
guard player.id == activePlayer?.id else {
return
}
+ let session = session(for: player)
+ if activeSession !== session || !session.isActive {
+ activateSession(for: player) { [weak self] isActive in
+ guard isActive else { return }
+ self?.updateNowPlayingInfoOnMain(for: player)
+ }
+ return
+ }
+ guard let nowPlayingInfoCenter else { return }
let nextArtworkItemIdentifier = player.ref.currentItem.map { ObjectIdentifier($0) }
if currentArtworkItemIdentifier != nextArtworkItemIdentifier {
@@ -68,10 +145,16 @@ class MediaController {
} else {
resetArtworkState()
nowPlayingInfo.removeValue(forKey: MPMediaItemPropertyArtwork)
+ removeAnimatedArtwork(&nowPlayingInfo)
applyAssetMetadata(&nowPlayingInfo, for: player)
}
nowPlayingInfoCenter.nowPlayingInfo = nowPlayingInfo
+ // The state itself, alongside the rate above: what tells the system this
+ // is playback in progress rather than metadata sitting idle.
+ if #available(iOS 13.0, *) {
+ nowPlayingInfoCenter.playbackState = player.isPlaying ? .playing : .paused
+ }
}
private func applyPlaybackInfo(_ info: inout [String: Any], for player: AudioPlayer) {
@@ -80,7 +163,11 @@ class MediaController {
info.removeValue(forKey: MPMediaItemPropertyPlaybackDuration)
info.removeValue(forKey: MPNowPlayingInfoPropertyElapsedPlaybackTime)
} else {
- info[MPMediaItemPropertyPlaybackDuration] = player.duration
+ // AVPlayer has no duration for a transcoded stream, and 0 shows as
+ // --:-- with no progress: the caller's length stands in until it has one.
+ let known = player.duration
+ let fallback = (player.metadata?.durationMs ?? 0) / 1000.0
+ info[MPMediaItemPropertyPlaybackDuration] = known > 0 ? known : fallback
info[MPNowPlayingInfoPropertyElapsedPlaybackTime] = player.currentTime
}
info[MPNowPlayingInfoPropertyPlaybackRate] = player.isPlaying ? player.ref.rate : 0.0
@@ -139,17 +226,21 @@ class MediaController {
guard let artworkUrl = metadata.artworkUrl else {
resetArtworkState()
info.removeValue(forKey: MPMediaItemPropertyArtwork)
+ removeAnimatedArtwork(&info)
return
}
if currentArtworkUrl != artworkUrl {
currentArtworkUrl = artworkUrl
cachedArtwork = nil
+ cachedAnimatedSquareArtwork = nil
+ cachedAnimatedTallArtwork = nil
artworkLoadToken = nil
}
if let cachedArtwork {
info[MPMediaItemPropertyArtwork] = cachedArtwork
+ applyAnimatedArtwork(&info)
return
}
@@ -173,20 +264,79 @@ class MediaController {
return
}
- guard let artwork else {
+ guard let still = artwork.still else {
return
}
- self.cachedArtwork = artwork
+ self.cachedArtwork = still
+ self.cachedAnimatedSquareArtwork = artwork.square
+ self.cachedAnimatedTallArtwork = artwork.tall
+
+ guard let nowPlayingInfoCenter = self.nowPlayingInfoCenter else { return }
+ var latestNowPlayingInfo = nowPlayingInfoCenter.nowPlayingInfo ?? [String: Any]()
+ latestNowPlayingInfo[MPMediaItemPropertyArtwork] = still
+ self.applyAnimatedArtwork(&latestNowPlayingInfo)
+ nowPlayingInfoCenter.nowPlayingInfo = latestNowPlayingInfo
+
+ // A clip that was still encoding goes up as soon as it is on disk. The
+ // keys are only ever handed over with a file the system can open at
+ // once: its request comes with the artwork itself, and one answered
+ // after the wait left the lock screen on the preview frame for good —
+ // skipping on and back, which loads it all over again, was the only
+ // way out of it.
+ self.publishAnimatedWhenReady(artwork.square, url: artworkUrl, player: player, item: currentItemIdentifier)
+ self.publishAnimatedWhenReady(artwork.tall, url: artworkUrl, player: player, item: currentItemIdentifier)
+ }
+ }
- var latestNowPlayingInfo = self.nowPlayingInfoCenter.nowPlayingInfo ?? [String: Any]()
- latestNowPlayingInfo[MPMediaItemPropertyArtwork] = artwork
- self.nowPlayingInfoCenter.nowPlayingInfo = latestNowPlayingInfo
+ /**
+ * Publishes the animated artwork keys once `clip` is on disk, and only if
+ * this is still the artwork on screen. Every variant is written each time
+ * (`applyAnimatedArtwork` skips the ones not ready), so the last clip to
+ * finish carries both keys up together.
+ */
+ private func publishAnimatedWhenReady(
+ _ clip: AnyObject?,
+ url: URL,
+ player: AudioPlayer,
+ item: ObjectIdentifier?
+ ) {
+#if compiler(>=6.2)
+ if #available(iOS 26.0, tvOS 26.0, *) {
+ guard let clip = clip as? AnimatedArtwork else {
+ return
+ }
+ clip.whenReady { [weak self] in
+ guard let self else {
+ return
+ }
+ self.performOnMain {
+ guard self.cachedAnimatedSquareArtwork === clip || self.cachedAnimatedTallArtwork === clip,
+ self.currentArtworkUrl == url,
+ self.currentArtworkItemIdentifier == item,
+ self.activePlayer?.id == player.id
+ else {
+ return
+ }
+
+ guard let nowPlayingInfoCenter = self.nowPlayingInfoCenter else { return }
+ var latestNowPlayingInfo = nowPlayingInfoCenter.nowPlayingInfo ?? [String: Any]()
+ self.applyAnimatedArtwork(&latestNowPlayingInfo)
+ nowPlayingInfoCenter.nowPlayingInfo = latestNowPlayingInfo
+ }
+ }
}
+#endif
}
private func clearNowPlayingInfoOnMain() {
- nowPlayingInfoCenter.nowPlayingInfo = nil
+ if let nowPlayingInfoCenter {
+ nowPlayingInfoCenter.nowPlayingInfo = nil
+ if #available(iOS 13.0, *) {
+ nowPlayingInfoCenter.playbackState = .stopped
+ }
+ }
+ activeSession = nil
activePlayer = nil
isLiveStream = false
resetArtworkState()
@@ -204,35 +354,180 @@ class MediaController {
private func resetArtworkState() {
currentArtworkUrl = nil
cachedArtwork = nil
+ cachedAnimatedSquareArtwork = nil
+ cachedAnimatedTallArtwork = nil
artworkLoadToken = nil
}
- private func loadArtworkFromURL(url: URL, completion: @escaping (MPMediaItemArtwork?) -> Void) {
- URLSession.shared.dataTask(with: url) { data, _, error in
- if error != nil {
+ private struct LoadedArtwork {
+ let still: MPMediaItemArtwork?
+ let square: AnyObject?
+ let tall: AnyObject?
+ }
+
+ private func loadArtworkFromURL(url: URL, completion: @escaping (LoadedArtwork) -> Void) {
+ // Through the preloader, which `setNextSource` warms while the track it
+ // belongs to is still the NEXT one: the bytes, and the clip's encode
+ // behind them, are usually done before the lock screen asks, and a cover
+ // it has never seen is fetched once for every caller that wants it,
+ // instead of once each.
+ ArtworkPreloader.shared.fetch(url) { ready in
+ guard let ready, let image = UIImage(data: ready.data) else {
self.performOnMain {
- completion(nil)
+ completion(LoadedArtwork(still: nil, square: nil, tall: nil))
}
return
}
- guard let data, let image = UIImage(data: data) else {
- self.performOnMain {
- completion(nil)
+ // The request carries the size the surface wants: a square one (the
+ // Dynamic Island's compact artwork) gets the cover's centered square,
+ // while a taller one keeps the whole tall picture — which is what the
+ // full-screen lock screen draws.
+ let square = Self.centeredSquarePreview(of: image)
+ let still = MPMediaItemArtwork(boundsSize: image.size) { requested in
+ if let square, Self.isSquare(requested) {
+ return square
}
- return
+ return image
}
-
- let artwork = MPMediaItemArtwork(boundsSize: image.size) { _ in image }
self.performOnMain {
- completion(artwork)
+ completion(LoadedArtwork(still: still, square: ready.square, tall: ready.tall))
+ }
+ }
+ }
+
+ // Both variants are published: the system drops the key it does not support
+ // on its own, and gating on `supportedAnimatedArtworkKeys` is what kept the
+ // lock screen on the static artwork — the tall 3:4 variant is the one the
+ // iPhone lock screen draws full screen.
+ //
+ // Only with the clip on disk (`isOnDisk`): the system's request arrives with
+ // the key, and one that then waits for the encoder leaves the lock screen
+ // on the preview frame while the cover stands ready behind it.
+ private func applyAnimatedArtwork(_ info: inout [String: Any]) {
+#if compiler(>=6.2)
+ if #available(iOS 26.0, tvOS 26.0, *) {
+ if let clip = cachedAnimatedSquareArtwork as? AnimatedArtwork, let item = clip.item, clip.isOnDisk {
+ info[MPNowPlayingInfoProperty1x1AnimatedArtwork] = item
+ } else {
+ info.removeValue(forKey: MPNowPlayingInfoProperty1x1AnimatedArtwork)
+ }
+ if let clip = cachedAnimatedTallArtwork as? AnimatedArtwork, let item = clip.item, clip.isOnDisk {
+ info[MPNowPlayingInfoProperty3x4AnimatedArtwork] = item
+ } else {
+ info.removeValue(forKey: MPNowPlayingInfoProperty3x4AnimatedArtwork)
+ }
+ // An endpoint clip wins over an encoded one when it belongs to this
+ // artwork: it was already streaming while the encoder was still working.
+ if let key = streamTallArtworkUrl, key == currentArtworkUrl, let item = streamTallClip {
+ info[MPNowPlayingInfoProperty3x4AnimatedArtwork] = item
+ }
+ }
+#endif
+ }
+
+ /**
+ * Tall HLS clip from the animated-artwork endpoint for one static cover.
+ * Exported in the background and published only while it still belongs to
+ * this artwork: a slow export landing after a skip is dropped, and its file
+ * stays cached for the album either way. The static cover stays for every
+ * surface that asks square.
+ *
+ * The published info is rewritten ONLY when the export lands — the same
+ * moment the GIF path publishes. A bare rewrite at track-change time races
+ * the player's own metadata write and can publish an empty "now playing",
+ * which takes the lock screen and the Dynamic Island away until the next
+ * play/pause beat re-writes them (#260): with it, backgrounding a track
+ * showed no media controls at all.
+ */
+ func setStreamClip(tallUrl: URL?, for artworkUrl: URL?) {
+ performOnMain {
+ self.streamTallArtworkUrl = tallUrl == nil ? nil : artworkUrl
+ self.streamTallClip = nil
+ guard let tallUrl, let artworkUrl else {
+ return
+ }
+#if compiler(>=6.2)
+ if #available(iOS 26.0, tvOS 26.0, *) {
+ let playerId = self.activePlayer?.id
+ let item = self.currentArtworkItemIdentifier
+ Task.detached(priority: .userInitiated) { [weak self] in
+ let clip = await StreamArtworkClip.item(m3u8: tallUrl).value
+ guard let self, let clip else {
+ return
+ }
+ self.performOnMain {
+ guard self.streamTallArtworkUrl == artworkUrl,
+ self.currentArtworkUrl == artworkUrl,
+ self.currentArtworkItemIdentifier == item,
+ self.activePlayer?.id == playerId,
+ let nowPlayingInfoCenter = self.nowPlayingInfoCenter,
+ let published = nowPlayingInfoCenter.nowPlayingInfo,
+ !published.isEmpty
+ else {
+ // Nothing published yet: the metadata write that follows picks
+ // the clip up through `applyAnimatedArtwork` — publishing an
+ // empty info here is what took the controls away.
+ return
+ }
+ self.streamTallClip = clip
+ var latest = published
+ self.applyAnimatedArtwork(&latest)
+ nowPlayingInfoCenter.nowPlayingInfo = latest
+ }
+ }
}
+#endif
+ }
+ }
+
+ private func removeAnimatedArtwork(_ info: inout [String: Any]) {
+#if compiler(>=6.2)
+ if #available(iOS 26.0, tvOS 26.0, *) {
+ info.removeValue(forKey: MPNowPlayingInfoProperty1x1AnimatedArtwork)
+ info.removeValue(forKey: MPNowPlayingInfoProperty3x4AnimatedArtwork)
+ }
+#endif
+ }
+
+ // Whether a requested artwork size reads as a square: what the Dynamic
+ // Island's compact artwork asks for, as opposed to the tall sizes the
+ // full-screen lock screen asks for.
+ private static func isSquare(_ size: CGSize) -> Bool {
+ guard size.width > 0, size.height > 0 else {
+ return false
+ }
+ let difference = (size.width - size.height).magnitude
+ return difference <= max(size.width, size.height) * 0.1
+ }
+
+ // The cover's centered square, for the surfaces that ask square — and only
+ // when the cover is meaningfully tall. A square cover comes back untouched,
+ // so nothing about it changes anywhere.
+ private static func centeredSquarePreview(of image: UIImage) -> UIImage? {
+ let width = image.size.width
+ let height = image.size.height
+ guard width > 0, height > width * 1.05 else {
+ return nil
+ }
+ let side = min(width, height)
+ let format = UIGraphicsImageRendererFormat()
+ format.scale = image.scale
+ return UIGraphicsImageRenderer(size: CGSize(width: side, height: side), format: format).image { _ in
+ image.draw(at: CGPoint(x: (side - width) / 2, y: (side - height) / 2))
}
- .resume()
}
private func enableRemoteCommands(options: LockScreenOptions?) {
- remoteCommandCenter.playCommand.addTarget { [weak self] _ in
+ guard let remoteCommandCenter else { return }
+ updateRemoteCommandAvailability(options: options)
+
+ guard let activePlayer else { return }
+ let playerKey = ObjectIdentifier(activePlayer.ref)
+ guard remoteCommandTargets[playerKey] == nil else { return }
+ remoteCommandTargets[playerKey] = []
+
+ registerCommandTarget(for: playerKey, command: remoteCommandCenter.playCommand) { [weak self] _ in
guard let player = self?.activePlayer else {
return .commandFailed
}
@@ -241,7 +536,7 @@ class MediaController {
return .success
}
- remoteCommandCenter.pauseCommand.addTarget { [weak self] _ in
+ registerCommandTarget(for: playerKey, command: remoteCommandCenter.pauseCommand) { [weak self] _ in
guard let player = self?.activePlayer else {
return .commandFailed
}
@@ -250,7 +545,7 @@ class MediaController {
return .success
}
- remoteCommandCenter.togglePlayPauseCommand.addTarget { [weak self] _ in
+ registerCommandTarget(for: playerKey, command: remoteCommandCenter.togglePlayPauseCommand) { [weak self] _ in
guard let player = self?.activePlayer else {
return .commandFailed
}
@@ -263,20 +558,20 @@ class MediaController {
return .success
}
- remoteCommandCenter.changePlaybackPositionCommand.addTarget { [weak self] event in
+ registerCommandTarget(for: playerKey, command: remoteCommandCenter.changePlaybackPositionCommand) { [weak self] event in
guard let player = self?.activePlayer,
let event = event as? MPChangePlaybackPositionCommandEvent else {
return .commandFailed
}
let seekTime = CMTime(seconds: event.positionTime, preferredTimescale: 1)
- player.ref.seek(to: seekTime)
+ player.ref.seek(to: seekTime, toleranceBefore: .zero, toleranceAfter: .zero)
return .success
}
remoteCommandCenter.skipForwardCommand.preferredIntervals = [10.0]
- remoteCommandCenter.skipForwardCommand.addTarget { [weak self] event in
+ registerCommandTarget(for: playerKey, command: remoteCommandCenter.skipForwardCommand) { [weak self] event in
guard let player = self?.activePlayer,
let event = event as? MPSkipIntervalCommandEvent else {
return .commandFailed
@@ -290,7 +585,7 @@ class MediaController {
}
remoteCommandCenter.skipBackwardCommand.preferredIntervals = [10.0]
- remoteCommandCenter.skipBackwardCommand.addTarget { [weak self] event in
+ registerCommandTarget(for: playerKey, command: remoteCommandCenter.skipBackwardCommand) { [weak self] event in
guard let player = self?.activePlayer,
let event = event as? MPSkipIntervalCommandEvent else {
return .commandFailed
@@ -303,28 +598,69 @@ class MediaController {
return .success
}
+ registerCommandTarget(for: playerKey, command: remoteCommandCenter.nextTrackCommand) { [weak self] _ in
+ guard let player = self?.activePlayer else {
+ return .commandFailed
+ }
+
+ player.emit(event: "remoteNext")
+ return .success
+ }
+
+ registerCommandTarget(for: playerKey, command: remoteCommandCenter.previousTrackCommand) { [weak self] _ in
+ guard let player = self?.activePlayer else {
+ return .commandFailed
+ }
+
+ player.emit(event: "remotePrevious")
+ return .success
+ }
+ }
+
+ private func updateRemoteCommandAvailability(options: LockScreenOptions?) {
+ guard let remoteCommandCenter else { return }
remoteCommandCenter.playCommand.isEnabled = true
remoteCommandCenter.pauseCommand.isEnabled = true
remoteCommandCenter.togglePlayPauseCommand.isEnabled = true
remoteCommandCenter.changePlaybackPositionCommand.isEnabled = !isLiveStream
remoteCommandCenter.skipForwardCommand.isEnabled = options?.showSeekForward ?? false
remoteCommandCenter.skipBackwardCommand.isEnabled = options?.showSeekBackward ?? false
+ remoteCommandCenter.nextTrackCommand.isEnabled = options?.showSkipNext ?? false
+ remoteCommandCenter.previousTrackCommand.isEnabled = options?.showSkipPrevious ?? false
}
private func disableRemoteCommands() {
+ guard let remoteCommandCenter, let activePlayer else { return }
+ disableRemoteCommands(for: ObjectIdentifier(activePlayer.ref), remoteCommandCenter: remoteCommandCenter)
+ }
+
+ private func disableRemoteCommands(
+ for playerKey: ObjectIdentifier,
+ remoteCommandCenter: MPRemoteCommandCenter
+ ) {
remoteCommandCenter.playCommand.isEnabled = false
remoteCommandCenter.pauseCommand.isEnabled = false
remoteCommandCenter.togglePlayPauseCommand.isEnabled = false
remoteCommandCenter.changePlaybackPositionCommand.isEnabled = false
remoteCommandCenter.skipForwardCommand.isEnabled = false
remoteCommandCenter.skipBackwardCommand.isEnabled = false
-
- // Remove event targets
- remoteCommandCenter.playCommand.removeTarget(self)
- remoteCommandCenter.pauseCommand.removeTarget(self)
- remoteCommandCenter.togglePlayPauseCommand.removeTarget(self)
- remoteCommandCenter.changePlaybackPositionCommand.removeTarget(self)
- remoteCommandCenter.skipForwardCommand.removeTarget(self)
- remoteCommandCenter.skipBackwardCommand.removeTarget(self)
+ remoteCommandCenter.nextTrackCommand.isEnabled = false
+ remoteCommandCenter.previousTrackCommand.isEnabled = false
+
+ if let targets = remoteCommandTargets.removeValue(forKey: playerKey) {
+ let commands = [
+ remoteCommandCenter.playCommand,
+ remoteCommandCenter.pauseCommand,
+ remoteCommandCenter.togglePlayPauseCommand,
+ remoteCommandCenter.changePlaybackPositionCommand,
+ remoteCommandCenter.skipForwardCommand,
+ remoteCommandCenter.skipBackwardCommand,
+ remoteCommandCenter.nextTrackCommand,
+ remoteCommandCenter.previousTrackCommand,
+ ]
+ for (command, target) in zip(commands, targets) {
+ command.removeTarget(target)
+ }
+ }
}
}