diff --git a/README.md b/README.md index 8aa42823..51191fe2 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ *Like the project, make profit from it, or simply want to thank back? Please consider [sponsoring me](https://github.com/sponsors/natario1)!* -# Transcoder +## Transcoder Transcodes and compresses video files into the MP4 format, with audio support, using hardware-accelerated Android codecs available on the device. Works on API 18+. @@ -90,4 +90,4 @@ Transcoder.into(filePath) }).transcode() ``` -Take a look at the demo app for a complete example. \ No newline at end of file +Take a look at the demo app for a complete example. diff --git a/build.gradle.kts b/build.gradle.kts index a4bb9799..303461c6 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,20 +1,20 @@ buildscript { - extra["minSdkVersion"] = 18 + extra["minSdkVersion"] = 21 extra["compileSdkVersion"] = 30 extra["targetSdkVersion"] = 30 - + repositories { google() mavenCentral() - jcenter() + gradlePluginPortal() } dependencies { - classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:1.4.31") - classpath("com.android.tools.build:gradle:4.1.2") - classpath("org.jetbrains.dokka:dokka-gradle-plugin:1.4.30") // publisher uses 1.4.20 which goes OOM - classpath("io.deepmedia.tools:publisher:0.5.0") + classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:1.8.0") + classpath("com.android.tools.build:gradle:7.4.0") +// classpath("org.jetbrains.dokka:dokka-gradle-plugin:1.4.30") // publisher uses 1.4.20 which goes OOM +// classpath("io.deepmedia.tools:publisher:0.5.0") } } @@ -22,10 +22,9 @@ allprojects { repositories { google() mavenCentral() - jcenter() } } tasks.register("clean", Delete::class) { delete(buildDir) -} \ No newline at end of file +} diff --git a/lib/build.gradle.kts b/lib/build.gradle.kts index a670c667..1622ffe2 100644 --- a/lib/build.gradle.kts +++ b/lib/build.gradle.kts @@ -1,30 +1,32 @@ -import io.deepmedia.tools.publisher.common.GithubScm -import io.deepmedia.tools.publisher.common.License -import io.deepmedia.tools.publisher.common.Release -import io.deepmedia.tools.publisher.sonatype.Sonatype +//import io.deepmedia.tools.publisher.common.GithubScm +//import io.deepmedia.tools.publisher.common.License +//import io.deepmedia.tools.publisher.common.Release +//import io.deepmedia.tools.publisher.sonatype.Sonatype plugins { id("com.android.library") id("kotlin-android") - id("io.deepmedia.tools.publisher") +// id("io.deepmedia.tools.publisher") } android { setCompileSdkVersion(property("compileSdkVersion") as Int) defaultConfig { - setMinSdkVersion(property("minSdkVersion") as Int) - setTargetSdkVersion(property("targetSdkVersion") as Int) - versionCode = 1 - versionName = "0.10.3" + minSdk = (property("minSdkVersion") as Int) + targetSdk = (property("targetSdkVersion") as Int) testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + namespace = "com.otaliastudios.transcoder" } buildTypes["release"].isMinifyEnabled = false + buildFeatures { + buildConfig = true + } } - dependencies { api("com.otaliastudios.opengl:egloo:0.6.0") - api("androidx.annotation:annotation:1.1.0") + api("androidx.annotation:annotation:1.3.0") + api("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.6.4") androidTestImplementation("androidx.test:runner:1.3.0") androidTestImplementation("androidx.test:rules:1.3.0") @@ -32,6 +34,7 @@ dependencies { androidTestImplementation("org.mockito:mockito-android:2.28.2") } +/* publisher { project.description = "Accelerated video transcoding using Android MediaCodec API without native code (no LGPL/patent issues)." project.artifact = "transcoder" @@ -61,3 +64,4 @@ publisher { signing.password = "SIGNING_PASSWORD" } } +*/ diff --git a/lib/src/main/AndroidManifest.xml b/lib/src/main/AndroidManifest.xml index f62d05a4..925315a4 100644 --- a/lib/src/main/AndroidManifest.xml +++ b/lib/src/main/AndroidManifest.xml @@ -1,5 +1,4 @@ - + diff --git a/lib/src/main/java/com/otaliastudios/transcoder/Thumbnailer.kt b/lib/src/main/java/com/otaliastudios/transcoder/Thumbnailer.kt index c519e49a..fce4929a 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/Thumbnailer.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/Thumbnailer.kt @@ -1,26 +1,25 @@ package com.otaliastudios.transcoder -import com.otaliastudios.transcoder.internal.thumbnails.ThumbnailsEngine -import com.otaliastudios.transcoder.internal.utils.ThreadPool -import com.otaliastudios.transcoder.thumbnail.Thumbnail -import java.util.concurrent.Callable -import java.util.concurrent.Future - -class Thumbnailer private constructor() { - - fun thumbnails(options: ThumbnailerOptions): Future { - return ThreadPool.executor.submit(Callable { - ThumbnailsEngine.thumbnails(options) - null - }) - } - - fun thumbnails(builder: ThumbnailerOptions.Builder.() -> Unit) = thumbnails( - options = ThumbnailerOptions.Builder().apply(builder).build() - ) - - companion object { - // Just for consistency with Transcoder class. - fun getInstance() = Thumbnailer() - } -} \ No newline at end of file +// import com.otaliastudios.transcoder.internal.thumbnails.ThumbnailsEngine +// import com.otaliastudios.transcoder.internal.utils.ThreadPool +// import java.util.concurrent.Callable +// import java.util.concurrent.Future +// +// class Thumbnailer private constructor() { +// +// fun thumbnails(options: ThumbnailerOptions): Future { +// return ThreadPool.executor.submit(Callable { +// ThumbnailsEngine.thumbnails(options) +// null +// }) +// } +// +// fun thumbnails(builder: ThumbnailerOptions.Builder.() -> Unit) = thumbnails( +// options = ThumbnailerOptions.Builder().apply(builder).build() +// ) +// +// companion object { +// // Just for consistency with Transcoder class. +// fun getInstance() = Thumbnailer() +// } +// } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/ThumbnailerListener.kt b/lib/src/main/java/com/otaliastudios/transcoder/ThumbnailerListener.kt index b0f89872..e79e4de5 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/ThumbnailerListener.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/ThumbnailerListener.kt @@ -11,4 +11,4 @@ interface ThumbnailerListener { fun onThumbnailsCanceled() = Unit fun onThumbnailsFailed(exception: Throwable) -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/ThumbnailerOptions.kt b/lib/src/main/java/com/otaliastudios/transcoder/ThumbnailerOptions.kt index 5a0a35d6..66418e14 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/ThumbnailerOptions.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/ThumbnailerOptions.kt @@ -1,9 +1,13 @@ +@file:Suppress("TooManyFunctions", "MagicNumber") + package com.otaliastudios.transcoder import android.content.Context import android.net.Uri import android.os.Handler import android.os.Looper +import com.otaliastudios.transcoder.internal.codec.TranscoderEventsListener +import com.otaliastudios.transcoder.internal.thumbnails.ThumbnailsEngine import com.otaliastudios.transcoder.resize.ExactResizer import com.otaliastudios.transcoder.resize.MultiResizer import com.otaliastudios.transcoder.resize.Resizer @@ -13,16 +17,16 @@ import com.otaliastudios.transcoder.source.FilePathDataSource import com.otaliastudios.transcoder.source.UriDataSource import com.otaliastudios.transcoder.thumbnail.ThumbnailRequest import java.io.FileDescriptor -import java.util.concurrent.Future @Suppress("unused") class ThumbnailerOptions( - val dataSources: List, - val resizer: Resizer, - val rotation: Int, - val thumbnailRequests: List, - val listener: ThumbnailerListener, - val listenerHandler: Handler + val dataSources: List, + val resizer: Resizer, + val rotation: Int, + val thumbnailRequests: List, + val listener: ThumbnailerListener?, + val listenerHandler: Handler, + val eventListener: TranscoderEventsListener?, ) { class Builder { @@ -34,19 +38,20 @@ class ThumbnailerOptions( private var rotation = 0 private var listener: ThumbnailerListener? = null private var listenerHandler: Handler? = null + private var eventListener: TranscoderEventsListener? = null fun addDataSource(dataSource: DataSource) = this.also { dataSources.add(dataSource) } - fun addDataSource(fileDescriptor: FileDescriptor) - = addDataSource(FileDescriptorDataSource(fileDescriptor)) + fun addDataSource(fileDescriptor: FileDescriptor) = + addDataSource(FileDescriptorDataSource(fileDescriptor)) - fun addDataSource(filePath: String) - = addDataSource(FilePathDataSource(filePath)) + fun addDataSource(filePath: String) = + addDataSource(FilePathDataSource(filePath)) - fun addDataSource(context: Context, uri: Uri) - = addDataSource(UriDataSource(context, uri)) + fun addDataSource(context: Context, uri: Uri) = + addDataSource(UriDataSource(context, uri)) /** * Sets the video output strategy. If absent, this defaults to 320x240 images. @@ -80,32 +85,31 @@ class ThumbnailerOptions( fun setListener(listener: ThumbnailerListener) = this.also { this.listener = listener } + fun setEventListener(listener: TranscoderEventsListener) = this.also { + this.eventListener = listener + } fun build(): ThumbnailerOptions { - require(dataSources.isNotEmpty()) { - "At least one data source is required!" - } - require(thumbnailRequests.isNotEmpty()) { - "At least one thumbnail request is required!" - } - val listener = requireNotNull(listener) { - "Listener can't be null." - } +// require(dataSources.isNotEmpty()) { +// "At least one data source is required!" +// } + val listenerHandler = listenerHandler - ?: Handler(Looper.myLooper() ?: Looper.getMainLooper()) + ?: Handler(Looper.myLooper() ?: Looper.getMainLooper()) val resizer = if (resizerSet) resizer else ExactResizer(320, 240) return ThumbnailerOptions( - dataSources = dataSources.toList(), - resizer = resizer, - rotation = rotation, - thumbnailRequests = thumbnailRequests.toList(), - listener = listener, - listenerHandler = listenerHandler + dataSources = dataSources.toList(), + resizer = resizer, + rotation = rotation, + thumbnailRequests = thumbnailRequests.toList(), + listener = listener, + listenerHandler = listenerHandler, + eventListener = eventListener ) } - fun thumbnails(): Future { - return Thumbnailer.getInstance().thumbnails(build()) + fun thumbnails(): ThumbnailsEngine { + return ThumbnailsEngine.thumbnails(build()) } } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/Transcoder.java b/lib/src/main/java/com/otaliastudios/transcoder/Transcoder.java index 12e32dd1..6fe4b722 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/Transcoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/Transcoder.java @@ -15,19 +15,24 @@ */ package com.otaliastudios.transcoder; +import android.media.MediaFormat; import android.os.Build; +import androidx.annotation.NonNull; +import androidx.annotation.RequiresApi; + +import com.otaliastudios.transcoder.common.TrackType; +import com.otaliastudios.transcoder.internal.Codecs; +import com.otaliastudios.transcoder.internal.pipeline.Pipeline; import com.otaliastudios.transcoder.internal.transcode.TranscodeEngine; import com.otaliastudios.transcoder.internal.utils.ThreadPool; import com.otaliastudios.transcoder.sink.DataSink; import com.otaliastudios.transcoder.validator.Validator; import java.io.FileDescriptor; -import java.util.concurrent.Callable; import java.util.concurrent.Future; -import androidx.annotation.NonNull; -import androidx.annotation.RequiresApi; +import kotlin.jvm.functions.Function4; public class Transcoder { /** @@ -92,16 +97,14 @@ public static TranscoderOptions.Builder into(@NonNull DataSink dataSink) { * Transcodes video file asynchronously. * * @param options The transcoder options. + * @param function * @return a Future that completes when transcoding is completed */ @NonNull - public Future transcode(@NonNull final TranscoderOptions options) { - return ThreadPool.getExecutor().submit(new Callable() { - @Override - public Void call() throws Exception { - TranscodeEngine.transcode(options); - return null; - } + public Future transcode(@NonNull final TranscoderOptions options, Function4 function) { + return ThreadPool.getExecutor().submit(() -> { + TranscodeEngine.transcode(options, function); + return null; }); } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java index 42b35679..99343e8a 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java @@ -1,12 +1,19 @@ package com.otaliastudios.transcoder; import android.content.Context; +import android.media.MediaFormat; import android.net.Uri; import android.os.Build; import android.os.Handler; import android.os.Looper; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; + import com.otaliastudios.transcoder.common.TrackType; +import com.otaliastudios.transcoder.internal.Codecs; +import com.otaliastudios.transcoder.internal.pipeline.Pipeline; import com.otaliastudios.transcoder.resample.AudioResampler; import com.otaliastudios.transcoder.resample.DefaultAudioResampler; import com.otaliastudios.transcoder.sink.DataSink; @@ -14,7 +21,6 @@ import com.otaliastudios.transcoder.source.DataSource; import com.otaliastudios.transcoder.source.FileDescriptorDataSource; import com.otaliastudios.transcoder.source.FilePathDataSource; -import com.otaliastudios.transcoder.source.BlankAudioDataSource; import com.otaliastudios.transcoder.source.UriDataSource; import com.otaliastudios.transcoder.strategy.DefaultAudioStrategy; import com.otaliastudios.transcoder.strategy.DefaultVideoStrategies; @@ -32,9 +38,7 @@ import java.util.List; import java.util.concurrent.Future; -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; -import androidx.annotation.RequiresApi; +import kotlin.jvm.functions.Function4; /** * Collects transcoding options consumed by {@link Transcoder}. @@ -56,7 +60,6 @@ private TranscoderOptions() {} private TranscoderListener listener; private Handler listenerHandler; - @NonNull public TranscoderListener getListener() { return listener; } @@ -337,9 +340,7 @@ public Builder setAudioResampler(@NonNull AudioResampler audioResampler) { @NonNull public TranscoderOptions build() { - if (listener == null) { - throw new IllegalStateException("listener can't be null"); - } + if (audioDataSources.isEmpty() && videoDataSources.isEmpty()) { throw new IllegalStateException("we need at least one data source"); } @@ -386,8 +387,8 @@ public TranscoderOptions build() { } @NonNull - public Future transcode() { - return Transcoder.getInstance().transcode(build()); + public Future transcode(Function4 function) { + return Transcoder.getInstance().transcode(build(), function); } } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/common/TrackType.kt b/lib/src/main/java/com/otaliastudios/transcoder/common/TrackType.kt index 2dbbf7d1..10946eb0 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/common/TrackType.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/common/TrackType.kt @@ -14,4 +14,4 @@ internal val MediaFormat.trackTypeOrNull get() = when { getString(MediaFormat.KEY_MIME)!!.startsWith("audio/") -> TrackType.AUDIO getString(MediaFormat.KEY_MIME)!!.startsWith("video/") -> TrackType.VIDEO else -> null -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/Codecs.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/Codecs.kt index 9a138acb..945f2e4b 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/Codecs.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/Codecs.kt @@ -1,3 +1,5 @@ +@file:Suppress("UnusedPrivateMember") + package com.otaliastudios.transcoder.internal import android.media.MediaCodec @@ -5,12 +7,8 @@ import android.media.MediaFormat import android.view.Surface import com.otaliastudios.transcoder.common.TrackStatus import com.otaliastudios.transcoder.common.TrackType -import com.otaliastudios.transcoder.internal.media.MediaFormatProvider import com.otaliastudios.transcoder.internal.utils.Logger import com.otaliastudios.transcoder.internal.utils.TrackMap -import com.otaliastudios.transcoder.internal.utils.trackMapOf -import com.otaliastudios.transcoder.source.DataSource -import com.otaliastudios.transcoder.strategy.TrackStrategy /** * Encoders are shared between segments. This is not strictly needed but it is more efficient @@ -19,10 +17,10 @@ import com.otaliastudios.transcoder.strategy.TrackStrategy * output timestamps, even if input timestamps are. This would later create crashes when passing * data to MediaMuxer / MPEG4Writer. */ -internal class Codecs( - private val sources: DataSources, - private val tracks: Tracks, - private val current: TrackMap +class Codecs( + private val sources: DataSources, + private val tracks: Tracks, + private val current: TrackMap ) { private val log = Logger("Codecs") @@ -40,6 +38,7 @@ internal class Codecs( private val lazyVideo by lazy { val format = tracks.outputFormats.video + val codec = MediaCodec.createEncoderByType(format.getString(MediaFormat.KEY_MIME)!!) codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) codec to codec.createInputSurface() @@ -66,4 +65,4 @@ internal class Codecs( it.first.release() } } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/CustomSegments.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/CustomSegments.kt new file mode 100644 index 00000000..91151590 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/CustomSegments.kt @@ -0,0 +1,113 @@ +@file:Suppress("ReturnCount") + +package com.otaliastudios.transcoder.internal + +import android.media.MediaFormat +import com.otaliastudios.transcoder.common.TrackStatus +import com.otaliastudios.transcoder.common.TrackType +import com.otaliastudios.transcoder.internal.pipeline.Pipeline +import com.otaliastudios.transcoder.internal.utils.Logger +import com.otaliastudios.transcoder.internal.utils.mutableTrackMapOf +import com.otaliastudios.transcoder.source.DataSource + +class CustomSegments( + private val sources: DataSources, + private val tracks: Tracks, + private val factory: (TrackType, DataSource, MediaFormat) -> Pipeline, +) { + + private val log = Logger("Segments") + private var currentSegment: Segment? = null + private var currentSegmentMapKey: String? = null + private val segmentMap = mutableMapOf() + + fun hasNext(type: TrackType): Boolean { + if (!sources.has(type)) return false + log.v( + "hasNext($type): segment=${currentSegment} lastIndex=${sources.getOrNull(type)?.lastIndex}" + + " canAdvance=${currentSegment?.canAdvance()}" + ) + val segment = currentSegment ?: return true // not started + return segment.canAdvance() + } + + fun hasNext() = hasNext(TrackType.VIDEO) + + + // it will be time dependent + // 1. make segments work for thumbnails as is + // 2. inject segments dynamically + // 3. seek to segment and destroy previous ones + // 4. destroy only if necessary, else reuse + + fun getSegment(id: String): Segment? { + return segmentMap.getOrPut(id) { + destroySegment() + tryCreateSegment(id).also { + currentSegment = it + currentSegmentMapKey = id + } + } + } + + fun releaseSegment(id: String) { + val segment = segmentMap[id] + segment?.let { + it.release() + val source = sources[it.type].firstOrNull { it.mediaId() == id } + if (tracks.active.has(it.type)) { + source?.releaseTrack(it.type) + } + segmentMap.remove(id) + if(currentSegment == segment) { + currentSegment = null + } + } + } + + fun release() = destroySegment(true) + + private fun tryCreateSegment(id: String): Segment? { + // Return null if out of bounds, either because segments are over or because the + // source set does not have sources for this track type. + val source = sources[TrackType.VIDEO].firstOrNull { it.mediaId() == id } ?: return null + source.init() + log.i("tryCreateSegment(${TrackType.VIDEO}, $id): created!") + if (tracks.active.has(TrackType.VIDEO)) { + source.selectTrack(TrackType.VIDEO) + } + // Update current index before pipeline creation, for other components + // who check it during pipeline init. + val pipeline = factory( + TrackType.VIDEO, + source, + tracks.outputFormats[TrackType.VIDEO] + ) + return Segment(TrackType.VIDEO, -1, pipeline, id) + } + + private fun destroySegment(releaseAll: Boolean = false) { + currentSegment?.let { segment -> + segment.release() + val source = sources[segment.type].firstOrNull { it.mediaId() == segment.source } + if (tracks.active.has(segment.type)) { + source?.releaseTrack(segment.type) + } + currentSegmentMapKey?.let { + segmentMap.remove(it) + } + currentSegment = null + currentSegmentMapKey = null + } + if (releaseAll) { + segmentMap.forEach { + it.value?.release() + } + segmentMap.clear() + } + } + private fun DataSource.init() = if (!isInitialized) initialize() else Unit + + private fun DataSource.deinit() = if (isInitialized) deinitialize() else Unit + +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/DataSources.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/DataSources.kt index cb441d7b..1c54868d 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/DataSources.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/DataSources.kt @@ -5,13 +5,12 @@ import com.otaliastudios.transcoder.TranscoderOptions import com.otaliastudios.transcoder.common.TrackType import com.otaliastudios.transcoder.internal.utils.Logger import com.otaliastudios.transcoder.internal.utils.TrackMap -import com.otaliastudios.transcoder.internal.utils.trackMapOf import com.otaliastudios.transcoder.source.BlankAudioDataSource import com.otaliastudios.transcoder.source.DataSource -internal class DataSources private constructor( - videoSources: List, - audioSources: List, +class DataSources constructor( + videoSources: List, + audioSources: List, ) : TrackMap> { constructor(options: TranscoderOptions) : this(options.videoDataSources, options.audioDataSources) @@ -29,20 +28,21 @@ internal class DataSources private constructor( audioSources.init() } - private val videoSources: List = run { - val valid = videoSources.count { it.getTrackFormat(TrackType.VIDEO) != null } - when (valid) { - 0 -> listOf().also { videoSources.deinit() } - videoSources.size -> videoSources - else -> videoSources // Tracks will crash + private var videoSources: List = updateVideoSources(videoSources) + set(value) { + field = updateVideoSources(value) + } + + private var audioSources: List = updateAudioSources(audioSources) + set(value) { + field = updateAudioSources(value) } - } - private val audioSources: List = run { - val valid = audioSources.count { it.getTrackFormat(TrackType.AUDIO) != null } - when (valid) { - 0 -> listOf().also { audioSources.deinit() } - audioSources.size -> audioSources + private fun updateAudioSources(sources: List) : List { + val valid = sources.count { it.getTrackFormat(TrackType.AUDIO) != null } + return when (valid) { + 0 -> listOf().also { sources.deinit() } + sources.size -> sources else -> { // Some tracks do not have audio, while some do. Replace with BlankAudio. audioSources.map { source -> @@ -53,6 +53,58 @@ internal class DataSources private constructor( } } + private fun updateVideoSources(sources: List): List { + val valid = sources.count { it.getTrackFormat(TrackType.VIDEO) != null } + return when (valid) { + 0 -> listOf().also { sources.deinit() } + sources.size -> sources + else -> sources // Tracks will crash + } + } + + fun addDataSource(dataSource: DataSource) { + addVideoDataSource(dataSource) + addAudioDataSource(dataSource) + } + + fun addVideoDataSource(dataSource: DataSource) { + dataSource.init() + if (dataSource.getTrackFormat(TrackType.VIDEO) != null && dataSource !in videoSources) { + videoSources = videoSources + dataSource + } + } + + fun addAudioDataSource(dataSource: DataSource) { + dataSource.init() + if (dataSource.getTrackFormat(TrackType.AUDIO) != null) { + audioSources = audioSources + dataSource + } + } + + fun getVideoSources() = videoSources + + fun removeDataSource(dataSourceId: String) { + removeAudioDataSource(dataSourceId) + removeVideoDataSource(dataSourceId) + } + + fun removeVideoDataSource(dataSourceId: String) { + val source = videoSources.find { it.mediaId() == dataSourceId } + if (source?.getTrackFormat(TrackType.VIDEO) != null) { + videoSources = videoSources - source + source.releaseTrack(TrackType.VIDEO) + } + source?.deinit() + } + + fun removeAudioDataSource(dataSourceId: String) { + val source = audioSources.find { it.mediaId() == dataSourceId } + if (source?.getTrackFormat(TrackType.AUDIO) != null) { + audioSources = audioSources - source + source.releaseTrack(TrackType.AUDIO) + } + source?.deinit() + } override fun get(type: TrackType) = when (type) { TrackType.AUDIO -> audioSources TrackType.VIDEO -> videoSources @@ -68,4 +120,4 @@ internal class DataSources private constructor( audio.forEach { it.deinit() } log.i("release(): released.") } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/Segment.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/Segment.kt index b8884169..1a6d8e47 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/Segment.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/Segment.kt @@ -5,13 +5,14 @@ import com.otaliastudios.transcoder.internal.pipeline.Pipeline import com.otaliastudios.transcoder.internal.pipeline.State import com.otaliastudios.transcoder.internal.utils.Logger -internal class Segment( - val type: TrackType, - val index: Int, - private val pipeline: Pipeline, +class Segment( + val type: TrackType, + val index: Int, + private val pipeline: Pipeline, + val source: String = "", ) { - private val log = Logger("Segment($type,$index)") + private val log = Logger("Segment($type,$index,$source)") private var state: State? = null fun advance(): Boolean { @@ -27,4 +28,4 @@ internal class Segment( fun release() { pipeline.release() } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/Segments.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/Segments.kt index bdcc31a1..bbe96a01 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/Segments.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/Segments.kt @@ -1,3 +1,5 @@ +@file:Suppress("ReturnCount") + package com.otaliastudios.transcoder.internal import android.media.MediaFormat @@ -5,13 +7,13 @@ import com.otaliastudios.transcoder.common.TrackStatus import com.otaliastudios.transcoder.common.TrackType import com.otaliastudios.transcoder.internal.pipeline.Pipeline import com.otaliastudios.transcoder.internal.utils.Logger -import com.otaliastudios.transcoder.internal.utils.TrackMap import com.otaliastudios.transcoder.internal.utils.mutableTrackMapOf -internal class Segments( - private val sources: DataSources, - private val tracks: Tracks, - private val factory: (TrackType, Int, TrackStatus, MediaFormat) -> Pipeline +class Segments( + private val sources: DataSources, + private val tracks: Tracks, + private val factory: (TrackType, Int, TrackStatus, MediaFormat) -> Pipeline, + private val ownsLifeCycle: Boolean = true ) { private val log = Logger("Segments") @@ -21,7 +23,10 @@ internal class Segments( fun hasNext(type: TrackType): Boolean { if (!sources.has(type)) return false - log.v("hasNext($type): segment=${current.getOrNull(type)} lastIndex=${sources.getOrNull(type)?.lastIndex} canAdvance=${current.getOrNull(type)?.canAdvance()}") + log.v( + "hasNext($type): segment=${current.getOrNull(type)} lastIndex=${sources.getOrNull(type)?.lastIndex}" + + " canAdvance=${current.getOrNull(type)?.canAdvance()}" + ) val segment = current.getOrNull(type) ?: return true // not started val lastIndex = sources.getOrNull(type)?.lastIndex ?: return false // no track! return segment.canAdvance() || segment.index < lastIndex @@ -50,8 +55,12 @@ internal class Segments( // create a new one. destroySegment will increase the requested index, // so if this is the last one, we'll return null. else -> { - destroySegment(current[type]) - next(type) + if (ownsLifeCycle) { + destroySegment(current[type]) + next(type) + } else { + null + } } } } @@ -85,15 +94,15 @@ internal class Segments( // who check it during pipeline init. currentIndex[type] = index val pipeline = factory( - type, - index, - tracks.all[type], - tracks.outputFormats[type] + type, + index, + tracks.all[type], + tracks.outputFormats[type] ) return Segment( - type = type, - index = index, - pipeline = pipeline + type = type, + index = index, + pipeline = pipeline ).also { current[type] = it } @@ -107,4 +116,4 @@ internal class Segments( } requestedIndex[segment.type] = segment.index + 1 } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/Timer.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/Timer.kt index 9d9b822e..ce377219 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/Timer.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/Timer.kt @@ -1,3 +1,5 @@ +@file:Suppress("MagicNumber", "UnusedPrivateMember") + package com.otaliastudios.transcoder.internal import com.otaliastudios.transcoder.common.TrackType @@ -6,11 +8,11 @@ import com.otaliastudios.transcoder.internal.utils.TrackMap import com.otaliastudios.transcoder.source.DataSource import com.otaliastudios.transcoder.time.TimeInterpolator -internal class Timer( - private val interpolator: TimeInterpolator, - private val sources: DataSources, - private val tracks: Tracks, - private val current: TrackMap +class Timer( + private val interpolator: TimeInterpolator, + private val sources: DataSources, + private val tracks: Tracks, + private val current: TrackMap ) { private val log = Logger("Timer") @@ -60,12 +62,10 @@ internal class Timer( fun localize(type: TrackType, index: Int, positionUs: Long): Long? { if (!tracks.active.has(type)) return null val behindUs = sources[type] - .filterIndexed { i, _ -> i < index } - .durationUs(-1) + .filterIndexed { i, _ -> i < index } + .durationUs(-1) val localizedUs = positionUs - behindUs - if (localizedUs < 0L) return null - if (localizedUs > sources[type][index].durationUs) return null - return localizedUs + return if (localizedUs < 0L || localizedUs > sources[type][index].durationUs) null else localizedUs } fun interpolator(type: TrackType, index: Int) = interpolators.getOrPut(type to index) { @@ -92,4 +92,4 @@ internal class Timer( } } } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/Tracks.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/Tracks.kt index bafcc1cd..ea892023 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/Tracks.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/Tracks.kt @@ -10,35 +10,40 @@ import com.otaliastudios.transcoder.internal.utils.trackMapOf import com.otaliastudios.transcoder.source.DataSource import com.otaliastudios.transcoder.strategy.TrackStrategy -internal class Tracks( - strategies: TrackMap, - sources: DataSources, - videoRotation: Int, - forceCompression: Boolean +class Tracks( + val strategies: TrackMap, + val sources: DataSources, + val videoRotation: Int, + val forceCompression: Boolean ) { private val log = Logger("Tracks") - val all: TrackMap - - val outputFormats: TrackMap + lateinit var all: TrackMap + lateinit var outputFormats: TrackMap + lateinit var active: TrackMap init { + updateTracksInfo() + } + + fun updateTracksInfo() { val (audioFormat, audioStatus) = resolveTrack(TrackType.AUDIO, strategies.audio, sources.audioOrNull()) val (videoFormat, videoStatus) = resolveTrack(TrackType.VIDEO, strategies.video, sources.videoOrNull()) all = trackMapOf( - video = resolveVideoStatus(videoStatus, forceCompression, videoRotation), - audio = resolveAudioStatus(audioStatus, forceCompression) + video = resolveVideoStatus(videoStatus, forceCompression, videoRotation), + audio = resolveAudioStatus(audioStatus, forceCompression) ) outputFormats = trackMapOf(video = videoFormat, audio = audioFormat) log.i("init: videoStatus=$videoStatus, resolvedVideoStatus=${all.video}, videoFormat=$videoFormat") log.i("init: audioStatus=$audioStatus, resolvedAudioStatus=${all.audio}, audioFormat=$audioFormat") - } - val active: TrackMap = trackMapOf( + active = trackMapOf( video = all.video.takeIf { it.isTranscoding }, audio = all.audio.takeIf { it.isTranscoding } - ) + ) + } + private fun resolveVideoStatus(status: TrackStatus, forceCompression: Boolean, rotation: Int): TrackStatus { val force = forceCompression || rotation != 0 @@ -53,9 +58,9 @@ internal class Tracks( } private fun resolveTrack( - type: TrackType, - strategy: TrackStrategy, - sources: List? // null or not-empty + type: TrackType, + strategy: TrackStrategy, + sources: List? // null or not-empty ): Pair { if (sources == null) { return MediaFormat() to TrackStatus.ABSENT @@ -79,4 +84,4 @@ internal class Tracks( else -> error("Of all $type sources, some have a $type track, some don't.") } } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/AudioEngine.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/AudioEngine.kt index cfc526aa..de3b2362 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/AudioEngine.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/AudioEngine.kt @@ -1,13 +1,20 @@ +@file:Suppress("ReturnCount") + package com.otaliastudios.transcoder.internal.audio import android.media.MediaFormat -import android.media.MediaFormat.* +import android.media.MediaFormat.KEY_CHANNEL_COUNT +import android.media.MediaFormat.KEY_SAMPLE_RATE import android.view.Surface import com.otaliastudios.transcoder.internal.audio.remix.AudioRemixer -import com.otaliastudios.transcoder.internal.codec.* -import com.otaliastudios.transcoder.internal.pipeline.* +import com.otaliastudios.transcoder.internal.codec.DecoderChannel +import com.otaliastudios.transcoder.internal.codec.DecoderData +import com.otaliastudios.transcoder.internal.codec.DecoderTimerData +import com.otaliastudios.transcoder.internal.codec.EncoderChannel +import com.otaliastudios.transcoder.internal.codec.EncoderData +import com.otaliastudios.transcoder.internal.pipeline.QueuedStep +import com.otaliastudios.transcoder.internal.pipeline.State import com.otaliastudios.transcoder.internal.utils.Logger -import com.otaliastudios.transcoder.internal.utils.trackMapOf import com.otaliastudios.transcoder.resample.AudioResampler import com.otaliastudios.transcoder.stretch.AudioStretcher import java.util.concurrent.atomic.AtomicInteger @@ -16,13 +23,13 @@ import kotlin.math.floor /** * Performs audio rendering, from decoder output to encoder input, applying sample rate conversion, - * remixing, stretching. TODO: With some extra work this could be split in different steps. + * remixing, stretching. */ -internal class AudioEngine( - private val stretcher: AudioStretcher, - private val resampler: AudioResampler, - private val targetFormat: MediaFormat -): QueuedStep(), DecoderChannel { +class AudioEngine( + private val stretcher: AudioStretcher, + private val resampler: AudioResampler, + private val targetFormat: MediaFormat +) : QueuedStep(), DecoderChannel { companion object { private val ID = AtomicInteger(0) @@ -46,16 +53,19 @@ internal class AudioEngine( this.rawFormat = rawFormat remixer = AudioRemixer[rawFormat.channels, targetFormat.channels] chunks = ChunkQueue(rawFormat.sampleRate, rawFormat.channels) + resampler.createStream(rawFormat.sampleRate, targetFormat.sampleRate, targetFormat.channels) } override fun enqueueEos(data: DecoderData) { log.i("enqueueEos()") data.release(false) chunks.enqueueEos() + resampler.destroyStream() } override fun enqueue(data: DecoderData) { val stretch = (data as? DecoderTimerData)?.timeStretch ?: 1.0 + log.v("Enqueue: ${data.timeUs} $this") chunks.enqueue(data.buffer.asShortBuffer(), data.timeUs, stretch) { data.release(false) } @@ -72,7 +82,7 @@ internal class AudioEngine( } val outBuffer = outBytes.asShortBuffer() return chunks.drain( - eos = State.Eos(EncoderData(outBytes, outId, 0)) + eos = State.Eos(EncoderData(outBytes, outId, 0)) ) { inBuffer, timeUs, stretch -> val outSize = outBuffer.remaining() val inSize = inBuffer.remaining() @@ -103,16 +113,21 @@ internal class AudioEngine( // Resample resampler.resample( - remixBuffer, rawFormat.sampleRate, - outBuffer, targetFormat.sampleRate, - targetFormat.channels) + remixBuffer, + rawFormat.sampleRate, + outBuffer, + targetFormat.sampleRate, + targetFormat.channels + ) outBuffer.flip() // Adjust position and dispatch. outBytes.clear() outBytes.limit(outBuffer.limit() * BYTES_PER_SHORT) outBytes.position(outBuffer.position() * BYTES_PER_SHORT) + + log.v("Drain: ts: $timeUs size: ${outBuffer.limit()}") State.Ok(EncoderData(outBytes, outId, timeUs)) } } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/SonicAudioEngine.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/SonicAudioEngine.kt new file mode 100644 index 00000000..43bbff51 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/SonicAudioEngine.kt @@ -0,0 +1,103 @@ +@file:Suppress("ReturnCount") + +package com.otaliastudios.transcoder.internal.audio + +import android.media.MediaFormat +import android.media.MediaFormat.KEY_CHANNEL_COUNT +import android.media.MediaFormat.KEY_SAMPLE_RATE +import android.view.Surface +import com.otaliastudios.transcoder.internal.audio.remix.AudioRemixer +import com.otaliastudios.transcoder.internal.codec.DecoderChannel +import com.otaliastudios.transcoder.internal.codec.DecoderData +import com.otaliastudios.transcoder.internal.codec.DecoderTimerData +import com.otaliastudios.transcoder.internal.codec.EncoderChannel +import com.otaliastudios.transcoder.internal.codec.EncoderData +import com.otaliastudios.transcoder.internal.pipeline.QueuedStep +import com.otaliastudios.transcoder.internal.pipeline.State +import com.otaliastudios.transcoder.internal.utils.Logger +import java.util.concurrent.atomic.AtomicInteger + +/** + * Performs audio rendering, from decoder output to encoder input, applying sample rate conversion, + * remixing, stretching using sonic. + */ +class SonicAudioEngine( + private val targetFormat: MediaFormat, + private val stretch : Float = 1.0f +) : QueuedStep(), DecoderChannel { + + companion object { + private val ID = AtomicInteger(0) + } + private val log = Logger("AudioEngine(${ID.getAndIncrement()})") + + override val channel = this + private val buffers = ShortBuffers() + + private val MediaFormat.sampleRate get() = getInteger(KEY_SAMPLE_RATE) + private val MediaFormat.channels get() = getInteger(KEY_CHANNEL_COUNT) + + private lateinit var rawFormat: MediaFormat + private lateinit var chunks: ChunkQueue + private lateinit var remixer: AudioRemixer + private lateinit var sonicExo: SonicAudioProcessor + override fun handleSourceFormat(sourceFormat: MediaFormat): Surface? = null + + override fun handleRawFormat(rawFormat: MediaFormat) { + log.i("handleRawFormat($rawFormat)") + this.rawFormat = rawFormat + remixer = AudioRemixer[rawFormat.channels, targetFormat.channels] + chunks = ChunkQueue(rawFormat.sampleRate, rawFormat.channels) + sonicExo = SonicAudioProcessor(rawFormat.sampleRate, rawFormat.channels, stretch,1f, targetFormat.sampleRate) + } + + override fun enqueueEos(data: DecoderData) { + log.i("enqueueEos()") + data.release(false) + chunks.enqueueEos() + sonicExo.queueEndOfStream() + } + + override fun enqueue(data: DecoderData) { + val stretch = (data as? DecoderTimerData)?.timeStretch ?: 1.0 + log.v("Enqueue: ${data.timeUs} $this") + chunks.enqueue(data.buffer.asShortBuffer(), data.timeUs, stretch) { + data.release(false) + } + } + + override fun drain(): State { + if (chunks.isEmpty()) { + log.i("drain(): no chunks, waiting...") + return State.Wait + } + val (outBytes, outId) = next.buffer() ?: return run { + log.i("drain(): no next buffer, waiting...") + State.Wait + } + val outBuffer = outBytes.asShortBuffer() + return chunks.drain( + eos = State.Eos(EncoderData(outBytes, outId, 0)) + ) { inBuffer, timeUs, stretch -> + + sonicExo.queueInput(inBuffer) + + val stretchBuffer = buffers.acquire("stretch", outBuffer.capacity()) + + sonicExo.getOutput(stretchBuffer) + stretchBuffer.flip() + + // Remix + remixer.remix(stretchBuffer, outBuffer) + outBuffer.flip() + + // Adjust position and dispatch. + outBytes.clear() + outBytes.limit(outBuffer.limit() * BYTES_PER_SHORT) + outBytes.position(outBuffer.position() * BYTES_PER_SHORT) + + log.v("Drain: ts: $timeUs size: ${outBuffer.limit()}") + State.Ok(EncoderData(outBytes, outId, timeUs)) + } + } +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/SonicAudioProcessor.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/SonicAudioProcessor.kt new file mode 100644 index 00000000..730ff884 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/SonicAudioProcessor.kt @@ -0,0 +1,531 @@ +package com.otaliastudios.transcoder.internal.audio + +import java.nio.ShortBuffer +import java.util.Arrays + +/** + * Sonic audio stream processor for time/pitch stretching. + * + * + * Based on https://github.com/waywardgeek/sonic. + */ +/* package */ +internal class SonicAudioProcessor( + private val inputSampleRateHz: Int, + private val channelCount: Int, + private val speed: Float, + private val pitch: Float, + outputSampleRateHz: Int +) { + private val rate: Float + private val minPeriod: Int + private val maxPeriod: Int + private val maxRequiredFrameCount: Int + private val downSampleBuffer: ShortArray + private var inputBuffer: ShortArray + private var inputFrameCount = 0 + private var outputBuffer: ShortArray + private var outputFrameCount = 0 + private var pitchBuffer: ShortArray + private var pitchFrameCount = 0 + private var oldRatePosition = 0 + private var newRatePosition = 0 + private var remainingInputToCopyFrameCount = 0 + private var prevPeriod = 0 + private var prevMinDiff = 0 + private var minDiff = 0 + private var maxDiff = 0 + + /** + * Creates a new Sonic audio stream processor. + * + * @param inputSampleRateHz The sample rate of input audio, in hertz. + * @param channelCount The number of channels in the input audio. + * @param speed The speedup factor for output audio. + * @param pitch The pitch factor for output audio. + * @param outputSampleRateHz The sample rate for output audio, in hertz. + */ + init { + rate = inputSampleRateHz.toFloat() / outputSampleRateHz + minPeriod = inputSampleRateHz / MAXIMUM_PITCH + maxPeriod = inputSampleRateHz / MINIMUM_PITCH + maxRequiredFrameCount = 2 * maxPeriod + downSampleBuffer = ShortArray(maxRequiredFrameCount) + inputBuffer = ShortArray(maxRequiredFrameCount * channelCount) + outputBuffer = ShortArray(maxRequiredFrameCount * channelCount) + pitchBuffer = ShortArray(maxRequiredFrameCount * channelCount) + } + + /** + * Returns the number of bytes that have been input, but will not be processed until more input + * data is provided. + */ + val pendingInputBytes: Int + get() = inputFrameCount * channelCount * BYTES_PER_SAMPLE + + /** + * Queues remaining data from `buffer`, and advances its position by the number of bytes + * consumed. + * + * @param buffer A [ShortBuffer] containing input data between its position and limit. + */ + fun queueInput(buffer: ShortBuffer) { + val framesToWrite = buffer.remaining() / channelCount + val bytesToWrite = framesToWrite * channelCount * 2 + inputBuffer = ensureSpaceForAdditionalFrames(inputBuffer, inputFrameCount, framesToWrite) + buffer[inputBuffer, inputFrameCount * channelCount, bytesToWrite / 2] + inputFrameCount += framesToWrite + processStreamInput() + } + + /** + * Gets available output, outputting to the start of `buffer`. The buffer's position will be + * advanced by the number of bytes written. + * + * @param buffer A [ShortBuffer] into which output will be written. + */ + fun getOutput(buffer: ShortBuffer) { + val framesToRead = Math.min(buffer.remaining() / channelCount, outputFrameCount) + buffer.put(outputBuffer, 0, framesToRead * channelCount) + outputFrameCount -= framesToRead + System.arraycopy( + outputBuffer, + framesToRead * channelCount, + outputBuffer, + 0, + outputFrameCount * channelCount + ) + } + + /** + * Forces generating output using whatever data has been queued already. No extra delay will be + * added to the output, but flushing in the middle of words could introduce distortion. + */ + fun queueEndOfStream() { + val remainingFrameCount = inputFrameCount + val s = speed / pitch + val r = rate * pitch + val expectedOutputFrames = + outputFrameCount + ((remainingFrameCount / s + pitchFrameCount) / r + 0.5f).toInt() + + // Add enough silence to flush both input and pitch buffers. + inputBuffer = ensureSpaceForAdditionalFrames( + inputBuffer, inputFrameCount, remainingFrameCount + 2 * maxRequiredFrameCount + ) + for (xSample in 0 until 2 * maxRequiredFrameCount * channelCount) { + inputBuffer[remainingFrameCount * channelCount + xSample] = 0 + } + inputFrameCount += 2 * maxRequiredFrameCount + processStreamInput() + // Throw away any extra frames we generated due to the silence we added. + if (outputFrameCount > expectedOutputFrames) { + outputFrameCount = expectedOutputFrames + } + // Empty input and pitch buffers. + inputFrameCount = 0 + remainingInputToCopyFrameCount = 0 + pitchFrameCount = 0 + } + + /** Clears state in preparation for receiving a new stream of input buffers. */ + fun flush() { + inputFrameCount = 0 + outputFrameCount = 0 + pitchFrameCount = 0 + oldRatePosition = 0 + newRatePosition = 0 + remainingInputToCopyFrameCount = 0 + prevPeriod = 0 + prevMinDiff = 0 + minDiff = 0 + maxDiff = 0 + } + + /** Returns the size of output that can be read with [.getOutput], in bytes. */ + val outputSize: Int + get() = outputFrameCount * channelCount * BYTES_PER_SAMPLE + // Internal methods. + /** + * Returns `buffer` or a copy of it, such that there is enough space in the returned buffer + * to store `newFrameCount` additional frames. + * + * @param buffer The buffer. + * @param frameCount The number of frames already in the buffer. + * @param additionalFrameCount The number of additional frames that need to be stored in the + * buffer. + * @return A buffer with enough space for the additional frames. + */ + private fun ensureSpaceForAdditionalFrames( + buffer: ShortArray, frameCount: Int, additionalFrameCount: Int + ): ShortArray { + val currentCapacityFrames = buffer.size / channelCount + return if (frameCount + additionalFrameCount <= currentCapacityFrames) { + buffer + } else { + val newCapacityFrames = 3 * currentCapacityFrames / 2 + additionalFrameCount + Arrays.copyOf(buffer, newCapacityFrames * channelCount) + } + } + + private fun removeProcessedInputFrames(positionFrames: Int) { + val remainingFrames = inputFrameCount - positionFrames + System.arraycopy( + inputBuffer, + positionFrames * channelCount, + inputBuffer, + 0, + remainingFrames * channelCount + ) + inputFrameCount = remainingFrames + } + + private fun copyToOutput(samples: ShortArray, positionFrames: Int, frameCount: Int) { + outputBuffer = ensureSpaceForAdditionalFrames(outputBuffer, outputFrameCount, frameCount) + System.arraycopy( + samples, + positionFrames * channelCount, + outputBuffer, + outputFrameCount * channelCount, + frameCount * channelCount + ) + outputFrameCount += frameCount + } + + private fun copyInputToOutput(positionFrames: Int): Int { + val frameCount = Math.min(maxRequiredFrameCount, remainingInputToCopyFrameCount) + copyToOutput(inputBuffer, positionFrames, frameCount) + remainingInputToCopyFrameCount -= frameCount + return frameCount + } + + private fun downSampleInput(samples: ShortArray, position: Int, skip: Int) { + // If skip is greater than one, average skip samples together and write them to the down-sample + // buffer. If channelCount is greater than one, mix the channels together as we down sample. + var position = position + val frameCount = maxRequiredFrameCount / skip + val samplesPerValue = channelCount * skip + position *= channelCount + for (i in 0 until frameCount) { + var value = 0 + for (j in 0 until samplesPerValue) { + value += samples[position + i * samplesPerValue + j].toInt() + } + value /= samplesPerValue + downSampleBuffer[i] = value.toShort() + } + } + + private fun findPitchPeriodInRange( + samples: ShortArray, + position: Int, + minPeriod: Int, + maxPeriod: Int + ): Int { + // Find the best frequency match in the range, and given a sample skip multiple. For now, just + // find the pitch of the first channel. + var position = position + var bestPeriod = 0 + var worstPeriod = 255 + var minDiff = 1 + var maxDiff = 0 + position *= channelCount + for (period in minPeriod..maxPeriod) { + var diff = 0 + for (i in 0 until period) { + val sVal = samples[position + i] + val pVal = samples[position + period + i] + diff += Math.abs(sVal - pVal) + } + // Note that the highest number of samples we add into diff will be less than 256, since we + // skip samples. Thus, diff is a 24 bit number, and we can safely multiply by numSamples + // without overflow. + if (diff * bestPeriod < minDiff * period) { + minDiff = diff + bestPeriod = period + } + if (diff * worstPeriod > maxDiff * period) { + maxDiff = diff + worstPeriod = period + } + } + this.minDiff = minDiff / bestPeriod + this.maxDiff = maxDiff / worstPeriod + return bestPeriod + } + + /** + * Returns whether the previous pitch period estimate is a better approximation, which can occur + * at the abrupt end of voiced words. + */ + private fun previousPeriodBetter(minDiff: Int, maxDiff: Int): Boolean { + if (minDiff == 0 || prevPeriod == 0) { + return false + } + if (maxDiff > minDiff * 3) { + // Got a reasonable match this period. + return false + } + return if (minDiff * 2 <= prevMinDiff * 3) { + // Mismatch is not that much greater this period. + false + } else true + } + + private fun findPitchPeriod(samples: ShortArray, position: Int): Int { + // Find the pitch period. This is a critical step, and we may have to try multiple ways to get a + // good answer. This version uses AMDF. To improve speed, we down sample by an integer factor + // get in the 11 kHz range, and then do it again with a narrower frequency range without down + // sampling. + var period: Int + val retPeriod: Int + val skip = if (inputSampleRateHz > AMDF_FREQUENCY) inputSampleRateHz / AMDF_FREQUENCY else 1 + if (channelCount == 1 && skip == 1) { + period = findPitchPeriodInRange(samples, position, minPeriod, maxPeriod) + } else { + downSampleInput(samples, position, skip) + period = findPitchPeriodInRange(downSampleBuffer, 0, minPeriod / skip, maxPeriod / skip) + if (skip != 1) { + period *= skip + var minP = period - skip * 4 + var maxP = period + skip * 4 + if (minP < minPeriod) { + minP = minPeriod + } + if (maxP > maxPeriod) { + maxP = maxPeriod + } + period = if (channelCount == 1) { + findPitchPeriodInRange(samples, position, minP, maxP) + } else { + downSampleInput(samples, position, 1) + findPitchPeriodInRange(downSampleBuffer, 0, minP, maxP) + } + } + } + retPeriod = if (previousPeriodBetter(minDiff, maxDiff)) { + prevPeriod + } else { + period + } + prevMinDiff = minDiff + prevPeriod = period + return retPeriod + } + + private fun moveNewSamplesToPitchBuffer(originalOutputFrameCount: Int) { + val frameCount = outputFrameCount - originalOutputFrameCount + pitchBuffer = ensureSpaceForAdditionalFrames(pitchBuffer, pitchFrameCount, frameCount) + System.arraycopy( + outputBuffer, + originalOutputFrameCount * channelCount, + pitchBuffer, + pitchFrameCount * channelCount, + frameCount * channelCount + ) + outputFrameCount = originalOutputFrameCount + pitchFrameCount += frameCount + } + + private fun removePitchFrames(frameCount: Int) { + if (frameCount == 0) { + return + } + System.arraycopy( + pitchBuffer, + frameCount * channelCount, + pitchBuffer, + 0, + (pitchFrameCount - frameCount) * channelCount + ) + pitchFrameCount -= frameCount + } + + private fun interpolate( + `in`: ShortArray, + inPos: Int, + oldSampleRate: Int, + newSampleRate: Int + ): Short { + val left = `in`[inPos] + val right = `in`[inPos + channelCount] + val position = newRatePosition * oldSampleRate + val leftPosition = oldRatePosition * newSampleRate + val rightPosition = (oldRatePosition + 1) * newSampleRate + val ratio = rightPosition - position + val width = rightPosition - leftPosition + return ((ratio * left + (width - ratio) * right) / width).toShort() + } + + private fun adjustRate(rate: Float, originalOutputFrameCount: Int) { + if (outputFrameCount == originalOutputFrameCount) { + return + } + var newSampleRate = (inputSampleRateHz / rate).toInt() + var oldSampleRate = inputSampleRateHz + // Set these values to help with the integer math. + while (newSampleRate > 1 shl 14 || oldSampleRate > 1 shl 14) { + newSampleRate /= 2 + oldSampleRate /= 2 + } + moveNewSamplesToPitchBuffer(originalOutputFrameCount) + // Leave at least one pitch sample in the buffer. + for (position in 0 until pitchFrameCount - 1) { + while ((oldRatePosition + 1) * newSampleRate > newRatePosition * oldSampleRate) { + outputBuffer = ensureSpaceForAdditionalFrames( + outputBuffer, outputFrameCount, /* additionalFrameCount= */1 + ) + for (i in 0 until channelCount) { + outputBuffer[outputFrameCount * channelCount + i] = interpolate( + pitchBuffer, + position * channelCount + i, + oldSampleRate, + newSampleRate + ) + } + newRatePosition++ + outputFrameCount++ + } + oldRatePosition++ + if (oldRatePosition == oldSampleRate) { + oldRatePosition = 0 + // Assertions.checkState(newRatePosition == newSampleRate); + newRatePosition = 0 + } + } + removePitchFrames(pitchFrameCount - 1) + } + + private fun skipPitchPeriod( + samples: ShortArray, + position: Int, + speed: Float, + period: Int + ): Int { + // Skip over a pitch period, and copy period/speed samples to the output. + val newFrameCount: Int + if (speed >= 2.0f) { + newFrameCount = (period / (speed - 1.0f)).toInt() + } else { + newFrameCount = period + remainingInputToCopyFrameCount = (period * (2.0f - speed) / (speed - 1.0f)).toInt() + } + outputBuffer = ensureSpaceForAdditionalFrames(outputBuffer, outputFrameCount, newFrameCount) + overlapAdd( + newFrameCount, + channelCount, + outputBuffer, + outputFrameCount, + samples, + position, + samples, + position + period + ) + outputFrameCount += newFrameCount + return newFrameCount + } + + private fun insertPitchPeriod( + samples: ShortArray, + position: Int, + speed: Float, + period: Int + ): Int { + // Insert a pitch period, and determine how much input to copy directly. + val newFrameCount: Int + if (speed < 0.5f) { + newFrameCount = (period * speed / (1.0f - speed)).toInt() + } else { + newFrameCount = period + remainingInputToCopyFrameCount = + (period * (2.0f * speed - 1.0f) / (1.0f - speed)).toInt() + } + outputBuffer = + ensureSpaceForAdditionalFrames(outputBuffer, outputFrameCount, period + newFrameCount) + System.arraycopy( + samples, + position * channelCount, + outputBuffer, + outputFrameCount * channelCount, + period * channelCount + ) + overlapAdd( + newFrameCount, + channelCount, + outputBuffer, + outputFrameCount + period, + samples, + position + period, + samples, + position + ) + outputFrameCount += period + newFrameCount + return newFrameCount + } + + private fun changeSpeed(speed: Float) { + if (inputFrameCount < maxRequiredFrameCount) { + return + } + val frameCount = inputFrameCount + var positionFrames = 0 + do { + positionFrames += if (remainingInputToCopyFrameCount > 0) { + copyInputToOutput(positionFrames) + } else { + val period = findPitchPeriod(inputBuffer, positionFrames) + if (speed > 1.0) { + period + skipPitchPeriod(inputBuffer, positionFrames, speed, period) + } else { + insertPitchPeriod(inputBuffer, positionFrames, speed, period) + } + } + } while (positionFrames + maxRequiredFrameCount <= frameCount) + removeProcessedInputFrames(positionFrames) + } + + private fun processStreamInput() { + // Resample as many pitch periods as we have buffered on the input. + val originalOutputFrameCount = outputFrameCount + val s = speed / pitch + val r = rate * pitch + if (s > 1.00001 || s < 0.99999) { + changeSpeed(s) + } else { + copyToOutput(inputBuffer, 0, inputFrameCount) + inputFrameCount = 0 + } + if (r != 1.0f) { + adjustRate(r, originalOutputFrameCount) + } + } + + companion object { + private const val MINIMUM_PITCH = 65 + private const val MAXIMUM_PITCH = 400 + private const val AMDF_FREQUENCY = 4000 + private const val BYTES_PER_SAMPLE = 2 + private fun overlapAdd( + frameCount: Int, + channelCount: Int, + out: ShortArray, + outPosition: Int, + rampDown: ShortArray, + rampDownPosition: Int, + rampUp: ShortArray, + rampUpPosition: Int + ) { + for (i in 0 until channelCount) { + var o = outPosition * channelCount + i + var u = rampUpPosition * channelCount + i + var d = rampDownPosition * channelCount + i + for (t in 0 until frameCount) { + out[o] = + ((rampDown[d] * (frameCount - t) + rampUp[u] * t) / frameCount).toShort() + o += channelCount + d += channelCount + u += channelCount + } + } + } + } +} \ No newline at end of file diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/chunks.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/chunks.kt index 52eaa357..517a4b1c 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/chunks.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/chunks.kt @@ -1,12 +1,13 @@ package com.otaliastudios.transcoder.internal.audio +import android.util.Log import java.nio.ShortBuffer private data class Chunk( - val buffer: ShortBuffer, - val timeUs: Long, - val timeStretch: Double, - val release: () -> Unit + val buffer: ShortBuffer, + val timeUs: Long, + val timeStretch: Double, + val release: () -> Unit ) { companion object { val Eos = Chunk(ShortBuffer.allocate(0), 0, 0.0, {}) @@ -19,13 +20,17 @@ private data class Chunk( * big enough to contain the full processed size, in which case we want to consume only * part of the input buffer and keep it available for the next cycle. */ -internal class ChunkQueue(private val sampleRate: Int, private val channels: Int) { +class ChunkQueue(private val sampleRate: Int, private val channels: Int) { private val queue = ArrayDeque() fun isEmpty() = queue.isEmpty() fun enqueue(buffer: ShortBuffer, timeUs: Long, timeStretch: Double, release: () -> Unit) { - require(buffer.hasRemaining()) + if (!buffer.hasRemaining()) { + Log.w("ChunkQueue", "enqueue: " + "buffer is empty") + } +// require(buffer.hasRemaining()) + queue.addLast(Chunk(buffer, timeUs, timeStretch, release)) } @@ -44,9 +49,11 @@ internal class ChunkQueue(private val sampleRate: Int, private val channels: Int head.buffer.limit(limit) if (head.buffer.hasRemaining()) { val consumed = size - head.buffer.remaining() - queue.addFirst(head.copy( + queue.addFirst( + head.copy( timeUs = shortsToUs(consumed, sampleRate, channels) - )) + ) + ) } else { // buffer consumed! head.release() diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/conversions.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/conversions.kt index 6ca67e3f..dd9e7f19 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/conversions.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/conversions.kt @@ -1,3 +1,5 @@ +@file:Suppress("MagicNumber") + package com.otaliastudios.transcoder.internal.audio import kotlin.math.ceil @@ -6,21 +8,21 @@ private const val BYTES_PER_SAMPLE_PER_CHANNEL = 2 // Assuming 16bit audio, so 2 private const val MICROSECONDS_PER_SECOND = 1000000L internal fun bytesToUs( - bytes: Int /* bytes */, - sampleRate: Int /* samples/sec */, - channels: Int /* channel */ + bytes: Int /* bytes */, + sampleRate: Int /* samples/sec */, + channels: Int /* channel */ ): Long { val byteRatePerChannel = sampleRate * BYTES_PER_SAMPLE_PER_CHANNEL // bytes/sec/channel val byteRate = byteRatePerChannel * channels // bytes/sec return MICROSECONDS_PER_SECOND * bytes / byteRate // usec } -internal fun bitRate(sampleRate: Int, channels: Int): Int { +fun bitRate(sampleRate: Int, channels: Int): Int { val byteRate = channels * sampleRate * BYTES_PER_SAMPLE_PER_CHANNEL return byteRate * 8 } -internal fun samplesToBytes(samples: Int, channels: Int): Int { +fun samplesToBytes(samples: Int, channels: Int): Int { val bytesPerSample = BYTES_PER_SAMPLE_PER_CHANNEL * channels return samples * bytesPerSample } @@ -37,4 +39,4 @@ internal fun shortsToUs(shorts: Int, sampleRate: Int, channels: Int): Long { internal fun usToShorts(us: Long, sampleRate: Int, channels: Int): Int { return usToBytes(us, sampleRate, channels) / BYTES_PER_SHORT -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/remix/AudioRemixer.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/remix/AudioRemixer.kt index fac8b7cd..3e4ccabf 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/remix/AudioRemixer.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/remix/AudioRemixer.kt @@ -6,7 +6,7 @@ import java.nio.ShortBuffer * Remixes audio data. See [DownMixAudioRemixer], [UpMixAudioRemixer] or [PassThroughAudioRemixer] * for concrete implementations. */ -internal interface AudioRemixer { +interface AudioRemixer { /** * Remixes input audio from input buffer into the output buffer. @@ -30,4 +30,4 @@ internal interface AudioRemixer { else -> PassThroughAudioRemixer() } } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/shorts.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/shorts.kt index 80959159..2c237be5 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/shorts.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/audio/shorts.kt @@ -6,15 +6,15 @@ import java.nio.ShortBuffer internal const val BYTES_PER_SHORT = 2 -internal class ShortBuffers { +class ShortBuffers { private val map = mutableMapOf() fun acquire(name: String, size: Int): ShortBuffer { var current = map[name] if (current == null || current.capacity() < size) { current = ByteBuffer.allocateDirect(size * BYTES_PER_SHORT) - .order(ByteOrder.nativeOrder()) - .asShortBuffer() + .order(ByteOrder.nativeOrder()) + .asShortBuffer() } current!!.clear() current.limit(size) @@ -22,4 +22,4 @@ internal class ShortBuffers { map[name] = current } } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/Decoder.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/Decoder.kt index fba72dd9..cbf22683 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/Decoder.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/Decoder.kt @@ -1,13 +1,17 @@ package com.otaliastudios.transcoder.internal.codec +import android.media.MediaCodec import android.media.MediaCodec.* +import android.media.MediaCodecInfo +import android.media.MediaCodecList import android.media.MediaFormat +import android.os.Build import android.view.Surface +import com.otaliastudios.transcoder.BuildConfig import com.otaliastudios.transcoder.common.trackType import com.otaliastudios.transcoder.internal.data.ReaderChannel import com.otaliastudios.transcoder.internal.data.ReaderData import com.otaliastudios.transcoder.internal.media.MediaCodecBuffers -import com.otaliastudios.transcoder.internal.pipeline.BaseStep import com.otaliastudios.transcoder.internal.pipeline.Channel import com.otaliastudios.transcoder.internal.pipeline.QueuedStep import com.otaliastudios.transcoder.internal.pipeline.State @@ -15,36 +19,107 @@ import com.otaliastudios.transcoder.internal.utils.Logger import com.otaliastudios.transcoder.internal.utils.trackMapOf import java.nio.ByteBuffer import java.util.concurrent.atomic.AtomicInteger -import kotlin.properties.Delegates import kotlin.properties.Delegates.observable - -internal open class DecoderData( - val buffer: ByteBuffer, - val timeUs: Long, - val release: (render: Boolean) -> Unit +open class DecoderData( + val buffer: ByteBuffer, + val timeUs: Long, + val release: (render: Boolean) -> Unit ) -internal interface DecoderChannel : Channel { +interface DecoderChannel : Channel { fun handleSourceFormat(sourceFormat: MediaFormat): Surface? fun handleRawFormat(rawFormat: MediaFormat) } -internal class Decoder( - private val format: MediaFormat, // source.getTrackFormat(track) - continuous: Boolean, // relevant if the source sends no-render chunks. should we compensate or not? +class Decoder( + private val format: MediaFormat, // source.getTrackFormat(track) + continuous: Boolean, // relevant if the source sends no-render chunks. should we compensate or not? + private val useSwFor4K: Boolean = false, + private val eventListener: TranscoderEventsListener? = null, + val shouldFlush: (() -> Boolean)? = null ) : QueuedStep(), ReaderChannel { companion object { private val ID = trackMapOf(AtomicInteger(0), AtomicInteger(0)) + private const val timeoutUs = 2000L + private const val VERBOSE = false } + private var retry: Boolean = true + private val log = Logger("Decoder(${format.trackType},${ID[format.trackType].getAndIncrement()})") override val channel = this - private val codec = createDecoderByType(format.getString(MediaFormat.KEY_MIME)!!) + private var codec = createDecoderByType(format, useSwFor4K && format.is4K()) + + private var decoderReady = false + + @Suppress("MagicNumber") + private fun MediaFormat.is4K(): Boolean { + val width = format.getInteger(MediaFormat.KEY_WIDTH) + val height = format.getInteger(MediaFormat.KEY_HEIGHT) + return width * height > 2120 * 2120 + } + + private fun MediaCodecInfo.isHardwareAcceleratedCompat() : Boolean { + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + isHardwareAccelerated + } else { + val codecName = name.lowercase() + val isSoftware = (codecName.startsWith("omx.google.") + || codecName.startsWith("omx.ffmpeg.") + || (codecName.startsWith("omx.sec.") && codecName.contains(".sw.")) + || codecName == "omx.qcom.video.decoder.hevcswvdec" + || codecName.startsWith("c2.android.") + || codecName.startsWith("c2.google.") + || (!codecName.startsWith("omx.") && !codecName.startsWith("c2."))) + + if (isSoftware) { + log.i("sw codec: $name") + } + !isSoftware + } + } + + private fun createDecoderByType(format: MediaFormat, useSoftware: Boolean = false): MediaCodec { + val mime = format.getString(MediaFormat.KEY_MIME)!! + + val allCodecs = MediaCodecList(MediaCodecList.ALL_CODECS) + + var codecName: String? = null + for (info in allCodecs.codecInfos) { + if (info.isEncoder || (info.isHardwareAcceleratedCompat() && useSoftware)) { + // log.e("Rejecting codec: ${info.name}") + continue + } + try { + val caps = info.getCapabilitiesForType(mime) + if (caps != null && caps.isFormatSupported(format)) { + codecName = info.name + break + } else { + // log.e("Rejecting decoder: ${info.name}") + } + } catch (e: IllegalArgumentException) { + log.e("Unsupported codec type: $mime") + } + } + log.i("Using codec: $codecName for format: $format") + try { + if (codecName != null) { + return createByCodecName(codecName) + } + } catch (e: Exception) { + e.printStackTrace() + log.e("Exception while creating codec by name: $codecName :: ${e.message}") + return createDecoderByType(mime) + } + return createDecoderByType(mime) + } + private val buffers by lazy { MediaCodecBuffers(codec) } - private var info = BufferInfo() + private var info = MediaCodec.BufferInfo() private val dropper = DecoderDropper(continuous) private var dequeuedInputs by observable(0) { _, _, _ -> printDequeued() } @@ -55,14 +130,83 @@ internal class Decoder( override fun initialize(next: DecoderChannel) { super.initialize(next) - log.i("initialize()") + val videoCapabilities = codec.codecInfo.getCapabilitiesForType( + format.getString( + MediaFormat.KEY_MIME + )!! + ).videoCapabilities + if (videoCapabilities != null) { + log.i( + "initialize(): ${codec.codecInfo.name}, for format $format, " + + "supportedHeightRange ${videoCapabilities.supportedHeights} " + + "supportedWidthRange ${videoCapabilities.supportedWidths}" + ) + } val surface = next.handleSourceFormat(format) - codec.configure(format, surface, null, 0) - codec.start() + try { + codec.configure(format, surface, null, 0) + } + catch (e: Exception) { + eventListener?.onDecoderConfigureFailure(codec.name, format, e) + if(BuildConfig.DEBUG) { + log.e("Failed while configuring codec ${codec.name} for format $format") + logCodecException(e) + } + if (retry) { + retry = false + val mime = format.getString(MediaFormat.KEY_MIME)!! + codec = createDecoderByType(mime) + initialize(next) + return + } + } + try { + codec.start() + } + catch (e: Exception) { + eventListener?.onDecoderStartFailure(codec.name, format, e) + if(BuildConfig.DEBUG) { + log.e("Failed while starting codec ${codec.name} for format $format") + logCodecException(e) + } + throw e + } + decoderReady = false + } + + private fun logCodecException(e: Exception) { + when(e) { + is CodecException -> { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + log.e("CodecException: " + + "diagnosticInfo: ${e.diagnosticInfo}, " + + "isRecoverable: ${e.isRecoverable}, " + + "isTransient: ${e.isTransient}, " + + "errorCode:${e.errorCode}") + } + else { + log.e("CodecException: " + + "diagnosticInfo: ${e.diagnosticInfo}, " + + "isRecoverable: ${e.isRecoverable}, " + + "isTransient: ${e.isTransient}") + + } + } + is java.lang.IllegalArgumentException -> { + log.e("IllegalArgumentException: ${e.message}") + } + is java.lang.IllegalStateException -> { + log.e("IllegalStateException: ${e.message}") + } + } } override fun buffer(): Pair? { - val id = codec.dequeueInputBuffer(0) + if (shouldFlush?.invoke() == true && decoderReady) { + log.i("codec flush") + codec.flush() + } + val id = codec.dequeueInputBuffer(timeoutUs) return if (id >= 0) { dequeuedInputs++ buffers.getInputBuffer(id) to id @@ -83,19 +227,26 @@ internal class Decoder( dequeuedInputs-- val (chunk, id) = data val flag = if (chunk.keyframe) BUFFER_FLAG_SYNC_FRAME else 0 + if(VERBOSE) { + log.v("enqueue(): queueInputBuffer ${chunk.timeUs}") + } codec.queueInputBuffer(id, chunk.buffer.position(), chunk.buffer.remaining(), chunk.timeUs, flag) dropper.input(chunk.timeUs, chunk.render) } override fun drain(): State { - val result = codec.dequeueOutputBuffer(info, 0) + val result = codec.dequeueOutputBuffer(info, timeoutUs) return when (result) { INFO_TRY_AGAIN_LATER -> { log.i("drain(): got INFO_TRY_AGAIN_LATER, waiting.") State.Wait } INFO_OUTPUT_FORMAT_CHANGED -> { - log.i("drain(): got INFO_OUTPUT_FORMAT_CHANGED, handling format and retrying. format=${codec.outputFormat}") + log.i( + "drain(): got INFO_OUTPUT_FORMAT_CHANGED," + + " handling format and retrying. format=${codec.outputFormat}" + ) + decoderReady = true next.handleRawFormat(codec.outputFormat) State.Retry } @@ -110,12 +261,20 @@ internal class Decoder( if (timeUs != null /* && (isEos || info.size > 0) */) { dequeuedOutputs++ val buffer = buffers.getOutputBuffer(result) + // Ideally, we shouldn't rely on the fact that the buffer is properly configured. + // We should configure its position and limit based on the buffer info's position and size. val data = DecoderData(buffer, timeUs) { + if(VERBOSE) { + log.v("drain(): released successfully presentation ts ${info.presentationTimeUs} and $timeUs") + } codec.releaseOutputBuffer(result, it) dequeuedOutputs-- } if (isEos) State.Eos(data) else State.Ok(data) } else { + if(VERBOSE) { + log.v("drain(): released because decoder dropper gave null ts ${info.presentationTimeUs}") + } codec.releaseOutputBuffer(result, false) State.Wait }.also { @@ -127,7 +286,12 @@ internal class Decoder( override fun release() { log.i("release(): releasing codec. dequeuedInputs=$dequeuedInputs dequeuedOutputs=$dequeuedOutputs") - codec.stop() - codec.release() + try { + codec.stop() + codec.release() + } + catch (e : Exception) { + eventListener?.onDecoderReleaseFailure(codec.name, format, e) + } } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/DecoderDropper.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/DecoderDropper.kt index 9809b2a4..6f6d6944 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/DecoderDropper.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/DecoderDropper.kt @@ -1,3 +1,5 @@ +@file:Suppress("ReturnCount") + package com.otaliastudios.transcoder.internal.codec import com.otaliastudios.transcoder.internal.utils.Logger @@ -21,7 +23,7 @@ import com.otaliastudios.transcoder.source.DataSource * is screwed. Also, if the source jumps forward using seek, we won't catch the jump. This class * catches discontinuities only through changes in the render boolean passed to [input]. */ -internal class DecoderDropper(private val continuous: Boolean) { +class DecoderDropper(private val continuous: Boolean) { private val log = Logger("DecoderDropper") private val closedDeltas = mutableMapOf() @@ -32,11 +34,11 @@ internal class DecoderDropper(private val continuous: Boolean) { private var firstOutputUs: Long? = null private fun debug(message: String, important: Boolean = false) { - /* val full = "$message firstInputUs=$firstInputUs " + - "validInputUs=[${closedRanges.joinToString { - "$it(deltaUs=${closedDeltas[it]})" - }}] pendingRangeUs=${pendingRange}" - if (important) log.w(full) else log.v(full) */ + val full = "$message firstInputUs=$firstInputUs " + + "validInputUs=[${closedRanges.joinToString { + "$it(deltaUs=${closedDeltas[it]})" + }}] pendingRangeUs=$pendingRange" + if (important) log.w(full) else log.v(full) } fun input(timeUs: Long, render: Boolean) { @@ -46,8 +48,8 @@ internal class DecoderDropper(private val continuous: Boolean) { if (render) { debug("INPUT: inputUs=$timeUs") // log.v("TDBG inputUs=$timeUs") - if (pendingRange == null) pendingRange = timeUs..Long.MAX_VALUE - else pendingRange = pendingRange!!.first..timeUs + pendingRange = if (pendingRange == null) timeUs..Long.MAX_VALUE + else pendingRange!!.first.coerceAtMost(timeUs)..timeUs } else { debug("INPUT: Got SKIPPING input! inputUs=$timeUs") if (pendingRange != null && pendingRange!!.last != Long.MAX_VALUE) { @@ -89,4 +91,4 @@ internal class DecoderDropper(private val continuous: Boolean) { debug("OUTPUT: SKIPPING! outputTimeUs=$timeUs", important = true) return null } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/DecoderTimer.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/DecoderTimer.kt index 4c9459a6..155b84b1 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/DecoderTimer.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/DecoderTimer.kt @@ -6,17 +6,17 @@ import com.otaliastudios.transcoder.internal.pipeline.State import com.otaliastudios.transcoder.time.TimeInterpolator import java.nio.ByteBuffer -internal class DecoderTimerData( - buffer: ByteBuffer, - val rawTimeUs: Long, - timeUs: Long, - val timeStretch: Double, - release: (render: Boolean) -> Unit +class DecoderTimerData( + buffer: ByteBuffer, + val rawTimeUs: Long, + timeUs: Long, + val timeStretch: Double, + release: (render: Boolean) -> Unit ) : DecoderData(buffer, timeUs, release) -internal class DecoderTimer( - private val track: TrackType, - private val interpolator: TimeInterpolator, +class DecoderTimer( + private val track: TrackType, + private val interpolator: TimeInterpolator, ) : DataStep() { private var lastTimeUs: Long? = null @@ -41,12 +41,14 @@ internal class DecoderTimer( lastTimeUs = timeUs lastRawTimeUs = rawTimeUs - return State.Ok(DecoderTimerData( + return State.Ok( + DecoderTimerData( buffer = state.value.buffer, rawTimeUs = rawTimeUs, timeUs = timeUs, timeStretch = timeStretch, release = state.value.release - )) + ) + ) } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/Encoder.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/Encoder.kt index 730c5633..87a63e59 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/Encoder.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/Encoder.kt @@ -1,10 +1,15 @@ +@file:Suppress("MagicNumber") + package com.otaliastudios.transcoder.internal.codec import android.media.MediaCodec -import android.media.MediaCodec.* +import android.media.MediaCodec.BUFFER_FLAG_CODEC_CONFIG +import android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM +import android.media.MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED +import android.media.MediaCodec.INFO_OUTPUT_FORMAT_CHANGED +import android.media.MediaCodec.INFO_TRY_AGAIN_LATER import android.view.Surface import com.otaliastudios.transcoder.common.TrackType -import com.otaliastudios.transcoder.common.trackType import com.otaliastudios.transcoder.internal.Codecs import com.otaliastudios.transcoder.internal.data.WriterChannel import com.otaliastudios.transcoder.internal.data.WriterData @@ -16,34 +21,33 @@ import com.otaliastudios.transcoder.internal.utils.Logger import com.otaliastudios.transcoder.internal.utils.trackMapOf import java.nio.ByteBuffer import java.util.concurrent.atomic.AtomicInteger -import kotlin.properties.Delegates import kotlin.properties.Delegates.observable -internal data class EncoderData( - val buffer: ByteBuffer?, // If present, it must have correct position/remaining! - val id: Int, - val timeUs: Long +data class EncoderData( + val buffer: ByteBuffer?, // If present, it must have correct position/remaining! + val id: Int, + val timeUs: Long ) { companion object { val Empty = EncoderData(null, 0, 0L) } } -internal interface EncoderChannel : Channel { +interface EncoderChannel : Channel { val surface: Surface? fun buffer(): Pair? } -internal class Encoder( - private val codec: MediaCodec, - override val surface: Surface?, - ownsCodecStart: Boolean, - private val ownsCodecStop: Boolean, +class Encoder( + private val codec: MediaCodec, + override val surface: Surface?, + ownsCodecStart: Boolean, + private val ownsCodecStop: Boolean, ) : QueuedStep(), EncoderChannel { constructor(codecs: Codecs, type: TrackType) : this( - codecs.encoders[type].first, - codecs.encoders[type].second, - codecs.ownsEncoderStart[type], - codecs.ownsEncoderStop[type] + codecs.encoders[type].first, + codecs.encoders[type].second, + codecs.ownsEncoderStart[type], + codecs.ownsEncoderStop[type] ) companion object { @@ -51,7 +55,7 @@ internal class Encoder( } private val type = if (surface != null) TrackType.VIDEO else TrackType.AUDIO - private val log = Logger("Encoder(${type},${ID[type].getAndIncrement()})") + private val log = Logger("Encoder($type,${ID[type].getAndIncrement()})") private var dequeuedInputs by observable(0) { _, _, _ -> printDequeued() } private var dequeuedOutputs by observable(0) { _, _, _ -> printDequeued() } private fun printDequeued() { @@ -62,8 +66,7 @@ internal class Encoder( private val buffers by lazy { MediaCodecBuffers(codec) } - private var info = BufferInfo() - + private var info = MediaCodec.BufferInfo() init { log.i("Encoder: ownsStart=$ownsCodecStart ownsStop=$ownsCodecStop") @@ -140,7 +143,12 @@ internal class Encoder( val isEos = info.flags and BUFFER_FLAG_END_OF_STREAM != 0 val flags = info.flags and BUFFER_FLAG_END_OF_STREAM.inv() val buffer = buffers.getOutputBuffer(result) - val timeUs = info.presentationTimeUs + val timeUs = if (info.presentationTimeUs < 0) 0 else info.presentationTimeUs + if (isEos && timeUs == 0L) { + info.offset = 0 + info.size = 0 + info.presentationTimeUs = 0 + } buffer.clear() buffer.limit(info.offset + info.size) buffer.position(info.offset) @@ -155,9 +163,9 @@ internal class Encoder( } override fun release() { - log.i("release(): ownsStop=$ownsCodecStop dequeuedInputs=${dequeuedInputs} dequeuedOutputs=$dequeuedOutputs") + log.i("release(): ownsStop=$ownsCodecStop dequeuedInputs=$dequeuedInputs dequeuedOutputs=$dequeuedOutputs") if (ownsCodecStop) { codec.stop() } } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/TranscoderEventsListener.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/TranscoderEventsListener.kt new file mode 100644 index 00000000..bfa07afc --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/codec/TranscoderEventsListener.kt @@ -0,0 +1,13 @@ +package com.otaliastudios.transcoder.internal.codec + +import android.media.MediaFormat + +interface TranscoderEventsListener { + + fun onDecoderConfigureFailure(codecName: String, format: MediaFormat, exception: Exception) + + fun onDecoderStartFailure(codecName: String, format: MediaFormat, exception: Exception) + + fun onDecoderReleaseFailure(codecName: String, format: MediaFormat, exception: Exception) + +} \ No newline at end of file diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/data/Bridge.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/data/Bridge.kt index 5a75f657..f5d23023 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/data/Bridge.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/data/Bridge.kt @@ -8,8 +8,8 @@ import com.otaliastudios.transcoder.internal.utils.Logger import java.nio.ByteBuffer import java.nio.ByteOrder -internal class Bridge(private val format: MediaFormat) - : Step, ReaderChannel { +class Bridge(private val format: MediaFormat) : + Step, ReaderChannel { private val log = Logger("Bridge") private val bufferSize = format.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE) @@ -33,4 +33,4 @@ internal class Bridge(private val format: MediaFormat) val result = WriterData(chunk.buffer, chunk.timeUs, flags) {} return if (state is State.Eos) State.Eos(result) else State.Ok(result) } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/data/Reader.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/data/Reader.kt index 020149a5..51ddf169 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/data/Reader.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/data/Reader.kt @@ -8,16 +8,15 @@ import com.otaliastudios.transcoder.internal.utils.Logger import com.otaliastudios.transcoder.source.DataSource import java.nio.ByteBuffer +data class ReaderData(val chunk: DataSource.Chunk, val id: Int) -internal data class ReaderData(val chunk: DataSource.Chunk, val id: Int) - -internal interface ReaderChannel : Channel { +interface ReaderChannel : Channel { fun buffer(): Pair? } -internal class Reader( - private val source: DataSource, - private val track: TrackType +class Reader( + private val source: DataSource, + private val track: TrackType ) : BaseStep() { private val log = Logger("Reader") @@ -35,7 +34,7 @@ internal class Reader( } override fun step(state: State.Ok, fresh: Boolean): State { - return if (source.isDrained) { + return if (source.isDrained || state is State.Eos) { log.i("Source is drained! Returning Eos as soon as possible.") nextBufferOrWait { byteBuffer, id -> byteBuffer.limit(0) @@ -55,4 +54,4 @@ internal class Reader( } } } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/data/ReaderTimer.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/data/ReaderTimer.kt index 57a48562..17ed6831 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/data/ReaderTimer.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/data/ReaderTimer.kt @@ -5,13 +5,13 @@ import com.otaliastudios.transcoder.internal.pipeline.DataStep import com.otaliastudios.transcoder.internal.pipeline.State import com.otaliastudios.transcoder.time.TimeInterpolator -internal class ReaderTimer( - private val track: TrackType, - private val interpolator: TimeInterpolator +class ReaderTimer( + private val track: TrackType, + private val interpolator: TimeInterpolator ) : DataStep() { override fun step(state: State.Ok, fresh: Boolean): State { if (state is State.Eos) return state state.value.chunk.timeUs = interpolator.interpolate(track, state.value.chunk.timeUs) return state } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/data/Seeker.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/data/Seeker.kt index 610f5647..fa86baf3 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/data/Seeker.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/data/Seeker.kt @@ -1,33 +1,28 @@ package com.otaliastudios.transcoder.internal.data -import com.otaliastudios.transcoder.common.TrackType import com.otaliastudios.transcoder.internal.pipeline.BaseStep import com.otaliastudios.transcoder.internal.pipeline.Channel import com.otaliastudios.transcoder.internal.pipeline.State import com.otaliastudios.transcoder.internal.utils.Logger import com.otaliastudios.transcoder.source.DataSource -import java.nio.ByteBuffer -internal class Seeker( - private val source: DataSource, - positions: List, - private val seek: (Long) -> Boolean +class Seeker( + private val source: DataSource, + private val shouldSeek: () -> Pair ) : BaseStep() { private val log = Logger("Seeker") override val channel = Channel - private val positions = positions.toMutableList() override fun step(state: State.Ok, fresh: Boolean): State { - if (positions.isNotEmpty()) { - if (seek(positions.first())) { - log.i("Seeking to next position ${positions.first()}") - val next = positions.removeFirst() - source.seekTo(next) - } else { - // log.v("Not seeking to next Request. head=${positions.first()}") - } + val shouldSeek = shouldSeek() + val position = shouldSeek.first + if (shouldSeek.second) { + log.i("Seeking to next position $position where currentReaderTime=${source.getPositionUs()}") + source.seekTo(position) + } else { + log.i("Skipping seek for $position where currentReaderTime=${source.getPositionUs()}") } return state } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/data/Writer.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/data/Writer.kt index 1d8fc899..40448bab 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/data/Writer.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/data/Writer.kt @@ -10,20 +10,21 @@ import com.otaliastudios.transcoder.internal.utils.Logger import com.otaliastudios.transcoder.sink.DataSink import java.nio.ByteBuffer -internal data class WriterData( - val buffer: ByteBuffer, - val timeUs: Long, - val flags: Int, - val release: () -> Unit +data class WriterData( + val buffer: ByteBuffer, + val timeUs: Long, + val flags: Int, + val release: () -> Unit ) -internal interface WriterChannel : Channel { +interface WriterChannel : Channel { fun handleFormat(format: MediaFormat) } -internal class Writer( - private val sink: DataSink, - private val track: TrackType +class Writer( + private val sink: DataSink, + private val track: TrackType, + private val processedTimeStamp: ((Long) -> Unit)? = null ) : Step, WriterChannel { override val channel = this @@ -40,15 +41,16 @@ internal class Writer( val (buffer, timestamp, flags) = state.value val eos = state is State.Eos info.set( - buffer.position(), - buffer.remaining(), - timestamp, - if (eos) { - flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM - } else flags + buffer.position(), + buffer.remaining(), + timestamp, + if (eos) { + flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM + } else flags ) sink.writeTrack(track, buffer, info) + processedTimeStamp?.invoke(if (!eos) timestamp else -1) state.value.release() return if (eos) State.Eos(Unit) else State.Ok(Unit) } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/CustomPipeline.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/CustomPipeline.kt new file mode 100644 index 00000000..0fa4923d --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/CustomPipeline.kt @@ -0,0 +1,70 @@ +package com.otaliastudios.transcoder.internal.pipeline + +import com.otaliastudios.transcoder.internal.codec.EncoderChannel +import com.otaliastudios.transcoder.internal.codec.EncoderData +import com.otaliastudios.transcoder.internal.utils.Logger + +class CustomPipeline private constructor(name: String, private val chain: List) { + + private val log = Logger("Pipeline($name)") + private var headState: State.Ok = State.Ok(Unit) + private var headIndex = 0 + + init { + chain.zipWithNext().reversed().forEach { (first, next) -> + first.initialize(next = next.channel) + } + } + + // Returns Eos, Ok or Wait + fun execute(): State { + log.v("execute(): starting. head=$headIndex steps=${chain.size} remaining=${chain.size - headIndex}") + val head = headIndex + var state = headState + chain.forEachIndexed { index, step -> + if (index < head) return@forEachIndexed + val fresh = head == 0 || index != head + state = executeStep(state, step, fresh) ?: run { + log.v( + "execute(): step ${step.name} (#$index/${chain.size}) is waiting." + + " headState=$headState headIndex=$headIndex" + ) + return State.Wait + } + // log.v("execute(): executed ${step.name} (#$index/${chain.size}). result=$state") + if (state is State.Eos) { + log.i("execute(): EOS from ${step.name} (#$index/${chain.size}).") + headState = state + headIndex = index + 1 + } + } + return when { + chain.isEmpty() -> State.Eos(EncoderData.Empty) + state is State.Eos -> State.Eos(EncoderData.Empty) + else -> State.Ok(state.value as EncoderData) + } + } + + fun release() { + chain.forEach { it.release() } + } + + private fun executeStep(previous: State.Ok, step: AnyStep, fresh: Boolean): State.Ok? { + return when (val state = step.step(previous, fresh)) { + is State.Ok -> state + is State.Retry -> executeStep(previous, step, fresh = false) + is State.Wait -> null + } + } + + companion object { + @Suppress("UNCHECKED_CAST") + fun build( + name: String, + builder: () -> Pipeline.Builder<*, EncoderChannel> = + { Pipeline.Builder() } + ): CustomPipeline { + return CustomPipeline(name, builder().steps as List) + } + } +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/Pipeline.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/Pipeline.kt index d64a1016..f57bfc83 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/Pipeline.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/Pipeline.kt @@ -2,10 +2,9 @@ package com.otaliastudios.transcoder.internal.pipeline import com.otaliastudios.transcoder.internal.utils.Logger +internal typealias AnyStep = Step -private typealias AnyStep = Step - -internal class Pipeline private constructor(name: String, private val chain: List) { +class Pipeline private constructor(name: String, private val chain: List) { private val log = Logger("Pipeline($name)") private var headState: State.Ok = State.Ok(Unit) @@ -26,7 +25,10 @@ internal class Pipeline private constructor(name: String, private val chain: Lis if (index < head) return@forEachIndexed val fresh = head == 0 || index != head state = executeStep(state, step, fresh) ?: run { - log.v("execute(): step ${step.name} (#$index/${chain.size}) is waiting. headState=$headState headIndex=$headIndex") + log.v( + "execute(): step ${step.name} (#$index/${chain.size}) is waiting. " + + "headState=$headState headIndex=$headIndex" + ) return State.Wait } // log.v("execute(): executed ${step.name} (#$index/${chain.size}). result=$state") @@ -47,9 +49,12 @@ internal class Pipeline private constructor(name: String, private val chain: Lis chain.forEach { it.release() } } - private fun executeStep(previous: State.Ok, step: AnyStep, fresh: Boolean): State.Ok? { - val state = step.step(previous, fresh) - return when (state) { + private fun executeStep( + previous: State.Ok, + step: AnyStep, + fresh: Boolean + ): State.Ok? { + return when (val state = step.step(previous, fresh)) { is State.Ok -> state is State.Retry -> executeStep(previous, step, fresh = false) is State.Wait -> null @@ -58,25 +63,25 @@ internal class Pipeline private constructor(name: String, private val chain: Lis companion object { @Suppress("UNCHECKED_CAST") - internal fun build(name: String, builder: () -> Builder<*, Channel> = { Builder() }): Pipeline { + fun build( + name: String, + builder: () -> Builder<*, Channel> = { Builder() } + ): Pipeline { return Pipeline(name, builder().steps as List) } } - class Builder internal constructor( - internal val steps: List> = listOf() + class Builder internal constructor( + internal val steps: List> = listOf() ) { - operator fun plus( - step: Step - ): Builder = Builder(steps + step) + operator fun plus( + step: Step + ): Builder = Builder(steps + step) } } -internal operator fun < - CurrData: Any, CurrChannel: Channel, - NewData: Any, NewChannel: Channel -> Step.plus( - other: Step -): Pipeline.Builder { +operator fun +Step.plus(other: Step): + Pipeline.Builder { return Pipeline.Builder(listOf(this)) + other -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/State.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/State.kt index 4fb77c79..f33a99da 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/State.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/State.kt @@ -1,6 +1,6 @@ package com.otaliastudios.transcoder.internal.pipeline -internal sealed class State { +sealed class State { // Running open class Ok(val value: T) : State() { @@ -21,4 +21,4 @@ internal sealed class State { object Retry : State() { override fun toString() = "State.Retry" } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/Step.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/Step.kt index 7159182b..4e61c993 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/Step.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/Step.kt @@ -1,16 +1,16 @@ package com.otaliastudios.transcoder.internal.pipeline // TODO this could be Any -internal interface Channel { +interface Channel { companion object : Channel } -internal interface Step< - Input: Any, - InputChannel: Channel, - Output: Any, - OutputChannel: Channel -> { +interface Step< + Input : Any, + InputChannel : Channel, + Output : Any, + OutputChannel : Channel + > { val channel: InputChannel fun initialize(next: OutputChannel) = Unit @@ -20,4 +20,4 @@ internal interface Step< fun release() = Unit } -internal val Step<*, *, *, *>.name get() = this::class.simpleName \ No newline at end of file +val Step<*, *, *, *>.name get() = this::class.simpleName diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/pipelines.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/pipelines.kt index e0a4b33e..c4845aed 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/pipelines.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/pipelines.kt @@ -1,16 +1,18 @@ +@file:Suppress("MagicNumber", "LongParameterList") + package com.otaliastudios.transcoder.internal.pipeline import android.media.MediaFormat import com.otaliastudios.transcoder.common.TrackType import com.otaliastudios.transcoder.internal.Codecs import com.otaliastudios.transcoder.internal.audio.AudioEngine -import com.otaliastudios.transcoder.internal.data.* -import com.otaliastudios.transcoder.internal.data.Reader -import com.otaliastudios.transcoder.internal.data.ReaderTimer -import com.otaliastudios.transcoder.internal.data.Writer import com.otaliastudios.transcoder.internal.codec.Decoder import com.otaliastudios.transcoder.internal.codec.DecoderTimer import com.otaliastudios.transcoder.internal.codec.Encoder +import com.otaliastudios.transcoder.internal.data.Bridge +import com.otaliastudios.transcoder.internal.data.Reader +import com.otaliastudios.transcoder.internal.data.ReaderTimer +import com.otaliastudios.transcoder.internal.data.Writer import com.otaliastudios.transcoder.internal.video.VideoPublisher import com.otaliastudios.transcoder.internal.video.VideoRenderer import com.otaliastudios.transcoder.resample.AudioResampler @@ -19,65 +21,71 @@ import com.otaliastudios.transcoder.source.DataSource import com.otaliastudios.transcoder.stretch.AudioStretcher import com.otaliastudios.transcoder.time.TimeInterpolator -internal fun EmptyPipeline() = Pipeline.build("Empty") +fun emptyPipeline() = Pipeline.build("Empty") -internal fun PassThroughPipeline( - track: TrackType, - source: DataSource, - sink: DataSink, - interpolator: TimeInterpolator +internal fun passThroughPipeline( + track: TrackType, + source: DataSource, + sink: DataSink, + interpolator: TimeInterpolator ) = Pipeline.build("PassThrough($track)") { Reader(source, track) + - ReaderTimer(track, interpolator) + - Bridge(source.getTrackFormat(track)!!) + - Writer(sink, track) + ReaderTimer(track, interpolator) + + Bridge(source.getTrackFormat(track)!!) + + Writer(sink, track) } -internal fun RegularPipeline( - track: TrackType, - source: DataSource, - sink: DataSink, - interpolator: TimeInterpolator, - format: MediaFormat, - codecs: Codecs, - videoRotation: Int, - audioStretcher: AudioStretcher, - audioResampler: AudioResampler +internal fun regularPipeline( + track: TrackType, + source: DataSource, + sink: DataSink, + interpolator: TimeInterpolator, + format: MediaFormat, + codecs: Codecs, + videoRotation: Int, + audioStretcher: AudioStretcher, + audioResampler: AudioResampler ) = when (track) { - TrackType.VIDEO -> VideoPipeline(source, sink, interpolator, format, codecs, videoRotation) - TrackType.AUDIO -> AudioPipeline(source, sink, interpolator, format, codecs, audioStretcher, audioResampler) + TrackType.VIDEO -> videoPipeline(source, sink, interpolator, format, codecs, videoRotation) + TrackType.AUDIO -> audioPipeline(source, sink, interpolator, format, codecs, audioStretcher, audioResampler) } -private fun VideoPipeline( - source: DataSource, - sink: DataSink, - interpolator: TimeInterpolator, - format: MediaFormat, - codecs: Codecs, - videoRotation: Int +data class RenderingData( + val timeMsCurrent: Long, + val timeMsPrevious: Long, + val updateTime: () -> Unit +) + +private fun videoPipeline( + source: DataSource, + sink: DataSink, + interpolator: TimeInterpolator, + format: MediaFormat, + codecs: Codecs, + videoRotation: Int ) = Pipeline.build("Video") { Reader(source, TrackType.VIDEO) + - Decoder(source.getTrackFormat(TrackType.VIDEO)!!, true) + - DecoderTimer(TrackType.VIDEO, interpolator) + - VideoRenderer(source.orientation, videoRotation, format) + - VideoPublisher() + - Encoder(codecs, TrackType.VIDEO) + - Writer(sink, TrackType.VIDEO) + Decoder(source.getTrackFormat(TrackType.VIDEO)!!, true) + + DecoderTimer(TrackType.VIDEO, interpolator) + + VideoRenderer(source.orientation, videoRotation, format) + + VideoPublisher() + + Encoder(codecs, TrackType.VIDEO) + + Writer(sink, TrackType.VIDEO) } -private fun AudioPipeline( - source: DataSource, - sink: DataSink, - interpolator: TimeInterpolator, - format: MediaFormat, - codecs: Codecs, - audioStretcher: AudioStretcher, - audioResampler: AudioResampler +private fun audioPipeline( + source: DataSource, + sink: DataSink, + interpolator: TimeInterpolator, + format: MediaFormat, + codecs: Codecs, + audioStretcher: AudioStretcher, + audioResampler: AudioResampler ) = Pipeline.build("Audio") { Reader(source, TrackType.AUDIO) + - Decoder(source.getTrackFormat(TrackType.AUDIO)!!, true) + - DecoderTimer(TrackType.AUDIO, interpolator) + - AudioEngine(audioStretcher, audioResampler, format) + - Encoder(codecs, TrackType.AUDIO) + - Writer(sink, TrackType.AUDIO) -} \ No newline at end of file + Decoder(source.getTrackFormat(TrackType.AUDIO)!!, true) + + DecoderTimer(TrackType.AUDIO, interpolator) + + AudioEngine(audioStretcher, audioResampler, format) + + Encoder(codecs, TrackType.AUDIO) + + Writer(sink, TrackType.AUDIO) +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/steps.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/steps.kt index 9afe4ef7..37484d2f 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/steps.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/steps.kt @@ -1,32 +1,32 @@ package com.otaliastudios.transcoder.internal.pipeline -internal abstract class BaseStep< - Input: Any, - InputChannel: Channel, - Output: Any, - OutputChannel: Channel -> : Step { +abstract class BaseStep< + Input : Any, + InputChannel : Channel, + Output : Any, + OutputChannel : Channel + > : Step { protected lateinit var next: OutputChannel - private set + private set override fun initialize(next: OutputChannel) { this.next = next } } -internal abstract class DataStep : Step { +abstract class DataStep : Step { override lateinit var channel: C override fun initialize(next: C) { channel = next } } -internal abstract class QueuedStep< - Input: Any, - InputChannel: Channel, - Output: Any, - OutputChannel: Channel -> : BaseStep() { +abstract class QueuedStep< + Input : Any, + InputChannel : Channel, + Output : Any, + OutputChannel : Channel + > : BaseStep() { protected abstract fun enqueue(data: Input) @@ -41,4 +41,4 @@ internal abstract class QueuedStep< } return drain() } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/thumbnails/DefaultThumbnailsEngine.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/thumbnails/DefaultThumbnailsEngine.kt index 7560fa37..2b83dc43 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/thumbnails/DefaultThumbnailsEngine.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/thumbnails/DefaultThumbnailsEngine.kt @@ -1,140 +1,403 @@ +@file:Suppress("MagicNumber", "UnusedPrivateMember") + package com.otaliastudios.transcoder.internal.thumbnails -import android.graphics.Bitmap import android.media.MediaFormat -import android.media.MediaFormat.KEY_HEIGHT -import android.media.MediaFormat.KEY_WIDTH -import android.opengl.GLES20 -import com.otaliastudios.opengl.core.Egloo -import com.otaliastudios.transcoder.common.TrackStatus import com.otaliastudios.transcoder.common.TrackType +import com.otaliastudios.transcoder.internal.CustomSegments import com.otaliastudios.transcoder.internal.DataSources -import com.otaliastudios.transcoder.internal.Segments -import com.otaliastudios.transcoder.internal.Timer import com.otaliastudios.transcoder.internal.Tracks import com.otaliastudios.transcoder.internal.codec.Decoder -import com.otaliastudios.transcoder.internal.codec.EncoderChannel -import com.otaliastudios.transcoder.internal.codec.EncoderData +import com.otaliastudios.transcoder.internal.codec.TranscoderEventsListener import com.otaliastudios.transcoder.internal.data.Reader import com.otaliastudios.transcoder.internal.data.Seeker -import com.otaliastudios.transcoder.internal.pipeline.* -import com.otaliastudios.transcoder.internal.pipeline.Channel -import com.otaliastudios.transcoder.internal.pipeline.EmptyPipeline import com.otaliastudios.transcoder.internal.pipeline.Pipeline -import com.otaliastudios.transcoder.internal.utils.* -import com.otaliastudios.transcoder.internal.utils.forcingEos -import com.otaliastudios.transcoder.internal.video.VideoPublisher +import com.otaliastudios.transcoder.internal.pipeline.plus +import com.otaliastudios.transcoder.internal.utils.Logger +import com.otaliastudios.transcoder.internal.utils.trackMapOf import com.otaliastudios.transcoder.internal.video.VideoRenderer import com.otaliastudios.transcoder.internal.video.VideoSnapshots import com.otaliastudios.transcoder.resize.Resizer +import com.otaliastudios.transcoder.source.DataSource import com.otaliastudios.transcoder.strategy.DefaultVideoStrategy import com.otaliastudios.transcoder.strategy.RemoveTrackStrategy +import com.otaliastudios.transcoder.thumbnail.SingleThumbnailRequest import com.otaliastudios.transcoder.thumbnail.Thumbnail import com.otaliastudios.transcoder.thumbnail.ThumbnailRequest -import com.otaliastudios.transcoder.time.DefaultTimeInterpolator -import java.nio.ByteBuffer -import java.nio.ByteOrder -import kotlin.math.abs - -internal class DefaultThumbnailsEngine( - private val dataSources: DataSources, - private val rotation: Int, - resizer: Resizer, - requests: List +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.delay +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.receiveAsFlow +import kotlinx.coroutines.isActive +import java.util.ArrayList + +class DefaultThumbnailsEngine( + private val dataSources: DataSources, + private val rotation: Int, + resizer: Resizer, + private val eventListener: TranscoderEventsListener? ) : ThumbnailsEngine() { + private var shouldSeek = true + private var shouldFlush = false + private var finish = false private val log = Logger("ThumbnailsEngine") - + private var previousSnapshotUs = 0L // Huge framerate triks the VideoRenderer into not dropping frames, which is important // for thumbnail requests that want to catch the very last frame. - private val tracks = Tracks(trackMapOf( + private val tracks = Tracks( + trackMapOf( video = DefaultVideoStrategy.Builder() - .frameRate(120) - .addResizer(resizer) - .build(), + .frameRate(120) + .addResizer(resizer) + .build(), audio = RemoveTrackStrategy() - ), dataSources, rotation, true) + ), + dataSources, + rotation, + true + ) - private val segments = Segments(dataSources, tracks, ::createPipeline) - - private val timer = Timer(DefaultTimeInterpolator(), dataSources, tracks, segments.currentIndex) + private val segments = CustomSegments(dataSources, tracks, ::createPipeline) init { log.i("Created Tracks, Segments, Timer...") } - private val positions = requests.flatMap { request -> - val duration = timer.totalDurationUs - request.locate(duration).map { it to request } - }.sortedBy { it.first } - private class Stub( - val request: ThumbnailRequest, - val positionUs: Long, - val localizedUs: Long) { + val request: ThumbnailRequest, + val positionUs: Long, + val localizedUs: Long + ) { var actualLocalizedUs: Long = localizedUs + override fun toString(): String { + return request.sourcePath() + ":" + positionUs.toString() + } + } + + private val stubs = ArrayDeque() + + private inner class IgnoringEosDataSource( + private val source: DataSource, + ) : DataSource by source { + + override fun requestKeyFrameTimestamps() = source.requestKeyFrameTimestamps() + + override fun getKeyFrameTimestamps() = source.keyFrameTimestamps + + override fun getSeekThreshold() = source.seekThreshold + + override fun mediaId() = source.mediaId() + + override fun isDrained(): Boolean { + if (source.isDrained) { + source.seekTo(stubs.firstOrNull()?.positionUs ?: -1) + } + return source.isDrained + } } + private fun DataSource.ignoringEOS(): DataSource = IgnoringEosDataSource(this) private fun createPipeline( - type: TrackType, - index: Int, - status: TrackStatus, - outputFormat: MediaFormat + type: TrackType, + source: DataSource, + outputFormat: MediaFormat ): Pipeline { - log.i("Creating pipeline #$index. absoluteUs=${positions.joinToString { it.first.toString() }}") - val stubs = positions.mapNotNull { (positionUs, request) -> - val localizedUs = timer.localize(type, index, positionUs) - localizedUs?.let { Stub(request, positionUs, localizedUs) } - }.toMutableList() - - if (stubs.isEmpty()) return EmptyPipeline() - val source = dataSources[type][index].forcingEos { - stubs.isEmpty() - } - val positions = stubs.map { it.localizedUs } - log.i("Requests for step #$index: ${positions.joinToString()} [duration=${source.durationUs}]") + val source = source.ignoringEOS() + if(VERBOSE) { + log.i("Creating pipeline for $source. absoluteUs=${stubs.joinToString { it.toString() }}") + } + shouldSeek = true + shouldFlush = false return Pipeline.build("Thumbnails") { - Seeker(source, positions) { it == stubs.firstOrNull()?.localizedUs } + - Reader(source, type) + - Decoder(source.getTrackFormat(type)!!, continuous = false) + - VideoRenderer(source.orientation, rotation, outputFormat, flipY = true) + - VideoSnapshots(outputFormat, positions, 50 * 1000) { pos, bitmap -> - val stub = stubs.removeFirst() + Seeker(source) { + var seek = false + val requested = stubs.firstOrNull()?.positionUs ?: -1 + + if (!shouldSeek || requested == -1L) + return@Seeker Pair(requested, seek) + + val seekUs: Long + val current = source.positionUs + val threshold = stubs.firstOrNull()?.request?.threshold() ?: 0L + val nextKeyFrameIndex = source.search(requested) + + val nextKeyFrameUs = source.keyFrameAt(nextKeyFrameIndex) { Long.MAX_VALUE } + val previousKeyFrameUs = source.keyFrameAt(nextKeyFrameIndex - 1) { source.lastKeyFrame() } + + + val rightGap = nextKeyFrameUs - requested + val nextKeyFrameInThreshold = rightGap <= threshold + seek = nextKeyFrameInThreshold || previousKeyFrameUs > current || (current - requested > threshold) + seekUs = + (if (nextKeyFrameInThreshold) nextKeyFrameUs else previousKeyFrameUs) + source.seekThreshold + + if (VERBOSE) { + log.i( + "seek: current ${source.positionUs}," + + " requested $requested, threshold $threshold, nextKeyFrameUs $nextKeyFrameUs," + + " nextKeyFrameInThreshold:$nextKeyFrameInThreshold, seekUs: $seekUs, flushing : $seek" + ) + } + + shouldFlush = seek + shouldSeek = false + Pair(seekUs, seek) + } + + Reader(source, type) + + Decoder(source.getTrackFormat(type)!!, continuous = false, useSwFor4K = true, eventListener) { + shouldFlush.also { + shouldFlush = false + } + } + + VideoRenderer(source.orientation, rotation, outputFormat, flipY = true, true) { + stubs.firstOrNull()?.positionUs ?: -1 + } + + VideoSnapshots(outputFormat, fetchPosition) { pos, bitmap -> + val stub = stubs.removeFirstOrNull() + if (stub != null) { + shouldSeek = true stub.actualLocalizedUs = pos - log.i("Got snapshot. positionUs=${stub.positionUs} " + + previousSnapshotUs = pos + log.i( + "Got snapshot. positionUs=${stub.positionUs} " + "localizedUs=${stub.localizedUs} " + "actualLocalizedUs=${stub.actualLocalizedUs} " + - "deltaUs=${stub.localizedUs - stub.actualLocalizedUs}") + "deltaUs=${stub.localizedUs - stub.actualLocalizedUs}" + ) val thumbnail = Thumbnail(stub.request, stub.positionUs, bitmap) - progress(thumbnail) + val callbackStatus = progress.trySend(thumbnail) + if (VERBOSE) { + log.i("Callback Send Status ${callbackStatus.isSuccess}") + } } + } } } - private lateinit var progress: (Thumbnail) -> Unit + private val progress = Channel(Channel.BUFFERED) + + + private fun DataSource.lastKeyFrame() = keyFrameAt(keyFrameTimestamps.size - 1) + + override val progressFlow: Flow = progress.receiveAsFlow() - override fun thumbnails(progress: (Thumbnail) -> Unit) { - this.progress = progress - while (true) { - val advanced = segments.next(TrackType.VIDEO)?.advance() ?: false - val completed = !advanced && !segments.hasNext() // avoid calling hasNext if we advanced. - if (Thread.interrupted()) { - throw InterruptedException() - } else if (completed) { + private inline fun DataSource.keyFrameAt(index: Int, defaultValue: ((Int)-> Long) = {_ -> -1}) = + keyFrameTimestamps.getOrElse(index, defaultValue) + + private fun DataSource.search(timestampUs: Long): Int { + if (keyFrameTimestamps.isEmpty()) + requestKeyFrameTimestamps() + + val searchIndex = keyFrameTimestamps.binarySearch(timestampUs) + + val nextKeyFrameIndex = when { + searchIndex >= 0 -> searchIndex + else -> { + val index = -searchIndex - 1 + when { + index >= keyFrameTimestamps.size -> { + val ts = requestKeyFrameTimestamps() + if (ts == -1L) { + -1 + } else { + search(timestampUs) + } + } + index < keyFrameTimestamps.size -> index + else -> { + -1 // will never reach here. kotlin is stupid + } + } + } + } + + return nextKeyFrameIndex + } + + override fun addDataSource(dataSource: DataSource) { + if (dataSources.getVideoSources().find { it.mediaId() == dataSource.mediaId() } != null) { + return // dataSource already exists + } + dataSources.addVideoDataSource(dataSource) + tracks.updateTracksInfo() + if (tracks.active.has(TrackType.VIDEO) && dataSource.getTrackFormat(TrackType.VIDEO) != null) { + dataSource.selectTrack(TrackType.VIDEO) + } + } + + override fun removeDataSource(dataSourceId: String) { + segments.releaseSegment(dataSourceId) + dataSources.removeVideoDataSource(dataSourceId) + tracks.updateTracksInfo() + } + + override fun updateDataSources(dataSourcesNew: List) { + val currentVideoIds = dataSources.videoOrNull()?.map { it.mediaId() } + val newSourceIds = dataSourcesNew.map { it.mediaId() }.distinct() + val toAdd = newSourceIds - currentVideoIds + val toRemove = currentVideoIds?.minus(newSourceIds) + toAdd.forEach { id -> + val source = dataSourcesNew.first { it.mediaId() == id } + addDataSource(source) + } + toRemove?.forEach { id -> + removeDataSource(id) + } + } + + override suspend fun queueThumbnails(list: List) { + + val map = list.groupBy { it.sourcePath() } + + map.forEach { entry -> + val dataSource = getDataSourceByPath(entry.key) + if (dataSource != null) { + val duration = dataSource.getVideoTrackDuration() + val positions = entry.value.flatMap { request -> + request.locate(duration).map { it to request } + }.sortedBy { it.first } + stubs.addAll( + positions.map { (positionUs, request) -> + Stub(request, positionUs, positionUs) + }.toMutableList().reorder(dataSource) + ) + if (VERBOSE) { + log.i("Updating pipeline positions for segment source#$dataSource absoluteUs=${positions.joinToString { it.first.toString() }}, and stubs $stubs") + } + } + + } + + while (currentCoroutineContext().isActive && stubs.isNotEmpty()) { + var advanced = false + val stub = stubs.firstOrNull() + try { + val segment = stub?.request?.sourcePath()?.let { segments.getSegment(it) } + + if (VERBOSE) { + log.i("loop advancing for $segment") + } + advanced = try { + segment?.advance() ?: false + } catch (e: Exception) { + if (e !is CancellationException) { + currentCoroutineContext().ensureActive() + } + throw e + } + } catch (e: Exception) { + val path = stub?.request?.sourcePath() + if (path != null) { + val dataSource = getDataSourceByPath(path) + val bitmap = dataSource?.getFrameAtPosition(stub.positionUs, 150, 150) + if (bitmap != null) { + val thumbnail = Thumbnail(stub.request, stub.positionUs, bitmap) + progress.trySend(thumbnail) + stubs.removeFirst() + advanced = true + } + } + } + + // avoid calling hasNext if we advanced. + val completed = !advanced && !segments.hasNext() + if (completed || stubs.isEmpty()) { + log.i("loop broken $stubs $hasMoreRequestsIncoming") + if (!hasMoreRequestsIncoming) { + try { + segments.release() + } catch (e: IllegalStateException) { + + } + } break } else if (!advanced) { - Thread.sleep(WAIT_MS) + delay(WAIT_MS) + } + } + + } + + private fun DataSource.getVideoTrackDuration() = + getTrackFormat(TrackType.VIDEO)?.getLong(MediaFormat.KEY_DURATION) + ?: durationUs + + fun finish() { + this.finish = true + segments.release() + } + + override fun removePosition(sourcePath: String, sourceId: String, positionUs: Long) { + if (positionUs < 0) { +// val activeStub = stubs.firstOrNull()?.takeIf { it.request.sourceId() == sourceId } + stubs.removeAll { + it.request.sourceId() == sourceId } +// if (activeStub != null) { +// stubs.addFirst(activeStub) +// } + shouldSeek = true + return } + val isStubActive = + stubs.firstOrNull()?.request?.sourceId() == sourceId && positionUs == stubs.firstOrNull()?.positionUs && positionUs > 0 + if (isStubActive) { + return + } + + val dataSource = getDataSourceByPath(sourcePath) + if (dataSource != null) { + val duration = dataSource.getVideoTrackDuration() + val locatedTimestampUs = SingleThumbnailRequest(positionUs).locate(duration)[0] + val stub = + stubs.find { it.request.sourceId() == sourceId && it.positionUs == locatedTimestampUs } + if (stub != null) { + log.i("removePosition Match: $positionUs :$stubs") + stubs.remove(stub) + shouldSeek = true + } + } + } + + + override fun getDataSourceByPath(source: String): DataSource? { + return dataSources[TrackType.VIDEO].firstOrNull { it.mediaId() == source } + } + + private fun List.reorder(source: DataSource): Collection { + val bucketListMap = LinkedHashMap>() + val finalList = ArrayList() + + forEach { + val nextKeyFrameIndex = source.search(it.positionUs) + val previousKeyFrameUs = source.keyFrameAt(nextKeyFrameIndex - 1) { source.lastKeyFrame() } + + val list = bucketListMap.getOrPut(previousKeyFrameUs) { ArrayList() } + list.add(it) + } + bucketListMap.forEach { + finalList.addAll(it.value.sortedBy { it.positionUs }) + } + return finalList + } + + private val fetchPosition: () -> VideoSnapshots.Request? = { + if (stubs.isEmpty()) null + else VideoSnapshots.Request(stubs.first().localizedUs, stubs.first().request.threshold()) } override fun cleanup() { + runCatching { stubs.clear() } runCatching { segments.release() } runCatching { dataSources.release() } } companion object { - private val WAIT_MS = 10L - private val PROGRESS_LOOPS = 10L + private const val WAIT_MS = 5L + private const val VERBOSE = false } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/thumbnails/ThumbnailsEngine.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/thumbnails/ThumbnailsEngine.kt index 26eb75a7..0fd0c140 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/thumbnails/ThumbnailsEngine.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/thumbnails/ThumbnailsEngine.kt @@ -1,56 +1,62 @@ +@file:Suppress("TooGenericExceptionCaught") + package com.otaliastudios.transcoder.internal.thumbnails import com.otaliastudios.transcoder.ThumbnailerOptions -import com.otaliastudios.transcoder.Transcoder -import com.otaliastudios.transcoder.TranscoderOptions import com.otaliastudios.transcoder.internal.DataSources import com.otaliastudios.transcoder.internal.utils.Logger -import com.otaliastudios.transcoder.internal.utils.trackMapOf +import com.otaliastudios.transcoder.source.DataSource import com.otaliastudios.transcoder.thumbnail.Thumbnail +import com.otaliastudios.transcoder.thumbnail.ThumbnailRequest +import kotlinx.coroutines.flow.Flow + +abstract class ThumbnailsEngine { + + abstract val progressFlow: Flow + + abstract fun addDataSource(dataSource: DataSource) + + abstract fun removeDataSource(dataSourceId: String) + + abstract fun updateDataSources(dataSources: List) + + abstract suspend fun queueThumbnails(list: List) -internal abstract class ThumbnailsEngine { + abstract fun removePosition(sourcePath: String, sourceId: String, positionUs: Long) - abstract fun thumbnails(progress: (Thumbnail) -> Unit) + abstract fun getDataSourceByPath(source: String): DataSource? abstract fun cleanup() + @Volatile + var hasMoreRequestsIncoming = true + companion object { private val log = Logger("ThumbnailsEngine") private fun Throwable.isInterrupted(): Boolean { - if (this is InterruptedException) return true - if (this == this.cause) return false - return this.cause?.isInterrupted() ?: false + return when { + this is InterruptedException -> true + this == this.cause -> false + else -> this.cause?.isInterrupted() ?: false + } } + private lateinit var dispatcher: ThumbnailsDispatcher + @JvmStatic - fun thumbnails(options: ThumbnailerOptions) { + fun thumbnails(options: ThumbnailerOptions): ThumbnailsEngine { log.i("thumbnails(): called...") - var engine: ThumbnailsEngine? = null - val dispatcher = ThumbnailsDispatcher(options) - try { - engine = DefaultThumbnailsEngine( - dataSources = DataSources(options), - rotation = options.rotation, - resizer = options.resizer, - requests = options.thumbnailRequests - ) - engine.thumbnails { - dispatcher.dispatchThumbnail(it) - } - dispatcher.dispatchCompletion() - } catch (e: Exception) { - if (e.isInterrupted()) { - log.i("Transcode canceled.", e) - dispatcher.dispatchCancel() - } else { - log.e("Unexpected error while transcoding.", e) - dispatcher.dispatchFailure(e) - throw e - } - } finally { - engine?.cleanup() - } + dispatcher = ThumbnailsDispatcher(options) + + val engine = DefaultThumbnailsEngine( + dataSources = DataSources(options), + rotation = options.rotation, + resizer = options.resizer, + eventListener = options.eventListener + ) + return engine } } -} \ No newline at end of file + +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/transcode/DefaultTranscodeEngine.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/transcode/DefaultTranscodeEngine.kt index 985d7ff4..571965d1 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/transcode/DefaultTranscodeEngine.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/transcode/DefaultTranscodeEngine.kt @@ -1,18 +1,20 @@ +@file:Suppress("MagicNumber", "LongParameterList") + package com.otaliastudios.transcoder.internal.transcode import android.media.MediaFormat import com.otaliastudios.transcoder.common.TrackStatus import com.otaliastudios.transcoder.common.TrackType -import com.otaliastudios.transcoder.internal.* import com.otaliastudios.transcoder.internal.Codecs import com.otaliastudios.transcoder.internal.DataSources +import com.otaliastudios.transcoder.internal.Segment import com.otaliastudios.transcoder.internal.Segments import com.otaliastudios.transcoder.internal.Timer import com.otaliastudios.transcoder.internal.Tracks -import com.otaliastudios.transcoder.internal.pipeline.EmptyPipeline -import com.otaliastudios.transcoder.internal.pipeline.PassThroughPipeline import com.otaliastudios.transcoder.internal.pipeline.Pipeline -import com.otaliastudios.transcoder.internal.pipeline.RegularPipeline +import com.otaliastudios.transcoder.internal.pipeline.emptyPipeline +import com.otaliastudios.transcoder.internal.pipeline.passThroughPipeline +import com.otaliastudios.transcoder.internal.pipeline.regularPipeline import com.otaliastudios.transcoder.internal.utils.Logger import com.otaliastudios.transcoder.internal.utils.TrackMap import com.otaliastudios.transcoder.internal.utils.forcingEos @@ -24,15 +26,16 @@ import com.otaliastudios.transcoder.stretch.AudioStretcher import com.otaliastudios.transcoder.time.TimeInterpolator import com.otaliastudios.transcoder.validator.Validator -internal class DefaultTranscodeEngine( - private val dataSources: DataSources, - private val dataSink: DataSink, - strategies: TrackMap, - private val validator: Validator, - private val videoRotation: Int, - private val audioStretcher: AudioStretcher, - private val audioResampler: AudioResampler, - interpolator: TimeInterpolator +class DefaultTranscodeEngine( + private val dataSources: DataSources, + private val dataSink: DataSink, + strategies: TrackMap, + private val validator: Validator, + private val videoRotation: Int, + private val audioStretcher: AudioStretcher, + private val audioResampler: AudioResampler, + interpolator: TimeInterpolator, + private val pipelineFactory: ((TrackType, DataSink, Codecs, MediaFormat) -> Pipeline)?, ) : TranscodeEngine() { private val log = Logger("TranscodeEngine") @@ -61,10 +64,10 @@ internal class DefaultTranscodeEngine( } private fun createPipeline( - type: TrackType, - index: Int, - status: TrackStatus, - outputFormat: MediaFormat + type: TrackType, + index: Int, + status: TrackStatus, + outputFormat: MediaFormat ): Pipeline { log.w("createPipeline($type, $index, $status), format=$outputFormat") val interpolator = timer.interpolator(type, index) @@ -75,13 +78,19 @@ internal class DefaultTranscodeEngine( timer.positionUs[type] > timer.totalDurationUs + 100L } val sink = dataSink.ignoringEos { index < sources.lastIndex } + if (pipelineFactory != null) { + return pipelineFactory.invoke(type, sink, codecs, outputFormat) + } + return when (status) { - TrackStatus.ABSENT -> EmptyPipeline() - TrackStatus.REMOVING -> EmptyPipeline() - TrackStatus.PASS_THROUGH -> PassThroughPipeline(type, source, sink, interpolator) - TrackStatus.COMPRESSING -> RegularPipeline(type, - source, sink, interpolator, outputFormat, codecs, - videoRotation, audioStretcher, audioResampler) + TrackStatus.ABSENT -> emptyPipeline() + TrackStatus.REMOVING -> emptyPipeline() + TrackStatus.PASS_THROUGH -> passThroughPipeline(type, source, sink, interpolator) + TrackStatus.COMPRESSING -> regularPipeline( + type, + source, sink, interpolator, outputFormat, codecs, + videoRotation, audioStretcher, audioResampler + ) } } @@ -100,7 +109,8 @@ internal class DefaultTranscodeEngine( */ override fun transcode(progress: (Double) -> Unit) { var loop = 0L - log.i("transcode(): about to start, " + + log.i( + "transcode(): about to start, " + "durationUs=${timer.totalDurationUs}, " + "audioUs=${timer.durationUs.audioOrNull()}, " + "videoUs=${timer.durationUs.videoOrNull()}" @@ -139,9 +149,8 @@ internal class DefaultTranscodeEngine( runCatching { codecs.release() } } - companion object { - private val WAIT_MS = 10L - private val PROGRESS_LOOPS = 10L + private const val WAIT_MS = 10L + private const val PROGRESS_LOOPS = 10L } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/transcode/TranscodeDispatcher.java b/lib/src/main/java/com/otaliastudios/transcoder/internal/transcode/TranscodeDispatcher.java deleted file mode 100644 index 2db927ed..00000000 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/transcode/TranscodeDispatcher.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.otaliastudios.transcoder.internal.transcode; - -import android.os.Handler; - -import androidx.annotation.NonNull; - -import com.otaliastudios.transcoder.TranscoderListener; -import com.otaliastudios.transcoder.TranscoderOptions; - -/** - * Wraps a TranscoderListener and posts events on the given handler. - */ -class TranscodeDispatcher { - - private final Handler mHandler; - private final TranscoderListener mListener; - - TranscodeDispatcher(@NonNull TranscoderOptions options) { - mHandler = options.getListenerHandler(); - mListener = options.getListener(); - } - - void dispatchCancel() { - mHandler.post(new Runnable() { - @Override - public void run() { - mListener.onTranscodeCanceled(); - } - }); - } - - void dispatchSuccess(final int successCode) { - mHandler.post(new Runnable() { - @Override - public void run() { - mListener.onTranscodeCompleted(successCode); - } - }); - } - - void dispatchFailure(@NonNull final Throwable exception) { - mHandler.post(new Runnable() { - @Override - public void run() { - mListener.onTranscodeFailed(exception); - } - }); - } - - void dispatchProgress(final double progress) { - mHandler.post(new Runnable() { - @Override - public void run() { - mListener.onTranscodeProgress(progress); - } - }); - } -} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/transcode/TranscodeDispatcher.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/transcode/TranscodeDispatcher.kt new file mode 100644 index 00000000..37f6e5c0 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/transcode/TranscodeDispatcher.kt @@ -0,0 +1,28 @@ +package com.otaliastudios.transcoder.internal.transcode + +import com.otaliastudios.transcoder.TranscoderOptions +import com.otaliastudios.transcoder.TranscoderListener + +/** + * Wraps a TranscoderListener and posts events on the given handler. + */ +internal class TranscodeDispatcher(options: TranscoderOptions) { + private val mHandler = options.listenerHandler + private val mListener = options.listener + fun dispatchCancel() { + mHandler.post { mListener?.onTranscodeCanceled() } + } + + fun dispatchSuccess(successCode: Int) { + mHandler.post { mListener?.onTranscodeCompleted(successCode) } + } + + fun dispatchFailure(exception: Throwable) { + mHandler.post { mListener?.onTranscodeFailed(exception) } + } + + fun dispatchProgress(progress: Double) { + mHandler.post { mListener?.onTranscodeProgress(progress) } + } + +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/transcode/TranscodeEngine.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/transcode/TranscodeEngine.kt index 34a6610b..6c33950d 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/transcode/TranscodeEngine.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/transcode/TranscodeEngine.kt @@ -1,12 +1,19 @@ +@file:Suppress("TooGenericExceptionCaught", "UnnecessaryAbstractClass") + package com.otaliastudios.transcoder.internal.transcode +import android.media.MediaFormat import com.otaliastudios.transcoder.Transcoder import com.otaliastudios.transcoder.TranscoderOptions +import com.otaliastudios.transcoder.common.TrackType +import com.otaliastudios.transcoder.internal.Codecs import com.otaliastudios.transcoder.internal.DataSources +import com.otaliastudios.transcoder.internal.pipeline.Pipeline import com.otaliastudios.transcoder.internal.utils.Logger import com.otaliastudios.transcoder.internal.utils.trackMapOf +import com.otaliastudios.transcoder.sink.DataSink -internal abstract class TranscodeEngine { +abstract class TranscodeEngine { abstract fun validate(): Boolean @@ -18,29 +25,35 @@ internal abstract class TranscodeEngine { private val log = Logger("TranscodeEngine") private fun Throwable.isInterrupted(): Boolean { - if (this is InterruptedException) return true - if (this == this.cause) return false - return this.cause?.isInterrupted() ?: false + return when { + this is InterruptedException -> true + this == this.cause -> false + else -> this.cause?.isInterrupted() ?: false + } } @JvmStatic - fun transcode(options: TranscoderOptions) { + fun transcode( + options: TranscoderOptions, + function: ((TrackType, DataSink, Codecs, MediaFormat) -> Pipeline)?, + ) { log.i("transcode(): called...") var engine: TranscodeEngine? = null val dispatcher = TranscodeDispatcher(options) try { engine = DefaultTranscodeEngine( - dataSources = DataSources(options), - dataSink = options.dataSink, - strategies = trackMapOf( - video = options.videoTrackStrategy, - audio = options.audioTrackStrategy - ), - validator = options.validator, - videoRotation = options.videoRotation, - interpolator = options.timeInterpolator, - audioStretcher = options.audioStretcher, - audioResampler = options.audioResampler + dataSources = DataSources(options), + dataSink = options.dataSink, + strategies = trackMapOf( + video = options.videoTrackStrategy, + audio = options.audioTrackStrategy + ), + validator = options.validator, + videoRotation = options.videoRotation, + interpolator = options.timeInterpolator, + audioStretcher = options.audioStretcher, + audioResampler = options.audioResampler, + pipelineFactory = function ) if (!engine.validate()) { dispatcher.dispatchSuccess(Transcoder.SUCCESS_NOT_NEEDED) @@ -64,4 +77,4 @@ internal abstract class TranscodeEngine { } } } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/utils/ThreadPool.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/utils/ThreadPool.kt index 879920d6..2263d762 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/utils/ThreadPool.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/utils/ThreadPool.kt @@ -1,3 +1,5 @@ +@file:Suppress("MagicNumber") + package com.otaliastudios.transcoder.internal.utils import java.util.concurrent.LinkedBlockingQueue @@ -15,15 +17,16 @@ internal object ThreadPool { */ @JvmStatic val executor = ThreadPoolExecutor( - Runtime.getRuntime().availableProcessors() + 1, - Runtime.getRuntime().availableProcessors() + 1, - 60, - TimeUnit.SECONDS, - LinkedBlockingQueue(), - object : ThreadFactory { - private val count = AtomicInteger(1) - override fun newThread(r: Runnable): Thread { - return Thread(r, "TranscoderThread #" + count.getAndIncrement()) - } - }) -} \ No newline at end of file + Runtime.getRuntime().availableProcessors() + 1, + Runtime.getRuntime().availableProcessors() + 1, + 60, + TimeUnit.SECONDS, + LinkedBlockingQueue(), + object : ThreadFactory { + private val count = AtomicInteger(1) + override fun newThread(r: Runnable): Thread { + return Thread(r, "TranscoderThread #" + count.getAndIncrement()) + } + } + ) +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/utils/TrackMap.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/utils/TrackMap.kt index 46e64309..38877a63 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/utils/TrackMap.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/utils/TrackMap.kt @@ -54,4 +54,3 @@ private class DefaultTrackMap(video: T?, audio: T?) : MutableTrackMap { map[type] = value } } - diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/utils/debug.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/utils/debug.kt index d8dd1a47..ad01eb8e 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/utils/debug.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/utils/debug.kt @@ -1,3 +1,5 @@ +@file:Suppress("MagicNumber") + package com.otaliastudios.transcoder.internal.utils -internal fun stackTrace() = Thread.currentThread().stackTrace.drop(2).take(10).joinToString("\n") \ No newline at end of file +internal fun stackTrace() = Thread.currentThread().stackTrace.drop(2).take(10).joinToString("\n") diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/utils/eos.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/utils/eos.kt index c093facf..ee8fa271 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/utils/eos.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/utils/eos.kt @@ -2,19 +2,17 @@ package com.otaliastudios.transcoder.internal.utils import android.media.MediaCodec import com.otaliastudios.transcoder.common.TrackType -import com.otaliastudios.transcoder.internal.Segment -import com.otaliastudios.transcoder.internal.Segments import com.otaliastudios.transcoder.sink.DataSink import com.otaliastudios.transcoder.source.DataSource import java.nio.ByteBuffer // See https://github.com/natario1/Transcoder/issues/107 -internal fun DataSink.ignoringEos(ignore: () -> Boolean): DataSink - = EosIgnoringDataSink(this, ignore) +internal fun DataSink.ignoringEos(ignore: () -> Boolean): DataSink = + EosIgnoringDataSink(this, ignore) private class EosIgnoringDataSink( - private val sink: DataSink, - private val ignore: () -> Boolean, + private val sink: DataSink, + private val ignore: () -> Boolean, ) : DataSink by sink { private val info = MediaCodec.BufferInfo() override fun writeTrack(type: TrackType, byteBuffer: ByteBuffer, bufferInfo: MediaCodec.BufferInfo) { @@ -35,14 +33,14 @@ private class EosIgnoringDataSink( * This can happen if the user adds e.g. 1 minute of audio with 20 seconds of video. * In this case the video track must be stopped once the audio stops. */ -internal fun DataSource.forcingEos(force: () -> Boolean): DataSource - = EosForcingDataSource(this, force) +internal fun DataSource.forcingEos(force: () -> Boolean): DataSource = + EosForcingDataSource(this, force) private class EosForcingDataSource( - private val source: DataSource, - private val force: () -> Boolean, + private val source: DataSource, + private val force: () -> Boolean, ) : DataSource by source { override fun isDrained(): Boolean { return force() || source.isDrained } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/video/FrameDropper.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/video/FrameDropper.kt index 53087887..5c0043f3 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/video/FrameDropper.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/video/FrameDropper.kt @@ -1,8 +1,10 @@ +@file:Suppress("MagicNumber") + package com.otaliastudios.transcoder.internal.video import com.otaliastudios.transcoder.internal.utils.Logger -internal interface FrameDropper { +interface FrameDropper { fun shouldRender(timeUs: Long): Boolean } @@ -10,26 +12,33 @@ internal interface FrameDropper { * A very simple dropper, from * https://stackoverflow.com/questions/4223766/dropping-video-frames */ -internal fun FrameDropper(inputFps: Int, outputFps: Int) = object : FrameDropper { +fun frameDropper(inputFps: Int, outputFps: Int) = object : FrameDropper { private val log = Logger("FrameDropper") private val inputSpf = 1.0 / inputFps private val outputSpf = 1.0 / outputFps private var currentSpf = 0.0 private var frameCount = 0 + private var previousTs = 0.0 override fun shouldRender(timeUs: Long): Boolean { - currentSpf += inputSpf - if (frameCount++ == 0) { - log.v("RENDERING (first frame) - currentSpf=$currentSpf inputSpf=$inputSpf outputSpf=$outputSpf") - return true - } else if (currentSpf > outputSpf) { - currentSpf -= outputSpf - log.v("RENDERING - currentSpf=$currentSpf inputSpf=$inputSpf outputSpf=$outputSpf") - return true - } else { - log.v("DROPPING - currentSpf=$currentSpf inputSpf=$inputSpf outputSpf=$outputSpf") - return false + val timeS = timeUs / 1000.0 / 1000.0 + currentSpf += timeS - previousTs + previousTs = timeS + return when { + frameCount++ == 0 -> { + log.v("RENDERING (first frame) - currentSpf=$currentSpf inputSpf=$inputSpf outputSpf=$outputSpf") + true + } + currentSpf > outputSpf -> { + currentSpf -= outputSpf + log.v("RENDERING - currentSpf=$currentSpf inputSpf=$inputSpf outputSpf=$outputSpf") + true + } + else -> { + log.v("DROPPING - currentSpf=$currentSpf inputSpf=$inputSpf outputSpf=$outputSpf") + false + } } } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/video/VideoPublisher.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/video/VideoPublisher.kt index 1e9679bd..cb69ccd3 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/video/VideoPublisher.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/video/VideoPublisher.kt @@ -1,3 +1,5 @@ +@file:Suppress("MagicNumber") + package com.otaliastudios.transcoder.internal.video import android.opengl.EGL14 @@ -9,8 +11,7 @@ import com.otaliastudios.transcoder.internal.pipeline.Channel import com.otaliastudios.transcoder.internal.pipeline.State import com.otaliastudios.transcoder.internal.pipeline.Step - -internal class VideoPublisher: Step { +class VideoPublisher : Step { override val channel = Channel @@ -24,12 +25,12 @@ internal class VideoPublisher: Step } override fun step(state: State.Ok, fresh: Boolean): State { - if (state is State.Eos) { - return State.Eos(EncoderData.Empty) + return if (state is State.Eos) { + State.Eos(EncoderData.Empty) } else { - surface.setPresentationTime(state.value * 1000) + surface.setPresentationTime(state.value * 1000L) surface.swapBuffers() - return State.Ok(EncoderData.Empty) + State.Ok(EncoderData.Empty) } } @@ -37,4 +38,4 @@ internal class VideoPublisher: Step surface.release() core.release() } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/video/VideoRenderer.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/video/VideoRenderer.kt index cf470a66..e79e1805 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/video/VideoRenderer.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/video/VideoRenderer.kt @@ -1,7 +1,11 @@ +@file:Suppress("MagicNumber") + package com.otaliastudios.transcoder.internal.video import android.media.MediaFormat -import android.media.MediaFormat.* +import android.media.MediaFormat.KEY_FRAME_RATE +import android.media.MediaFormat.KEY_HEIGHT +import android.media.MediaFormat.KEY_WIDTH import android.view.Surface import com.otaliastudios.transcoder.internal.codec.DecoderChannel import com.otaliastudios.transcoder.internal.codec.DecoderData @@ -11,13 +15,15 @@ import com.otaliastudios.transcoder.internal.pipeline.State import com.otaliastudios.transcoder.internal.pipeline.Step import com.otaliastudios.transcoder.internal.utils.Logger +class VideoRenderer( + private val sourceRotation: Int, // intrinsic source rotation + private val extraRotation: Int, // any extra rotation in TranscoderOptions + private val targetFormat: MediaFormat, + flipY: Boolean = false, + private val isThumbnailer: Boolean = false, + private val getTargetTs: (() -> Long)? = null -internal class VideoRenderer( - private val sourceRotation: Int, // intrinsic source rotation - private val extraRotation: Int, // any extra rotation in TranscoderOptions - private val targetFormat: MediaFormat, - flipY: Boolean = false -): Step, DecoderChannel { +) : Step, DecoderChannel { private val log = Logger("VideoRenderer") @@ -51,10 +57,10 @@ internal class VideoRenderer( // Just a sanity check that the rotation coming from DataSource is not different from // the one found in the DataSource's MediaFormat for video. - val sourceRotation = runCatching { sourceFormat.getInteger(KEY_ROTATION_DEGREES) }.getOrElse { 0 } - if (sourceRotation != this.sourceRotation) { - error("Unexpected difference in rotation. DataSource=${this.sourceRotation}, MediaFormat=$sourceRotation") - } +// val sourceRotation = runCatching { sourceFormat.getInteger(KEY_ROTATION_DEGREES) }.getOrElse { 0 } +// if (sourceRotation != this.sourceRotation) { +// error("Unexpected difference in rotation. DataSource=${this.sourceRotation}, MediaFormat=$sourceRotation") +// } // Decoded video is rotated automatically starting from Android 5.0. Turn it off here because we // don't want to work on the rotated one, we apply rotation at rendering time. @@ -62,7 +68,7 @@ internal class VideoRenderer( sourceFormat.setInteger(KEY_ROTATION_DEGREES, 0) // Instead, apply the source rotation, plus the extra user rotation, to the renderer. - val rotation = (sourceRotation + extraRotation) % 360 + val rotation = (this.sourceRotation + extraRotation) % 360 frameDrawer.setRotation(rotation) // Depending on the rotation, we must also pass scale to the drawer due to how GL works. @@ -70,8 +76,16 @@ internal class VideoRenderer( val sourceWidth = sourceFormat.getInteger(KEY_WIDTH).toFloat() val sourceHeight = sourceFormat.getInteger(KEY_HEIGHT).toFloat() val sourceRatio = sourceWidth / sourceHeight - val targetWidth = (if (flip) targetFormat.getInteger(KEY_HEIGHT) else targetFormat.getInteger(KEY_WIDTH)).toFloat() - val targetHeight = (if (flip) targetFormat.getInteger(KEY_WIDTH) else targetFormat.getInteger(KEY_HEIGHT)).toFloat() + val targetWidth = ( + if (flip) targetFormat.getInteger(KEY_HEIGHT) else { + targetFormat.getInteger(KEY_WIDTH) + } + ).toFloat() + val targetHeight = ( + if (flip) targetFormat.getInteger(KEY_WIDTH) else { + targetFormat.getInteger(KEY_HEIGHT) + } + ).toFloat() val targetRatio = targetWidth / targetHeight var scaleX = 1f var scaleY = 1f @@ -83,9 +97,7 @@ internal class VideoRenderer( frameDrawer.setScale(scaleX, scaleY) // Create the frame dropper, now that we know the source FPS and the target FPS. - frameDropper = FrameDropper( - sourceFormat.getInteger(KEY_FRAME_RATE), - targetFormat.getInteger(KEY_FRAME_RATE)) + frameDropper = frameDropper(sourceFormat.getInteger(KEY_FRAME_RATE), targetFormat.getInteger(KEY_FRAME_RATE)) return frameDrawer.surface } @@ -96,7 +108,9 @@ internal class VideoRenderer( state.value.release(false) State.Eos(0L) } else { - if (frameDropper.shouldRender(state.value.timeUs)) { + if ((isThumbnailer && shouldRenderThumbnail(state.value.timeUs, getTargetTs?.invoke() ?: -1)) || + frameDropper.shouldRender(state.value.timeUs) + ) { state.value.release(true) frameDrawer.drawFrame() State.Ok(state.value.timeUs) @@ -107,7 +121,16 @@ internal class VideoRenderer( } } + private fun shouldRenderThumbnail(timeUs: Long, target: Long): Boolean { + if (timeUs >= target) { + log.i("Renderer: Rendering $timeUs : target: $target") + } else { + log.i("Renderer: Skipping $timeUs : target: $target") + } + return timeUs >= target + } + override fun release() { frameDrawer.release() } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/video/VideoSnapshots.kt b/lib/src/main/java/com/otaliastudios/transcoder/internal/video/VideoSnapshots.kt index e8ba3e35..74b61563 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/video/VideoSnapshots.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/video/VideoSnapshots.kt @@ -1,3 +1,5 @@ +@file:Suppress("MagicNumber") + package com.otaliastudios.transcoder.internal.video import android.graphics.Bitmap @@ -9,26 +11,21 @@ import android.opengl.GLES20 import com.otaliastudios.opengl.core.EglCore import com.otaliastudios.opengl.core.Egloo import com.otaliastudios.opengl.surface.EglOffscreenSurface -import com.otaliastudios.opengl.surface.EglSurface import com.otaliastudios.transcoder.internal.pipeline.BaseStep import com.otaliastudios.transcoder.internal.pipeline.Channel import com.otaliastudios.transcoder.internal.pipeline.State import com.otaliastudios.transcoder.internal.utils.Logger -import com.otaliastudios.transcoder.thumbnail.Thumbnail import java.nio.ByteBuffer import java.nio.ByteOrder -import kotlin.math.abs -internal class VideoSnapshots( - format: MediaFormat, - requests: List, - private val accuracyUs: Long, - private val onSnapshot: (Long, Bitmap) -> Unit +class VideoSnapshots( + format: MediaFormat, + private val fetchRequest: () -> Request?, + private val onSnapshot: (Long, Bitmap) -> Unit ) : BaseStep() { private val log = Logger("VideoSnapshots") override val channel = Channel - private val requests = requests.toMutableList() private val width = format.getInteger(KEY_WIDTH) private val height = format.getInteger(KEY_HEIGHT) private val core = EglCore(EGL14.EGL_NO_CONTEXT, EglCore.FLAG_RECORDABLE) @@ -37,13 +34,12 @@ internal class VideoSnapshots( } override fun step(state: State.Ok, fresh: Boolean): State { - if (requests.isEmpty()) return state - - val expectedUs = requests.first() - val deltaUs = abs(expectedUs - state.value) + val request = fetchRequest() ?: return state + val expectedUs = request.pts + val accuracyUs = request.threshold + val deltaUs = (expectedUs - state.value) if (deltaUs < accuracyUs || (state is State.Eos && expectedUs > state.value)) { log.i("Request MATCHED! expectedUs=$expectedUs actualUs=${state.value} deltaUs=$deltaUs") - requests.removeFirst() val buffer = ByteBuffer.allocateDirect(width * height * 4) buffer.order(ByteOrder.LITTLE_ENDIAN) GLES20.glReadPixels(0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, buffer) @@ -62,4 +58,5 @@ internal class VideoSnapshots( surface.release() core.release() } -} \ No newline at end of file + data class Request(val pts: Long, val threshold: Long) +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/resample/AudioResampler.java b/lib/src/main/java/com/otaliastudios/transcoder/resample/AudioResampler.java index 00c4290e..5ef8db62 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/resample/AudioResampler.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/resample/AudioResampler.java @@ -22,6 +22,17 @@ public interface AudioResampler { */ void resample(@NonNull final ShortBuffer inputBuffer, int inputSampleRate, @NonNull final ShortBuffer outputBuffer, int outputSampleRate, int channels); + /** + * createStream() and destroyStream() only to be implemented on Resamplers + * following a stream approach for continuous input buffers. Not for static one shot methods + * to resample. + * @param inputSampleRate the input sample rate + * @param outputSampleRate the output sample rate + * @param numChannels the number of channels + */ + void createStream(int inputSampleRate, int outputSampleRate, int numChannels); + void destroyStream(); + AudioResampler DOWNSAMPLE = new DownsampleAudioResampler(); AudioResampler UPSAMPLE = new UpsampleAudioResampler(); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/resample/DefaultAudioResampler.java b/lib/src/main/java/com/otaliastudios/transcoder/resample/DefaultAudioResampler.java index 9d87c911..d8efb290 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/resample/DefaultAudioResampler.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/resample/DefaultAudioResampler.java @@ -20,4 +20,14 @@ public void resample(@NonNull ShortBuffer inputBuffer, int inputSampleRate, @Non PASSTHROUGH.resample(inputBuffer, inputSampleRate, outputBuffer, outputSampleRate, channels); } } + + @Override + public void createStream(int inputSampleRate, int outputSampleRate, int numChannels) { + // do nothing + } + + @Override + public void destroyStream() { + // do nothing + } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/resample/DownsampleAudioResampler.java b/lib/src/main/java/com/otaliastudios/transcoder/resample/DownsampleAudioResampler.java index b22bf137..3e4486b3 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/resample/DownsampleAudioResampler.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/resample/DownsampleAudioResampler.java @@ -44,4 +44,14 @@ public void resample(@NonNull ShortBuffer inputBuffer, int inputSampleRate, @Non } } } + + @Override + public void createStream(int inputSampleRate, int outputSampleRate, int numChannels) { + // do nothing + } + + @Override + public void destroyStream() { + // do nothing + } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/resample/PassThroughAudioResampler.java b/lib/src/main/java/com/otaliastudios/transcoder/resample/PassThroughAudioResampler.java index 58423c5d..baa12bb8 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/resample/PassThroughAudioResampler.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/resample/PassThroughAudioResampler.java @@ -18,4 +18,14 @@ public void resample(@NonNull ShortBuffer inputBuffer, int inputSampleRate, } outputBuffer.put(inputBuffer); } + + @Override + public void createStream(int inputSampleRate, int outputSampleRate, int numChannels) { + // do nothing + } + + @Override + public void destroyStream() { + // do nothing + } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/resample/UpsampleAudioResampler.java b/lib/src/main/java/com/otaliastudios/transcoder/resample/UpsampleAudioResampler.java index 69fb6251..def9d107 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/resample/UpsampleAudioResampler.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/resample/UpsampleAudioResampler.java @@ -28,7 +28,7 @@ public void resample(@NonNull ShortBuffer inputBuffer, int inputSampleRate, @Non int remainingFakeSamples = fakeSamples; float remainingInputSamplesRatio = ratio(remainingInputSamples, inputSamples); float remainingFakeSamplesRatio = ratio(remainingFakeSamples, fakeSamples); - while (remainingInputSamples > 0 && remainingFakeSamples > 0) { + while (remainingInputSamples > 0 && remainingFakeSamples >= 0) { // Will this be an input sample or a fake sample? // Choose the one with the bigger ratio. if (remainingInputSamplesRatio >= remainingFakeSamplesRatio) { @@ -45,6 +45,16 @@ public void resample(@NonNull ShortBuffer inputBuffer, int inputSampleRate, @Non } } + @Override + public void createStream(int inputSampleRate, int outputSampleRate, int numChannels) { + // do nothing + } + + @Override + public void destroyStream() { + // do nothing + } + /** * We have different options here. * 1. Return a 0 sample. diff --git a/lib/src/main/java/com/otaliastudios/transcoder/sink/InvalidOutputFormatException.java b/lib/src/main/java/com/otaliastudios/transcoder/sink/InvalidOutputFormatException.java index 93f841d5..524ae18a 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/sink/InvalidOutputFormatException.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/sink/InvalidOutputFormatException.java @@ -23,7 +23,7 @@ /** * One of the exceptions possibly thrown by - * {@link Transcoder#transcode(TranscoderOptions)}, which means it can be + * {@link Transcoder#transcode(TranscoderOptions, kotlin.jvm.functions.Function4)}, which means it can be * passed to {@link TranscoderListener#onTranscodeFailed(Throwable)}. */ public class InvalidOutputFormatException extends RuntimeException { diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/ClipDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/ClipDataSource.java index 48a05bbd..485c11d0 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/ClipDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/ClipDataSource.java @@ -15,7 +15,7 @@ public ClipDataSource(@NonNull DataSource source, long clipStartUs) { public ClipDataSource(@NonNull DataSource source, long clipStartUs, long clipEndUs) { super(new TrimDataSource(source, clipStartUs, - getSourceDurationUs(source) - clipEndUs)); + Math.max(getSourceDurationUs(source) - clipEndUs, 0L))); } private static long getSourceDurationUs(@NonNull DataSource source) { diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java index 963e76d5..2be68243 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java @@ -1,5 +1,6 @@ package com.otaliastudios.transcoder.source; +import android.graphics.Bitmap; import android.media.MediaFormat; import androidx.annotation.NonNull; @@ -8,6 +9,7 @@ import com.otaliastudios.transcoder.common.TrackType; import java.nio.ByteBuffer; +import java.util.ArrayList; /** * Represents the source of input data. @@ -111,6 +113,26 @@ public interface DataSource { */ void releaseTrack(@NonNull TrackType type); + + /** + * Returns closest key frame to the required position. + * @param positionUs position + */ + default Bitmap getFrameAtPosition(long positionUs, int width, int height) { + return null; + }; + + default long requestKeyFrameTimestamps() { return -1;} + + default ArrayList getKeyFrameTimestamps() { + return new ArrayList<>(); + } + + default long getSeekThreshold() { + return 0; + } + + default String mediaId() { return "";} /** * Rewinds this source, moving it to its default state. * To be used again, tracks will be selected again. @@ -131,4 +153,7 @@ class Chunk { public long timeUs; public boolean render; } + class KeyFrames { + public ArrayList keyFrameTimestampListUs; + } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/DataSourceWrapper.java b/lib/src/main/java/com/otaliastudios/transcoder/source/DataSourceWrapper.java index 7fb6c1c3..a438d8c6 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/DataSourceWrapper.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/DataSourceWrapper.java @@ -1,6 +1,7 @@ package com.otaliastudios.transcoder.source; +import android.graphics.Bitmap; import android.media.MediaFormat; import androidx.annotation.NonNull; @@ -97,6 +98,11 @@ public void initialize() { } } + @Override + public Bitmap getFrameAtPosition(long positionUs, int width, int height) { + return mSource.getFrameAtPosition(positionUs, width, height); + } + @Override public void deinitialize() { mSource.deinitialize(); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/DefaultDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/DefaultDataSource.java index 70e88ceb..c9eefe78 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/DefaultDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/DefaultDataSource.java @@ -1,8 +1,10 @@ package com.otaliastudios.transcoder.source; +import android.graphics.Bitmap; import android.media.MediaExtractor; import android.media.MediaFormat; import android.media.MediaMetadataRetriever; +import android.os.Build; import androidx.annotation.NonNull; import androidx.annotation.Nullable; @@ -14,6 +16,8 @@ import com.otaliastudios.transcoder.internal.utils.MutableTrackMap; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.concurrent.atomic.AtomicInteger; @@ -43,6 +47,9 @@ public abstract class DefaultDataSource implements DataSource { private long mDontRenderRangeStart = -1L; private long mDontRenderRangeEnd = -1L; + private final ArrayList keyFrameTimestamps = new ArrayList<>(); + private final long SEEK_THRESHOLD = 10001L; // 10ms because extractor doesn't seek accurately + private final boolean VERBOSE = false; @Override public void initialize() { LOG.i("initialize(): initializing..."); @@ -51,7 +58,7 @@ public void initialize() { initializeExtractor(mExtractor); } catch (IOException e) { LOG.e("Got IOException while trying to open MediaExtractor.", e); - throw new RuntimeException(e); +// throw new RuntimeException(e); } mMetadata = new MediaMetadataRetriever(); initializeRetriever(mMetadata); @@ -88,6 +95,64 @@ public void initialize() { } */ } + @Override + public ArrayList getKeyFrameTimestamps() { + return keyFrameTimestamps; + } + + private Boolean lastKeyFrame = false; + @Override + public long requestKeyFrameTimestamps() { + + if(lastKeyFrame) return -1L; + if(keyFrameTimestamps.size() > 0) { + mExtractor.seekTo(keyFrameTimestamps.get(keyFrameTimestamps.size() - 1) + SEEK_THRESHOLD, MediaExtractor.SEEK_TO_NEXT_SYNC); + } + + long sampleTime = mExtractor.getSampleTime(); + + if (sampleTime == -1 || (keyFrameTimestamps.size() > 0 && sampleTime == keyFrameTimestamps.get(keyFrameTimestamps.size() - 1))) { + lastKeyFrame = true; + if (!keyFrameTimestamps.isEmpty()) { + mExtractor.seekTo(keyFrameTimestamps.get(keyFrameTimestamps.size() - 1) + SEEK_THRESHOLD, MediaExtractor.SEEK_TO_PREVIOUS_SYNC); + } else { + mExtractor.seekTo(0L, MediaExtractor.SEEK_TO_PREVIOUS_SYNC); + } + return -1; + } +// LOG.i("keyFrameStartTime:" + sampleTime); + + int count = 0; + long lastSampleTime = -1L; + int prefetchCount = 100; + while (sampleTime >= 0L && sampleTime != lastSampleTime && count < prefetchCount) { + if ((mExtractor.getSampleFlags() & MediaExtractor.SAMPLE_FLAG_SYNC) > 0) { + if (!keyFrameTimestamps.isEmpty() && sampleTime <= keyFrameTimestamps.get(keyFrameTimestamps.size() - 1)) { + Collections.sort(keyFrameTimestamps); + } + // list is ordered, so only last item can be same. + if (keyFrameTimestamps.isEmpty() || keyFrameTimestamps.get(keyFrameTimestamps.size() - 1) != sampleTime) { + keyFrameTimestamps.add(sampleTime); + } else { + sampleTime = -1; + break; + } + } + mExtractor.seekTo(sampleTime + SEEK_THRESHOLD, MediaExtractor.SEEK_TO_NEXT_SYNC); + lastSampleTime = sampleTime; + sampleTime = mExtractor.getSampleTime(); + count++; + } +// LOG.i("keyFrameStopCount:" + keyFrameTimestamps); + return sampleTime; + } + + + @Override + public long getSeekThreshold() { + return SEEK_THRESHOLD; + } + @Override public void deinitialize() { LOG.i("deinitialize(): deinitializing..."); @@ -135,6 +200,15 @@ public void releaseTrack(@NonNull TrackType type) { } } + @Override + public Bitmap getFrameAtPosition(long positionUs, int width, int height) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) { + return mMetadata.getScaledFrameAtTime(positionUs, MediaMetadataRetriever.OPTION_CLOSEST_SYNC, width, height); + } + Bitmap bitmap = mMetadata.getFrameAtTime(positionUs); + return Bitmap.createScaledBitmap(bitmap, width, height, true); + + } protected abstract void initializeExtractor(@NonNull MediaExtractor extractor) throws IOException; protected abstract void initializeRetriever(@NonNull MediaMetadataRetriever retriever); @@ -143,13 +217,15 @@ public void releaseTrack(@NonNull TrackType type) { public long seekTo(long desiredPositionUs) { boolean hasVideo = mSelectedTracks.contains(TrackType.VIDEO); boolean hasAudio = mSelectedTracks.contains(TrackType.AUDIO); - LOG.i("seekTo(): seeking to " + (mOriginUs + desiredPositionUs) - + " originUs=" + mOriginUs - + " extractorUs=" + mExtractor.getSampleTime() - + " externalUs=" + desiredPositionUs - + " hasVideo=" + hasVideo - + " hasAudio=" + hasAudio); if (hasVideo && hasAudio) { + if (VERBOSE) { + LOG.i("seekTo(): seeking to " + (mOriginUs + desiredPositionUs) + + " originUs=" + mOriginUs + + " extractorUs=" + mExtractor.getSampleTime() + + " externalUs=" + desiredPositionUs + + " hasVideo=" + hasVideo + + " hasAudio=" + hasAudio); + } // Special case: audio can be moved to any timestamp, but video will only stop in // sync frames. MediaExtractor is not smart enough to sync the two tracks at the // video sync frame, so we must take care of this with the following trick. @@ -162,7 +238,15 @@ public long seekTo(long desiredPositionUs) { mExtractor.seekTo(mExtractor.getSampleTime(), MediaExtractor.SEEK_TO_CLOSEST_SYNC); LOG.v("seekTo(): seek workaround completed. (extractorUs=" + mExtractor.getSampleTime() + ")"); } else { - mExtractor.seekTo(mOriginUs + desiredPositionUs, MediaExtractor.SEEK_TO_PREVIOUS_SYNC); + if (VERBOSE) { + LOG.i("seekTo(): seeking to " + (desiredPositionUs) + + " originUs=" + mOriginUs + + " extractorUs=" + mExtractor.getSampleTime() + + " externalUs=" + desiredPositionUs + + " hasVideo=" + hasVideo + + " hasAudio=" + hasAudio); + } + mExtractor.seekTo(desiredPositionUs, MediaExtractor.SEEK_TO_PREVIOUS_SYNC); } mDontRenderRangeStart = mExtractor.getSampleTime(); mDontRenderRangeEnd = mOriginUs + desiredPositionUs; diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java index 28f1b788..4ad12939 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/TrimDataSource.java @@ -55,8 +55,8 @@ public void initialize() { "start=" + trimStartUs + ", " + "end=" + trimEndUs + ", " + "duration=" + duration); - throw new IllegalArgumentException( - "Trim values cannot be greater than media duration."); +// throw new IllegalArgumentException( +// "Trim values cannot be greater than media duration."); } LOG.i("initialize(): duration=" + duration + " trimStart=" + trimStartUs @@ -97,6 +97,12 @@ public boolean canReadTrack(@NonNull TrackType type) { return super.canReadTrack(type); } + @Override + public void readTrack(@NonNull Chunk chunk) { + super.readTrack(chunk); + chunk.timeUs = chunk.timeUs - trimStartUs; + } + @Override public boolean isDrained() { // Enforce the trim end: this works thanks to the fact that extraDurationUs is added diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/UriDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/UriDataSource.java index 82562773..16b67a44 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/UriDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/UriDataSource.java @@ -30,6 +30,16 @@ protected void initializeExtractor(@NonNull MediaExtractor extractor) throws IOE @Override protected void initializeRetriever(@NonNull MediaMetadataRetriever retriever) { - retriever.setDataSource(context, uri); + try { + retriever.setDataSource(context, uri); + } catch (IllegalArgumentException ignored) { + + } } + + @Override + public String mediaId() { + return uri.toString(); + } + } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/CoverThumbnailRequest.kt b/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/CoverThumbnailRequest.kt index 4edebede..a511e5be 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/CoverThumbnailRequest.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/CoverThumbnailRequest.kt @@ -2,4 +2,4 @@ package com.otaliastudios.transcoder.thumbnail class CoverThumbnailRequest : ThumbnailRequest { override fun locate(durationUs: Long) = listOf(0L) -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/SingleThumbnailRequest.kt b/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/SingleThumbnailRequest.kt index 7e17a213..a40812ee 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/SingleThumbnailRequest.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/SingleThumbnailRequest.kt @@ -1,10 +1,10 @@ package com.otaliastudios.transcoder.thumbnail -class SingleThumbnailRequest(private val positionUs: Long) : ThumbnailRequest { +@Suppress("MagicNumber") +open class SingleThumbnailRequest(private val positionUs: Long) : ThumbnailRequest { override fun locate(durationUs: Long): List { - require(positionUs in 0L..durationUs) { - "Thumbnail position is out of range. position=$positionUs range=${0L..durationUs}" - } + val randomizer = (positionUs / 1000) % 10000 + val positionUs = positionUs.coerceIn(0L..durationUs - 135005 - randomizer) return listOf(positionUs) } -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/Thumbnail.kt b/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/Thumbnail.kt index 760ad983..fe96f706 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/Thumbnail.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/Thumbnail.kt @@ -2,8 +2,8 @@ package com.otaliastudios.transcoder.thumbnail import android.graphics.Bitmap -class Thumbnail internal constructor( - val request: ThumbnailRequest, - val positionUs: Long, - val bitmap: Bitmap -) \ No newline at end of file +class Thumbnail constructor( + val request: ThumbnailRequest, + val positionUs: Long, + val bitmap: Bitmap +) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/ThumbnailRequest.kt b/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/ThumbnailRequest.kt index 48019689..11b2f208 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/ThumbnailRequest.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/ThumbnailRequest.kt @@ -3,6 +3,11 @@ package com.otaliastudios.transcoder.thumbnail interface ThumbnailRequest { fun locate(durationUs: Long): List + fun threshold(): Long = 0 + + fun sourcePath() : String = "" + + fun sourceId() : String = "" // Could make it so that if locate() is empty, accept is called for each frame (no seeking). // But this only makes sense if accept signature has more information (segment, ...), and // it should also have a way to say - we're done, stop transcoding. @@ -10,4 +15,4 @@ interface ThumbnailRequest { // Could add resizing per request // val resizer = PassThroughResizer() -} \ No newline at end of file +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/UniformThumbnailRequest.kt b/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/UniformThumbnailRequest.kt index b87bf385..ba250a78 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/UniformThumbnailRequest.kt +++ b/lib/src/main/java/com/otaliastudios/transcoder/thumbnail/UniformThumbnailRequest.kt @@ -18,4 +18,4 @@ class UniformThumbnailRequest(private val count: Int) : ThumbnailRequest { } return list } -} \ No newline at end of file +}