由于假定的无序帧导致Android MediaMuxer崩溃

Question

我想记录从麦克风(也从蓝牙耳机麦克风，沿着这条路)的音频轨道，并将其转换为MPEG4 AAC格式。根据我的项目中与后端的通信规范的要求，音频必须分割成短(0.5秒长)块。作为一个简化的示例，我只是在提供的代码中将这些块保存为缓存中的文件(而不将它们发送到后端)。

为了实现这一点，我用AudioRecord以PCM-16格式录制音频，然后用MediaCodec将其转换为AAC，最后将其保存为MediaMuxer的MPEG4文件。

示例代码(基于这个例子)：

private const val TAG = "RECORDING"

private const val AUDIO_CHUNK_LENGTH_MS = 500L

private const val RECORDER_SAMPLERATE = 44100
private const val RECORDER_CHANNELS = AudioFormat.CHANNEL_IN_MONO
private const val RECORDER_AUDIO_ENCODING = AudioFormat.ENCODING_PCM_16BIT
private val BUFFER_SIZE = AudioRecord.getMinBufferSize(RECORDER_SAMPLERATE, RECORDER_CHANNELS, RECORDER_AUDIO_ENCODING)

class RecordingUtil2(
    val context: Context,
    private val dispatcher: CoroutineDispatcher = Dispatchers.Default
) {

    lateinit var audioRecord: AudioRecord
    lateinit var encoder: MediaCodec
    lateinit var mediaMuxer: MediaMuxer
    var trackId: Int = 0

    var chunkCuttingJob: Job? = null
    var recordingJob: Job? = null

    var audioStartTimeNs: Long = 0

    var currentFile: File? = null

    var chunkEnd = false

    private fun prepareRecorder() {
        audioRecord = AudioRecord(
            MediaRecorder.AudioSource.MIC, RECORDER_SAMPLERATE,
            AudioFormat.CHANNEL_IN_MONO,
            AudioFormat.ENCODING_PCM_16BIT, BUFFER_SIZE * 10
        )
    }

    private suspend fun startRecording() {
        Timber.tag(TAG).i("started recording, buffer size $BUFFER_SIZE")
        createTempFile()
        prepareRecorder()

        try {
            encoder = createMediaCodec(BUFFER_SIZE)
            encoder.start()
            createMuxer(encoder.outputFormat, currentFile!!)
            mediaMuxer.start()
        } catch (exception: Exception) {
            Timber.tag(TAG).w(exception)
        }

        audioStartTimeNs = System.nanoTime()
        audioRecord.startRecording()

        var bufferInfo = MediaCodec.BufferInfo()

        chunkCuttingJob = CoroutineScope(dispatcher).launch {
            while (isActive) {
                delay(AUDIO_CHUNK_LENGTH_MS)
                cutChunk()
            }
        }
        recordingJob = CoroutineScope(dispatcher).launch {
            val buffer2 = ByteArray(BUFFER_SIZE)

            do {
                val bytes = audioRecord.read(buffer2, 0, BUFFER_SIZE)

                if (bytes != BUFFER_SIZE) {
                    Timber.tag(TAG).w("read less bytes than full buffer ($bytes/$BUFFER_SIZE)")
                }

                encodeRawAudio(encoder, mediaMuxer, buffer2, bytes, bufferInfo, !isActive || chunkEnd)

                if (chunkEnd) {
                    recreateEncoderAndMuxer()
                    bufferInfo = MediaCodec.BufferInfo()
                    // delay here causes crash after first cut
                    //delay(100)
                }
                // delay here fixes crash in most cases
                //delay(100)
            } while(isActive)
        }
    }

    private fun recreateEncoderAndMuxer() {
        createTempFile()
        chunkEnd = false
        audioStartTimeNs = System.nanoTime()
        encoder.stop()
        encoder.release()
        encoder = createMediaCodec(BUFFER_SIZE)
        encoder.start()
        mediaMuxer.stop()
        mediaMuxer.release()
        createMuxer(encoder.outputFormat, currentFile!!)
        mediaMuxer.start()
    }

    private fun encodeRawAudio(encoder: MediaCodec, muxer: MediaMuxer, bytes: ByteArray, byteCount: Int, bufferInfo: MediaCodec.BufferInfo, last: Boolean = false) {
        with(encoder) {
            val infputBufferIndex = dequeueInputBuffer(10_000)
            val inputBuffer = getInputBuffer(infputBufferIndex)
            inputBuffer?.clear()
            inputBuffer?.put(bytes)
            val presentationTimeUs: Long = (System.nanoTime() - audioStartTimeNs) / 1000

            queueInputBuffer(infputBufferIndex, 0, byteCount, presentationTimeUs, if (last) BUFFER_FLAG_END_OF_STREAM else 0)

            var outputBufferIndex = dequeueOutputBuffer(bufferInfo, 0)
            Timber.tag(TAG).d("encoding $byteCount bytes, last = $last, time: $presentationTimeUs, buffer time: ${bufferInfo.presentationTimeUs}")

            while (outputBufferIndex != MediaCodec.INFO_TRY_AGAIN_LATER) {
                if (outputBufferIndex >= 0) {
                    val outputBuffer = getOutputBuffer(outputBufferIndex)

                    outputBuffer?.position(bufferInfo.offset)
                    outputBuffer?.limit(bufferInfo.offset + bufferInfo.size)

                    if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != MediaCodec.BUFFER_FLAG_CODEC_CONFIG) {
                        val data = ByteArray(outputBuffer!!.remaining())
                        outputBuffer.get(data)

                        muxer.writeSampleData(trackId, outputBuffer, bufferInfo)
                    }

                    outputBuffer?.clear()
                    releaseOutputBuffer(outputBufferIndex, false)
                }

                outputBufferIndex = encoder.dequeueOutputBuffer(bufferInfo, 0)
            }
        }
    }

    private fun cutChunk() {
        Timber.tag(TAG).i("cutting chunk")
        chunkEnd = true
    }

    private fun stopRecording() {
        Timber.tag(TAG).i("stopped recording")
        chunkCuttingJob?.cancel()
        chunkCuttingJob = null
        recordingJob?.cancel()
        recordingJob = null
        audioRecord.stop()
        encoder.release()
        mediaMuxer.stop()
        mediaMuxer.release()
    }

    suspend fun record(isRecording: Boolean) {
        if (isRecording) {
            startRecording()
        } else {
            stopRecording()
        }
    }

    private fun createMediaCodec(bufferSize: Int, existing: MediaCodec? = null): MediaCodec {
        val mediaFormat = MediaFormat().apply {
            setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_AUDIO_AAC)
            setInteger(MediaFormat.KEY_BIT_RATE, 32000)
            setInteger(MediaFormat.KEY_CHANNEL_COUNT, 1)
            setInteger(MediaFormat.KEY_SAMPLE_RATE, RECORDER_SAMPLERATE)
            setInteger(MediaFormat.KEY_AAC_PROFILE, CodecProfileLevel.AACObjectLC)
            setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, bufferSize)
        }

        val encoderString = MediaCodecList(MediaCodecList.REGULAR_CODECS).findEncoderForFormat(mediaFormat)

        Timber.tag(TAG).d("chosen codec: $encoderString")
        val mediaCodec = existing ?: MediaCodec.createByCodecName(encoderString)

        try {
            mediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
        } catch (e: Exception) {
            Timber.tag(TAG).w(e)
            mediaCodec.release()
        }
        return mediaCodec
    }

    private fun createMuxer(format: MediaFormat, file: File) {
        try {
            file.createNewFile()
            mediaMuxer = MediaMuxer(file.absolutePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
            trackId = mediaMuxer.addTrack(format)
        } catch (e: java.lang.Exception) {
            Timber.tag(TAG).e(e)
        }
    }

    private var currentIndex: Int = 0

    private fun createTempFile() {
        currentFile = File(context.cacheDir, "$currentIndex.m4a").also { it.createNewFile() }
        currentIndex++
    }
}

我在coroutine中运行这个代码，就像：

class MyViewModel : ViewModel() {
    fun startRecording() {
        val recordingUtil = RecordingUtil2(...)
        viewModelScope.launch(Dispatchers.Default) {
            recordingUtil.record(true)
        }
    }
}

我面临的问题是，在将几个块保存到连续文件后，MediaMuxer会由于MPEG4Writer中的异常而崩溃

E/MPEG4Writer: do not support out of order frames (timestamp: 13220 < last: 23219 for Audio track

但是，正如您在提供的代码中所看到的，时间戳是逐步生成的，并按照适当的顺序用作MediaCodec.queueInputBuffer(...)参数。

有趣的是(并可能暗示出了问题所在)，MPEG4Writer的异常消息说，每次的最后时间戳是23219 ，就像它是一个常量一样，而从本机平台代码判断它确实应该显示以前的帧时间戳，它不太可能是远大于0的常量。

更多来自崩溃的日志(用于上下文)

I/MPEG4Writer: Normal stop process
D/MPEG4Writer: Audio track stopping. Stop source
I/MPEG4Writer: Received total/0-length (22/1) buffers and encoded 22 frames. - Audio
D/MPEG4Writer: Audio track source stopping
D/MPEG4Writer: Audio track source stopped
I/MPEG4Writer: Audio track drift time: 0 us
D/MPEG4Writer: Audio track stopped. Stop source
D/MPEG4Writer: Stopping writer thread
D/MPEG4Writer: 0 chunks are written in the last batch
D/MPEG4Writer: Writer thread stopped
I/MPEG4Writer: Ajust the moov start time from 44099 us -> 44099 us
I/MPEG4Writer: The mp4 file will not be streamable.
D/MPEG4Writer: Audio track stopping. Stop source
D/RECORDING: encoding 3528 bytes, last = false, time: 79102, buffer time: 0
D/RECORDING: encoding 3528 bytes, last = false, time: 85883, buffer time: 0
D/RECORDING: encoding 3528 bytes, last = false, time: 89383, buffer time: 79102
I/MPEG4Writer: setStartTimestampUs: 79102 from Audio track
I/MPEG4Writer: Earliest track starting time: 79102
E/MPEG4Writer: do not support out of order frames (timestamp: 13220 < last: 23219 for Audio track
E/MPEG4Writer: 0 frames to dump timeStamps in Audio track 
I/MPEG4Writer: Received total/0-length (3/0) buffers and encoded 2 frames. - Audio
I/MPEG4Writer: Audio track drift time: 0 us
E/MediaAdapter: pushBuffer called before start
E/AndroidRuntime: FATAL EXCEPTION: DefaultDispatcher-worker-1
E/AndroidRuntime: FATAL EXCEPTION: DefaultDispatcher-worker-1
    Process: com.example, PID: 23499
    java.lang.IllegalStateException: writeSampleData returned an error

在连续录制和保存音频块的情况下的日志：

I/MPEG4Writer: Normal stop process
D/MPEG4Writer: Audio track stopping. Stop source
D/MPEG4Writer: Audio track source stopping
I/MPEG4Writer: Received total/0-length (18/0) buffers and encoded 18 frames. - Audio
D/MPEG4Writer: Audio track source stopped
I/MPEG4Writer: Audio track drift time: 0 us
D/MPEG4Writer: Audio track stopped. Stop source
D/MPEG4Writer: Stopping writer thread
D/MPEG4Writer: 0 chunks are written in the last batch
D/MPEG4Writer: Writer thread stopped
I/MPEG4Writer: Ajust the moov start time from 45890 us -> 45890 us
I/MPEG4Writer: The mp4 file will not be streamable.
D/MPEG4Writer: Audio track stopping. Stop source
D/RECORDING: encoding 3528 bytes, last = false, time: 44099, buffer time: 0
D/RECORDING: encoding 3528 bytes, last = false, time: 74366, buffer time: 44099
I/MPEG4Writer: setStartTimestampUs: 44099 from Audio track
I/MPEG4Writer: Earliest track starting time: 44099
D/RECORDING: encoding 3528 bytes, last = false, time: 116122, buffer time: 80805
D/RECORDING: encoding 3528 bytes, last = false, time: 156789, buffer time: 104025
D/RECORDING: encoding 3528 bytes, last = false, time: 196940, buffer time: 152221
D/RECORDING: encoding 3528 bytes, last = false, time: 235010, buffer time: 176108
D/RECORDING: encoding 3528 bytes, last = false, time: 275232, buffer time: 243989
D/RECORDING: encoding 3528 bytes, last = false, time: 316400, buffer time: 267209
D/RECORDING: encoding 3528 bytes, last = false, time: 361290, buffer time: 313871
D/RECORDING: encoding 3528 bytes, last = false, time: 401305, buffer time: 338259
D/RECORDING: encoding 3528 bytes, last = false, time: 441019, buffer time: 412824
D/RECORDING: encoding 3528 bytes, last = false, time: 481193, buffer time: 436044
I/RECORDING: cutting chunk
D/RECORDING: encoding 3528 bytes, last = true, time: 518624, buffer time: 458978
I/MediaCodec: Codec shutdown complete

我注意到，崩溃场景中的日志显示，BufferInfo包含两个初始帧的时间戳=0，而非崩溃日志总是只有一个这样的帧。然而，我观察到了同样的崩溃，有时只有一个时间戳=0帧，所以它可能与此无关。

有人能帮我解决这个问题吗？

Answer 1

看看带有安卓MediaCodec和MediaMuxer的Muxing音频的问题。

最好的猜测:编码器正在对输出做一些事情--也许把一个输入包分成两个输出包--这需要它合成一个时间戳。它获取数据包开始的时间戳，并根据比特率和字节数添加一个值。如果您生成的时间戳具有相当正确的表示时间，则在生成“中间”时间戳时，不应该看到时间戳向后移动。@fadden

我引用@fadden的评论。它解释了为什么MediaCodec会产生这种意外，而时间戳在某种程度上并不是增量的。

所以，你说

但是，正如您在提供的代码中所看到的，时间戳是逐步生成的，并用作MediaCodec.queueInputBuffer(.)以适当的顺序进行辩论。

这与您的代码无关，而是要提供单一的时间戳。如果您清楚地查看错误消息。

java.lang.IllegalStateException: writeSampleData returned an error

该错误被抛出给方法writeSampleData。因此，只需在BufferInfo.presentationTimeUs之前进行一些日志记录以查看muxer.writeSampleData。你会看到惊喜的。