Android平台RTSP|RTMP播放器如何回调YUV或RGB数据

技术背景

我们在做Android平台RTSP、RTMP播放器的时候，遇到这样的技术诉求，开发者除了希望低延迟的播放外，还想把数据回调上来，然后做视觉算法分析。

单纯地回调数据，不难，需要保证的是，在不影响播放、录像、快照等常规功能的前提下，尽可能高效的数据回调。

技术实现

以大牛直播SDK的SmartPlayer为例，点开始播放之前，初始化参数的时候，我们设置YUV或RGB数据回调：

设置YUV或RGB数据回调：

       btnStartStopPlayback.setOnClickListener(new Button.OnClickListener() {

            // @Override
            public void onClick(View v) {

                if (isPlaying) {
                    Log.i(TAG, "Stop playback stream++");

                    int iRet = libPlayer.SmartPlayerStopPlay(playerHandle);

                    if (iRet != 0) {
                        Log.e(TAG, "Call SmartPlayerStopPlay failed..");
                        return;
                    }

                    btnHardwareDecoder.setEnabled(true);
                    btnLowLatency.setEnabled(true);

                    if (!isRecording) {
                        btnPopInputUrl.setEnabled(true);
                        btnPopInputKey.setEnabled(true);
                        btnSetPlayBuffer.setEnabled(true);
                        btnFastStartup.setEnabled(true);

                        btnRecoderMgr.setEnabled(true);
                        libPlayer.SmartPlayerClose(playerHandle);
                        playerHandle = 0;
                    }

                    isPlaying = false;
                    btnStartStopPlayback.setText("开始播放 ");

                    if (is_enable_hardware_render_mode && sSurfaceView != null) {
                        sSurfaceView.setVisibility(View.GONE);
                        sSurfaceView.setVisibility(View.VISIBLE);
                    }

                    Log.i(TAG, "Stop playback stream--");
                } else {
                    Log.i(TAG, "Start playback stream++");

                    if (!isRecording) {
                        InitAndSetConfig();
                    }

                    // 如果第二个参数设置为null，则播放纯音频
                    libPlayer.SmartPlayerSetSurface(playerHandle, sSurfaceView);

                    libPlayer.SmartPlayerSetRenderScaleMode(playerHandle, 1);

                    //int render_format = 1;
                    //libPlayer.SmartPlayerSetSurfaceRenderFormat(playerHandle, render_format);

                    //int is_enable_anti_alias = 1;
                    //libPlayer.SmartPlayerSetSurfaceAntiAlias(playerHandle, is_enable_anti_alias);

                    if (isHardwareDecoder && is_enable_hardware_render_mode) {
                        libPlayer.SmartPlayerSetHWRenderMode(playerHandle, 1);
                    }

                    // External Render test
                    //libPlayer.SmartPlayerSetExternalRender(playerHandle, new RGBAExternalRender(imageSavePath));
                    libPlayer.SmartPlayerSetExternalRender(playerHandle, new I420ExternalRender(imageSavePath));

                    libPlayer.SmartPlayerSetUserDataCallback(playerHandle, new UserDataCallback());
                    //libPlayer.SmartPlayerSetSEIDataCallback(playerHandle, new SEIDataCallback());

                    libPlayer.SmartPlayerSetAudioOutputType(playerHandle, 1);

                    if (isMute) {
                        libPlayer.SmartPlayerSetMute(playerHandle, isMute ? 1
                                : 0);
                    }

                    if (isHardwareDecoder) {
                        int isSupportHevcHwDecoder = libPlayer.SetSmartPlayerVideoHevcHWDecoder(playerHandle, 1);

                        int isSupportH264HwDecoder = libPlayer
                                .SetSmartPlayerVideoHWDecoder(playerHandle, 1);

                        Log.i(TAG, "isSupportH264HwDecoder: " + isSupportH264HwDecoder + ", isSupportHevcHwDecoder: " + isSupportHevcHwDecoder);
                    }

                    libPlayer.SmartPlayerSetLowLatencyMode(playerHandle, isLowLatency ? 1
                            : 0);

                    libPlayer.SmartPlayerSetFlipVertical(playerHandle, is_flip_vertical ? 1 : 0);

                    libPlayer.SmartPlayerSetFlipHorizontal(playerHandle, is_flip_horizontal ? 1 : 0);

                    libPlayer.SmartPlayerSetRotation(playerHandle, rotate_degrees);

                    libPlayer.SmartPlayerSetAudioVolume(playerHandle, curAudioVolume);

                    int iPlaybackRet = libPlayer
                            .SmartPlayerStartPlay(playerHandle);

                    if (iPlaybackRet != 0) {
                        Log.e(TAG, "Call SmartPlayerStartPlay failed..");
                        return;
                    }

                    btnStartStopPlayback.setText("停止播放 ");

                    btnPopInputUrl.setEnabled(false);
                    btnPopInputKey.setEnabled(false);
                    btnHardwareDecoder.setEnabled(false);
                    btnSetPlayBuffer.setEnabled(false);
                    btnLowLatency.setEnabled(false);
                    btnFastStartup.setEnabled(false);
                    btnRecoderMgr.setEnabled(false);

                    isPlaying = true;
                    Log.i(TAG, "Start playback stream--");
                }
            }
        });

对应的设置如下：

                    // External Render test
libPlayer.SmartPlayerSetExternalRender(playerHandle, new RGBAExternalRender(imageSavePath));
libPlayer.SmartPlayerSetExternalRender(playerHandle, new I420ExternalRender(imageSavePath));

如果是RGBA数据，处理如下：

/*
 * RGBA数据回调处理
 * Author: daniusdk.com
 */   
private static class RGBAExternalRender implements NTExternalRender {
        // public static final int NT_FRAME_FORMAT_RGBA = 1;
        // public static final int NT_FRAME_FORMAT_ABGR = 2;
        // public static final int NT_FRAME_FORMAT_I420 = 3;

        private final String image_path_;
        private long last_save_image_time_ms_;

        private int width_;
        private int height_;
        private int row_bytes_;

        private ByteBuffer rgba_buffer_;

        public RGBAExternalRender(String image_path) {
            this.image_path_ = image_path;
        }

        @Override
        public int getNTFrameFormat() {
            Log.i(TAG, "RGBAExternalRender::getNTFrameFormat return " + NT_FRAME_FORMAT_RGBA);
            return NT_FRAME_FORMAT_RGBA;
        }

        @Override
        public void onNTFrameSizeChanged(int width, int height) {
            width_ = width;
            height_ = height;

            row_bytes_ = width_ * 4;
            rgba_buffer_ = ByteBuffer.allocateDirect(row_bytes_ * height_);

            Log.i(TAG, "RGBAExternalRender::onNTFrameSizeChanged width_:" + width_ + " height_:" + height_);
        }

        @Override
        public ByteBuffer getNTPlaneByteBuffer(int index) {
            if (index == 0)
                return rgba_buffer_;

            Log.e(TAG, "RGBAExternalRender::getNTPlaneByteBuffer index error:" + index);
            return null;
        }

        @Override
        public int getNTPlanePerRowBytes(int index) {
            if (index == 0)
                return row_bytes_;

            Log.e(TAG, "RGBAExternalRender::getNTPlanePerRowBytes index error:" + index);
            return 0;
        }

        public void onNTRenderFrame(int width, int height, long timestamp) {
            if (rgba_buffer_ == null)
                return;

            rgba_buffer_.rewind();

            // copy buffer

            // test
            // byte[] test_buffer = new byte[16];
            // rgba_buffer_.get(test_buffer);

           Log.i(TAG, "RGBAExternalRender:onNTRenderFrame " + width + "*" + height + ", t:" + timestamp);

            // Log.i(TAG, "RGBAExternalRender:onNTRenderFrame rgba:" +
            // bytesToHexString(test_buffer));
        }
    }

如果是I420数据：

/*
 *YUV数据回调处理
 * Author: daniusdk.com
 */   
private static class I420ExternalRender implements NTExternalRender {
        // public static final int NT_FRAME_FORMAT_RGBA = 1;
        // public static final int NT_FRAME_FORMAT_ABGR = 2;
        // public static final int NT_FRAME_FORMAT_I420 = 3;

        private final String image_path_;
        private long last_save_image_time_ms_;

        private int width_;
        private int height_;

        private int y_row_bytes_;
        private int u_row_bytes_;
        private int v_row_bytes_;

        private ByteBuffer y_buffer_;
        private ByteBuffer u_buffer_;
        private ByteBuffer v_buffer_;

        public I420ExternalRender(String image_path) {
            this.image_path_ = image_path;
        }

        @Override
        public int getNTFrameFormat() {
            Log.i(TAG, "I420ExternalRender::getNTFrameFormat return " + NT_FRAME_FORMAT_I420);
            return NT_FRAME_FORMAT_I420;
        }

        @Override
        public void onNTFrameSizeChanged(int width, int height) {
            width_ = width;
            height_ = height;

            y_row_bytes_ = width;
            u_row_bytes_ = (width+1)/2;
            v_row_bytes_ = (width+1)/2;

            y_buffer_ = ByteBuffer.allocateDirect(y_row_bytes_*height_);
            u_buffer_ = ByteBuffer.allocateDirect(u_row_bytes_*((height_ + 1) / 2));
            v_buffer_ = ByteBuffer.allocateDirect(v_row_bytes_*((height_ + 1) / 2));

            Log.i(TAG, "I420ExternalRender::onNTFrameSizeChanged width_="
                    + width_ + " height_=" + height_ + " y_row_bytes_="
                    + y_row_bytes_ + " u_row_bytes_=" + u_row_bytes_
                    + " v_row_bytes_=" + v_row_bytes_);
        }

        @Override
        public ByteBuffer getNTPlaneByteBuffer(int index) {
            switch (index) {
                case 0:
                    return y_buffer_;
                case 1:
                    return u_buffer_;
                case 2:
                    return v_buffer_;
                default:
                    Log.e(TAG, "I420ExternalRender::getNTPlaneByteBuffer index error:" + index);
                    return null;
            }
        }

        @Override
        public int getNTPlanePerRowBytes(int index) {
            switch (index) {
                case 0:
                    return y_row_bytes_;
                case 1:
                    return u_row_bytes_;
                case 2:
                    return v_row_bytes_;
                default:
                    Log.e(TAG, "I420ExternalRender::getNTPlanePerRowBytes index error:" + index);
                    return 0;
            }
        }

        public void onNTRenderFrame(int width, int height, long timestamp) {
            if (null == y_buffer_ || null == u_buffer_ || null == v_buffer_)
                return;

            y_buffer_.rewind();
            u_buffer_.rewind();
            v_buffer_.rewind();

            Log.i(TAG, "I420ExternalRender::onNTRenderFrame " + width + "*" + height + ", t:" + timestamp);

            // copy buffer

            // test
            // byte[] test_buffer = new byte[16];
            // y_buffer_.get(test_buffer);

            // Log.i(TAG, "I420ExternalRender::onNTRenderFrame y data:" + bytesToHexString(test_buffer));

            // u_buffer_.get(test_buffer);
            // Log.i(TAG, "I420ExternalRender::onNTRenderFrame u data:" + bytesToHexString(test_buffer));

            // v_buffer_.get(test_buffer);
            // Log.i(TAG, "I420ExternalRender::onNTRenderFrame v data:" + bytesToHexString(test_buffer));
        }
    }

感兴趣的开发者，可以参考看看，如果需要测试，可以私信探讨。