使用openvidu共享维基AR

Question

我正在使用openvidu开发一个视频会议应用程序。我们正在尝试在呼叫中包含wikitude会话。问题是它们都需要访问摄像头，所以我有下一个场景:如果我首先实例化本地参与者视频，我不能启动wikitude会话，因为视频不能加载。如果我首先实例化维基会话，通话的其他参与者看不到设备视频。我能够为openvidu创建一个模仿摄像头的自定义视频采集器。它需要将它的每一帧都发送到works。

package org.webrtc;

import android.content.Context;
import android.graphics.Bitmap;

import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.atomic.AtomicReference;

public class CustomVideoCapturer implements VideoCapturer {
    private final static String TAG = "FileVideoCapturer";

    //private final FileVideoCapturer.VideoReader videoReader;
    private final Timer timer = new Timer();
    private CapturerObserver capturerObserver;

    private AtomicReference<Bitmap> image = new AtomicReference<Bitmap>();

    private final TimerTask tickTask = new TimerTask() {
        @Override
        public void run() {
            tick();
        }
    };

    public CustomVideoCapturer() {

    }

    public void tick() {
        Bitmap frame = image.get();
        if (frame != null && !frame.isRecycled()) {
            NV21Buffer nv21Buffer = new NV21Buffer(getNV21(frame),frame.getWidth(),frame.getHeight(), null);
            VideoFrame videoFrame = new VideoFrame(nv21Buffer, 0, System.nanoTime());
            capturerObserver.onFrameCaptured(videoFrame);
        }
    }

    byte [] getNV21(Bitmap image) {
        int [] argb = new int[image.getWidth() * image.getHeight()];

        image.getPixels(argb, 0, image.getWidth(), 0, 0, image.getWidth(), image.getHeight());

        byte [] yuv = new byte[image.getWidth()*image.getHeight()*3/2];
        encodeYUV420SP(yuv, argb, image.getWidth(), image.getHeight());

        image.recycle();

        return yuv;
    }

    void encodeYUV420SP(byte[] yuv420sp, int[] argb, int width, int height) {
        final int frameSize = width * height;

        int yIndex = 0;
        int uvIndex = frameSize;

        int a, R, G, B, Y, U, V;
        int index = 0;
        for (int j = 0; j < height; j++) {
            for (int i = 0; i < width; i++) {

                a = (argb[index] & 0xff000000) >> 24; // a is not used obviously
                R = (argb[index] & 0xff0000) >> 16;
                G = (argb[index] & 0xff00) >> 8;
                B = (argb[index] & 0xff) >> 0;

                // well known RGB to YUV algorithm
                Y = ( (  66 * R + 129 * G +  25 * B + 128) >> 8) +  16;
                U = ( ( -38 * R -  74 * G + 112 * B + 128) >> 8) + 128;
                V = ( ( 112 * R -  94 * G -  18 * B + 128) >> 8) + 128;

                // NV21 has a plane of Y and interleaved planes of VU each sampled by a factor of 2
                //    meaning for every 4 Y pixels there are 1 V and 1 U.  Note the sampling is every other
                //    pixel AND every other scanline.
                yuv420sp[yIndex++] = (byte) ((Y < 0) ? 0 : ((Y > 255) ? 255 : Y));
                if (j % 2 == 0 && index % 2 == 0) {
                    yuv420sp[uvIndex++] = (byte)((V<0) ? 0 : ((V > 255) ? 255 : V));
                    yuv420sp[uvIndex++] = (byte)((U<0) ? 0 : ((U > 255) ? 255 : U));
                }

                index ++;
            }
        }
    }

    public void sendFrame(Bitmap bitmap) {
        image.set(bitmap);
    }

    @Override
    public void initialize(SurfaceTextureHelper surfaceTextureHelper, Context applicationContext,
                           CapturerObserver capturerObserver) {
        this.capturerObserver = capturerObserver;
    }

    @Override
    public void startCapture(int width, int height, int framerate) {
        //timer.schedule(tickTask, 0, 1000 / framerate);
        threadCV().start();
    }

    Thread threadCV() {
        return new Thread() {
            @Override
            public void run() {
                while (true) {
                    if (image.get() != null) {
                        tick();
                    }

                    try {
                        Thread.sleep(10);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            }
        };
    }

    @Override
    public void stopCapture() throws InterruptedException {
        timer.cancel();
    }

    @Override
    public void changeCaptureFormat(int width, int height, int framerate) {
        // Empty on purpose
    }

    @Override
    public void dispose() {
        //videoReader.close();
    }

    @Override
    public boolean isScreencast() {
        return false;
    }

    private interface VideoReader {
        VideoFrame getNextFrame();

        void close();
    }

    /**
     * Read video data from file for the .y4m container.
     */

}

在本地参与者i上，然后使用此函数发送帧：

    public void sendFrame(Bitmap frame) {
        customVideoCapturer.sendFrame(frame);
    }

但我无法读取维基相机上的图像。有没有办法访问这些帧并重新发送它们？

Answer 1

例如Native Api sdk，version 9.10.0，根据来自维基支持https://support.wikitude.com/support/discussions/topics/5000096719?page=1的回答，要访问相机帧，应该创建一个自定义插件：https://www.wikitude.com/external/doc/documentation/latest/androidnative/pluginsapi.html#plugins-api