如何在视频播放时用PyQt从眼球跟踪器中提取眼球注视数据

Question

我正在构建一个眼睛凝视可视化工具，就像PyQt5的this一样，我也检查了这个post。以下是修改上述链接的代码。事实证明它是有效的，但视频总会卡住(音频正常，帧卡住，视频内容和椭圆，就像这个video)，有人能帮上忙吗？

import os
import time

from PyQt5 import QtCore, QtGui, QtWidgets, QtMultimedia, QtMultimediaWidgets
import tobii_research as tr
import numpy as np

class Widget(QtWidgets.QWidget):

    def __init__(self, parent=None):

        super(Widget, self).__init__(parent)

        #first window,just have a single button for play the video
        self.resize(256, 256)
        self.btn_play = QtWidgets.QPushButton(self)
        self.btn_play.setGeometry(QtCore.QRect(100, 100, 28, 28))
        self.btn_play.setObjectName("btn_open")
        self.btn_play.setText("Play")
        self.btn_play.clicked.connect(self.Play_video)#click to play video
        #

        self._scene = QtWidgets.QGraphicsScene(self)
        self._gv = QtWidgets.QGraphicsView(self._scene)
        #construct a videoitem for showing the video
        self._videoitem = QtMultimediaWidgets.QGraphicsVideoItem()
        #add it into the scene
        self._scene.addItem(self._videoitem)

        # assign _ellipse_item is the gaze data, and embed it into videoitem,so it can show above the video.
        self._ellipse_item = QtWidgets.QGraphicsEllipseItem(QtCore.QRectF(0, 0, 40, 40), self._videoitem)
        self._ellipse_item.setBrush(QtGui.QBrush(QtCore.Qt.black))
        self._ellipse_item.setPen(QtGui.QPen(QtCore.Qt.red))
        #self._scene.addItem(self._ellipse_item)
        self._gv.fitInView(self._videoitem)

        self._player = QtMultimedia.QMediaPlayer(self, QtMultimedia.QMediaPlayer.VideoSurface)
        self._player.setVideoOutput(self._videoitem)
        file = os.path.join(os.path.dirname(__file__), "video.mp4")#video.mp4 is under the same dirctory
        self._player.setMedia(QtMultimedia.QMediaContent(QtCore.QUrl.fromLocalFile(file)))
        print(f"self._videoitem::{self._videoitem.size()}")

        #get eye tracker
        self.eyetrackers = tr.find_all_eyetrackers()
        self.my_eyetracker = self.eyetrackers[0]

    def gaze_data_callback(self, gaze_data_):

        #for now, I don't know the coordinate system,just randomly assign the gaze data to test the functionality
        self._ellipse_item.setPos(float(np.random.choice(range(0, 300))), float(np.random.choice(range(0, 240))))
        print("time.time()::{}".format(time.time()))


    def Play_video(self):
        self.my_eyetracker.subscribe_to(tr.EYETRACKER_GAZE_DATA, self.gaze_data_callback, as_dictionary=True)
        #size = QtCore.QSizeF(1920.0, 1080.0)#I hope it can fullscreen the video
        #self._videoitem.setSize(size)
        #self._gv.showFullScreen()
        self._gv.resize(720,720)
        self._gv.show()
        self._player.play()


if __name__ == '__main__':
    import sys
    app = QtWidgets.QApplication(sys.argv)
    w = Widget()
    w.show()
    sys.exit(app.exec_())

此处提供cmd打印输出信息

​

​

Answer 1

根据警告消息：

QObject::startTimer: Timers can only be used with threads started with QThread

可以推断，与"my_eyetracker“关联的回调是在辅助线程中执行的，因此从另一个线程到主线程的项的位置将被更新，这可能会产生OP所描述的问题。

解决方案是通过信号将回调信息发送给向导。

import os
import time

from PyQt5 import QtCore, QtGui, QtWidgets, QtMultimedia, QtMultimediaWidgets
import tobii_research as tr
import numpy as np


class EyeTracker(QtCore.QObject):
    positionChanged = QtCore.pyqtSignal(float, float)

    def __init__(self, tracker, parent=None):
        super(EyeTracker, self).__init__(parent)
        self._tracker = tracker

    @property
    def tracker(self):
        return self._tracker

    def start(self):
        self.tracker.subscribe_to(
            tr.EYETRACKER_GAZE_DATA, self._callback, as_dictionary=True
        )

    def _callback(self, gaze_data_):
        self.positionChanged.emit(
            float(np.random.choice(range(0, 300))),
            float(np.random.choice(range(0, 240))),
        )
        print("time.time()::{}".format(time.time()))


class Widget(QtWidgets.QWidget):
    def __init__(self, parent=None):

        super(Widget, self).__init__(parent)

        # first window,just have a single button for play the video
        self.resize(256, 256)
        self.btn_play = QtWidgets.QPushButton(self)
        self.btn_play.setGeometry(QtCore.QRect(100, 100, 28, 28))
        self.btn_play.setObjectName("btn_open")
        self.btn_play.setText("Play")
        self.btn_play.clicked.connect(self.Play_video)  # click to play video
        #

        self._scene = QtWidgets.QGraphicsScene(self)
        self._gv = QtWidgets.QGraphicsView(self._scene)
        # construct a videoitem for showing the video
        self._videoitem = QtMultimediaWidgets.QGraphicsVideoItem()
        # add it into the scene
        self._scene.addItem(self._videoitem)

        # assign _ellipse_item is the gaze data, and embed it into videoitem,so it can show above the video.
        self._ellipse_item = QtWidgets.QGraphicsEllipseItem(
            QtCore.QRectF(0, 0, 40, 40), self._videoitem
        )
        self._ellipse_item.setBrush(QtGui.QBrush(QtCore.Qt.black))
        self._ellipse_item.setPen(QtGui.QPen(QtCore.Qt.red))
        # self._scene.addItem(self._ellipse_item)
        self._gv.fitInView(self._videoitem)

        self._player = QtMultimedia.QMediaPlayer(
            self, QtMultimedia.QMediaPlayer.VideoSurface
        )
        self._player.setVideoOutput(self._videoitem)
        file = os.path.join(
            os.path.dirname(__file__), "video.mp4"
        )  # video.mp4 is under the same dirctory
        self._player.setMedia(
            QtMultimedia.QMediaContent(QtCore.QUrl.fromLocalFile(file))
        )
        print(f"self._videoitem::{self._videoitem.size()}")

        # get eye tracker
        eyetrackers = tr.find_all_eyetrackers()
        self.tracker = EyeTracker(eyetrackers[0])
        self.tracker.positionChanged.connect(self._ellipse_item.setPos)

    def Play_video(self):
        self.tracker.start()
        # size = QtCore.QSizeF(1920.0, 1080.0)#I hope it can fullscreen the video
        # self._videoitem.setSize(size)
        # self._gv.showFullScreen()
        self._gv.resize(720, 720)
        self._gv.show()
        self._player.play()


if __name__ == "__main__":
    import sys

    app = QtWidgets.QApplication(sys.argv)
    w = Widget()
    w.show()
    sys.exit(app.exec_())