Tensorflow与OpenCV实时分类

Question

我正在测试机器学习水，并使用TS初始模型重新训练网络来分类我想要的对象。

最初，我的预测是在本地存储的图像上运行的，我意识到从文件中删除图形需要花费2-5秒的时间，同时运行实际的预测。

此后，我对代码进行了调整，以合并来自OpenCV的相机馈送，但在上述记录的时间里，视频延迟是不可避免的。

在最初的图形加载过程中，预计会有一个时间命中；这就是为什么initialSetup()会提前运行，但是2-5秒是荒谬的。我觉得我现在的应用程序；实时分类，这不是最好的方式加载它。还有别的办法吗？我知道，对于移动版本，TS建议减少图表。减肥法是来这里的路吗？如果重要的话，我的图表目前是87.4MB

除此之外，有没有办法加快预测过程呢？

import os
import cv2
import timeit
import numpy as np
import tensorflow as tf

camera = cv2.VideoCapture(0)

# Loads label file, strips off carriage return
label_lines = [line.rstrip() for line
               in tf.gfile.GFile('retrained_labels.txt')]

def grabVideoFeed():
    grabbed, frame = camera.read()
    return frame if grabbed else None

def initialSetup():
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
    start_time = timeit.default_timer()

    # This takes 2-5 seconds to run
    # Unpersists graph from file
    with tf.gfile.FastGFile('retrained_graph.pb', 'rb') as f:
        graph_def = tf.GraphDef()
        graph_def.ParseFromString(f.read())
        tf.import_graph_def(graph_def, name='')

    print 'Took {} seconds to unpersist the graph'.format(timeit.default_timer() - start_time)

def classify(image_data):
    print '********* Session Start *********'

    with tf.Session() as sess:
        start_time = timeit.default_timer()

        # Feed the image_data as input to the graph and get first prediction
        softmax_tensor = sess.graph.get_tensor_by_name('final_result:0')

        print 'Tensor', softmax_tensor

        print 'Took {} seconds to feed data to graph'.format(timeit.default_timer() - start_time)

        start_time = timeit.default_timer()

        # This takes 2-5 seconds as well
        predictions = sess.run(softmax_tensor, {'Mul:0': image_data})

        print 'Took {} seconds to perform prediction'.format(timeit.default_timer() - start_time)

        start_time = timeit.default_timer()

        # Sort to show labels of first prediction in order of confidence
        top_k = predictions[0].argsort()[-len(predictions[0]):][::-1]

        print 'Took {} seconds to sort the predictions'.format(timeit.default_timer() - start_time)

        for node_id in top_k:
            human_string = label_lines[node_id]
            score = predictions[0][node_id]
            print('%s (score = %.5f)' % (human_string, score))

        print '********* Session Ended *********'

initialSetup()

while True:
    frame = grabVideoFeed()

    if frame is None:
        raise SystemError('Issue grabbing the frame')

    frame = cv2.resize(frame, (299, 299), interpolation=cv2.INTER_CUBIC)

    # adhere to TS graph input structure
    numpy_frame = np.asarray(frame)
    numpy_frame = cv2.normalize(numpy_frame.astype('float'), None, -0.5, .5, cv2.NORM_MINMAX)
    numpy_final = np.expand_dims(numpy_frame, axis=0)

    classify(numpy_final)

    cv2.imshow('Main', frame)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

camera.release()
cv2.destroyAllWindows()

编辑1

调试代码之后，我意识到创建会话既是一项资源又是一项耗时的操作。

在前面的代码中，在运行预测的基础上为每个OpenCV提要创建了一个新会话。将OpenCV操作封装在单个会话中提供了大量的时间改进，但这仍然会增加初始运行的大量开销；预测需要2-3秒。然后，预测时间约为0.5s，使得摄像机的馈电仍然滞后。

import os
import cv2
import timeit
import numpy as np
import tensorflow as tf

camera = cv2.VideoCapture(0)

# Loads label file, strips off carriage return
label_lines = [line.rstrip() for line
               in tf.gfile.GFile('retrained_labels.txt')]

def grabVideoFeed():
    grabbed, frame = camera.read()
    return frame if grabbed else None

def initialSetup():
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
    start_time = timeit.default_timer()

    # This takes 2-5 seconds to run
    # Unpersists graph from file
    with tf.gfile.FastGFile('retrained_graph.pb', 'rb') as f:
        graph_def = tf.GraphDef()
        graph_def.ParseFromString(f.read())
        tf.import_graph_def(graph_def, name='')

    print 'Took {} seconds to unpersist the graph'.format(timeit.default_timer() - start_time)

initialSetup()

with tf.Session() as sess:
    start_time = timeit.default_timer()

    # Feed the image_data as input to the graph and get first prediction
    softmax_tensor = sess.graph.get_tensor_by_name('final_result:0')

    print 'Took {} seconds to feed data to graph'.format(timeit.default_timer() - start_time)

    while True:
        frame = grabVideoFeed()

        if frame is None:
            raise SystemError('Issue grabbing the frame')

        frame = cv2.resize(frame, (299, 299), interpolation=cv2.INTER_CUBIC)

        cv2.imshow('Main', frame)

        # adhere to TS graph input structure
        numpy_frame = np.asarray(frame)
        numpy_frame = cv2.normalize(numpy_frame.astype('float'), None, -0.5, .5, cv2.NORM_MINMAX)
        numpy_final = np.expand_dims(numpy_frame, axis=0)

        start_time = timeit.default_timer()

        # This takes 2-5 seconds as well
        predictions = sess.run(softmax_tensor, {'Mul:0': numpy_final})

        print 'Took {} seconds to perform prediction'.format(timeit.default_timer() - start_time)

        start_time = timeit.default_timer()

        # Sort to show labels of first prediction in order of confidence
        top_k = predictions[0].argsort()[-len(predictions[0]):][::-1]

        print 'Took {} seconds to sort the predictions'.format(timeit.default_timer() - start_time)

        for node_id in top_k:
            human_string = label_lines[node_id]
            score = predictions[0][node_id]
            print('%s (score = %.5f)' % (human_string, score))

        print '********* Session Ended *********'

        if cv2.waitKey(1) & 0xFF == ord('q'):
            sess.close()
            break

camera.release()
cv2.destroyAllWindows()

编辑2

在兜圈子之后，我无意中遇到了图量化和图变换，这些都是达到的结果。

原始图: 87.4MB

量化图: 87.5MB

变换图: 87.1MB

8位计算:22 use，但在使用时遇到这。

Answer 1

我有我自己的项目，工作类似(但要简单得多的模型)，它需要我大约0.1秒来运行我的预测实时。你通过重复使用会话做了正确的事情--我也是这样做的。我猜你的瓶颈是模型的大小。据我所知，盗梦空间模型是巨大的。在模型复杂性和运行时预测速度之间总是会有一个权衡。盗梦空间模型是准确的，但从来没有人说它是快速的。