在iOS 11中使用视觉框架跟踪对象

Question

我想用视觉框架来检测对象并跟踪该对象。我成功地完成了对目标的检测，也成功地完成了一些跟踪，但是我没有得到太多的精确跟踪。

我想要更多的准确性，同时转换帧，因为它经常失去准确性，而跟踪对象。

请检查以下代码以检测和跟踪对象：

import UIKit 
import AVFoundation
import Vision

class ViewController: UIViewController {

private lazy var captureSession: AVCaptureSession = {
   let session = AVCaptureSession()
   session.sessionPreset = AVCaptureSession.Preset.photo
   guard let backCamera = AVCaptureDevice.default(for: .video),
       let input = try? AVCaptureDeviceInput(device: backCamera) else 
           {
            return session
           }
    session.addInput(input)
    return session
 }()

private lazy var cameraLayer: AVCaptureVideoPreviewLayer = 
AVCaptureVideoPreviewLayer(session: self.captureSession)

private let handler = VNSequenceRequestHandler()
fileprivate var lastObservation: VNDetectedObjectObservation?

lazy var highlightView: UIView = {
    let view = UIView()
    view.layer.borderColor = UIColor.red.cgColor
    view.layer.borderWidth = 4
    view.backgroundColor = .clear
    return view
}()

override func viewDidLoad() {
   super.viewDidLoad()

    view.layer.addSublayer(cameraLayer)
   view.addSubview(highlightView)

   let output = AVCaptureVideoDataOutput()
   output.setSampleBufferDelegate(self, queue: DispatchQueue(label: 
"queue"))
   captureSession.addOutput(output)

   captureSession.startRunning()

   let tapGestureRecognizer = UITapGestureRecognizer(target: self, 
action: #selector(tapAction))
    view.addGestureRecognizer(tapGestureRecognizer)
}

override func viewDidLayoutSubviews() {
    super.viewDidLayoutSubviews()
    cameraLayer.frame = view.bounds
}

   // MARK: - Actions

  @objc private func tapAction(recognizer: UITapGestureRecognizer) {
    highlightView.frame.size = CGSize(width: 120, height: 120)
    highlightView.center = recognizer.location(in: view)

    let originalRect = highlightView.frame
    var convertedRect = 
    cameraLayer.metadataOutputRectConverted(fromLayerRect: 
    originalRect)
    convertedRect.origin.y = 1 - convertedRect.origin.y

     lastObservation = VNDetectedObjectObservation(boundingBox: 
     convertedRect)
}

fileprivate func handle(_ request: VNRequest, error: Error?) {
    DispatchQueue.main.async {
        guard let newObservation = request.results?.first as? 
VNDetectedObjectObservation else {
           return
        }
        self.lastObservation = newObservation
        var transformedRect = newObservation.boundingBox
        transformedRect.origin.y = 1 - transformedRect.origin.y
        let convertedRect = 
        self.cameraLayer.layerRectConverted(fromMetadataOutputRect: 
       transformedRect)
        self.highlightView.frame = convertedRect
    }
}
}

 extension ViewController: 
 AVCaptureVideoDataOutputSampleBufferDelegate {

func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: 
   CMSampleBuffer, from connection: AVCaptureConnection) {
        guard let pixelBuffer = 
    CMSampleBufferGetImageBuffer(sampleBuffer),
            let observation = lastObservation else {
                return
        } 
        let request = VNTrackObjectRequest(detectedObjectObservation: 
    observation) { [unowned self] request, error in
            self.handle(request, error: error)
       }
       request.trackingLevel = .accurate
        do {
            try handler.perform([request], on: pixelBuffer)
        }
        catch {
            print(error)
        }
       }
       }

任何帮助都将不胜感激！谢谢。

Answer 1

我不太擅长视觉和核心毫升，但显然你的代码看起来很好。您可以做的一件事是检查vision在缓冲区中没有任何跟踪，如果跟踪请求置信值降至0，则必须标记其属性isLastFrame true。

  if !trackingRequest.isLastFrame {
      if observation.confidence > 0.7 {
          trackingRequest.inputObservation = observation
      } else {
                trackingRequest.isLastFrame = true
            }
          newTrackingRequests.append(trackingRequest)

   }

这样就很容易找出视觉跟踪请求是否丢失跟踪对象，或者它只是跟踪错误的对象。