openCV doc

オプティカルフロー(calcOpticalFlowPyrLK)

Gallery

src

dst

what process?

疎なオプティカルフローを使って前後の差分から物体の移動量を計算しています。ピラミッド型Lucas-Kanade法のアルゴリズムを使っています。 ※疎なオプティカルフローとは、事前に算出した画像内の追跡する点をもとに、前後のフレーム間での移動量を計測する手法です。

cook code C++

#include <iostream>
#include <opencv2/core.hpp>
#include <opencv2/core/utility.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/video.hpp>

int main(int argc, const char* argv[])
{
    // ./out -src=[image path]
    cv::String keys = "{src||}";
    cv::CommandLineParser parser(argc, argv, keys);
    cv::String src_path = parser.get<cv::String>("src");

cv::VideoCapture capture(src_path);
    if (!capture.isOpened()){
        //error in opening the video input
        std::cerr << "Unable to open file!" << std::endl;
        return 0;
    }

/* 初期設定 */
    int cap_width = int(capture.get(cv::CAP_PROP_FRAME_WIDTH));
    int cap_height = int(capture.get(cv::CAP_PROP_FRAME_HEIGHT));
    int fps = int(capture.get(cv::CAP_PROP_FPS));
    int fourcc = cv::VideoWriter::fourcc('m','p','4','v');
    cv::Size video_size = cv::Size(cap_width,cap_height);

cv::VideoWriter writer("output.mp4", fourcc, fps, video_size);

cv::Mat old_frame, old_gray;
    std::vector<cv::Point2f> p0, p1;

capture >> old_frame;
    cv::cvtColor(old_frame, old_gray, cv::COLOR_BGR2GRAY);
    cv::goodFeaturesToTrack(old_gray, p0, 100, 0.3, 7, cv::Mat(), 7, true);

// Create a mask image for drawing purposes
    cv::Mat mask = cv::Mat::zeros(old_frame.size(), old_frame.type());

cv::Scalar color = cv::Scalar(255,51,204);

while(true){
        cv::Mat frame, frame_gray;
        capture >> frame;
        if (frame.empty())
            break;
        cv::cvtColor(frame, frame_gray, cv::COLOR_BGR2GRAY);

// calculate optical flow
        std::vector<uchar> status;
        std::vector<float> err;
        cv::TermCriteria criteria = cv::TermCriteria(
            (cv::TermCriteria::COUNT) + (cv::TermCriteria::EPS), 
            10, 
            0.03);
        cv::calcOpticalFlowPyrLK(old_gray, 
                                frame_gray, 
                                p0, 
                                p1, 
                                status, 
                                err, 
                                cv::Size(15,15), 
                                2, 
                                criteria,
                                cv::OPTFLOW_LK_GET_MIN_EIGENVALS);

std::vector<cv::Point2f> good_new;
        for(uint i = 0; i < p0.size(); i++)
        {
            // Select good points
            if(status[i] == 1) {
                good_new.push_back(p1[i]);
                // draw the tracks
                cv::line(mask,p1[i], p0[i], color, 2);
                cv::circle(frame, p1[i], 5, color, -1);
            }
        }
        cv::Mat img;
        cv::add(frame, mask, img);
        
        writer << img;
        old_gray = frame_gray.clone();
        p0 = good_new;
    }

capture.release();
    writer.release();

return 0;
}

cook code Python

import numpy as np
import cv2
import argparse

feature_params = dict(
    maxCorners=100,             # 保持するコーナー数, int
    qualityLevel=0.3,           # 最良値, double
    minDistance=7,              # この距離内のコーナーを棄却, double
    blockSize=7,                # 使用する近傍領域のサイズ, int
    useHarrisDetector=True,    # FalseならShi-Tomashi法
)

lk_params = dict(
    winSize=(15, 15),           # 検索ウィンドウのサイズ
    maxLevel=2,                 # 追加するピラミッド層数

# 検索を終了する条件
    criteria=(
        cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT,
        10,
        0.03
    ),

# 推測値や固有値の使用
    flags=cv2.OPTFLOW_LK_GET_MIN_EIGENVALS,
)

def main():

parser = argparse.ArgumentParser()
    parser.add_argument('--src')
    args = parser.parse_args()

cap = cv2.VideoCapture(args.src)

cap_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
    cap_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
    fps = cap.get(cv2.CAP_PROP_FPS)

fmt = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
    writer = cv2.VideoWriter('output_LK.mp4', fmt, fps, (cap_width,cap_height))

_ret, prev = cap.read()
    prevgray = cv2.cvtColor(prev, cv2.COLOR_BGR2GRAY)
    p0 = cv2.goodFeaturesToTrack(prevgray, mask = None, **feature_params)

mask = np.zeros_like(prev)
    color = (255,51,204)

while cap.isOpened():
        _ret, img = cap.read()
        
        if img is None:
        	break

if cv2.waitKey(30) & 0xFF == ord('q'):
        	break

gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        p1, st, err = cv2.calcOpticalFlowPyrLK(prevgray, gray, p0, None, **lk_params)

good_new = p1[st==1]
        good_old = p0[st==1]

for i,(new,old) in enumerate(zip(good_new,good_old)):
            a,b = new.ravel()
            c,d = old.ravel()
            mask = cv2.line(mask, (a,b),(c,d), color, 2)
            frame = cv2.circle(img,(a,b),5, color,-1)
        frame = cv2.add(img, mask)

writer.write(frame)
        prevgray = gray.copy()
        p0 = good_new.reshape(-1, 1, 2)

cap.release()
    writer.release()

if __name__ == '__main__':
    main()

Reference

https://github.com/opencv/opencv/blob/4.4.0/samples/cpp/tutorial_code/video/optical_flow/optical_flow.cpp
https://kuri-megane.hatenablog.jp/entry/2018/06/28/100000
http://labs.eecs.tottori-u.ac.jp/sd/Member/oyamada/OpenCV/html/py_tutorials/py_video/py_lucas_kanade/py_lucas_kanade.html
GIFMAGAZINE - タイガーステップ
Book : Book : 詳細 OpenCV3 16章キーポイントと記述子 pp480-492