＊別ブログに既出記事の再掲です。
iPhoneでのオプティカルフローの参考例がネット上で見当たらなかったので、がんばって挑戦してみました。cvCalcOpticalFlowFarnebackを使っています。
下記のページなどを参考にさせていただきました。AVCapture関連のところはほぼそのままです。
・OpenCVを利用したリアルタイムフィルタリングの基本
下記コードです。すべてViewControllerに入れています。
//
// ViewController.h
// optFlow
//
#import "UIKit/UIKit.h"
#import "Foundation/Foundation.h"
#import "opencv2/opencv.hpp"
#import "AVFoundation/AVFoundation.h"
#import "CoreVideo/CoreVideo.h"
@interface ViewController : UIViewController {
}
@end
************************************************
//
// ViewController.m
// optFlow
//
#ifdef __cplusplus
#import "opencv2/opencv.hpp"
#endif
#import "ViewController.h"
@interface ViewController () <AVCaptureVideoDataOutputSampleBufferDelegate>
@property (strong, nonatomic) AVCaptureSession *session;
@property (strong, nonatomic) AVCaptureDeviceInput *videoInput;
@property (strong, nonatomic) AVCaptureVideoDataOutput *videoDataOutput;
@property (nonatomic) dispatch_queue_t videoDataOutputQueue;
@property (strong, nonatomic) CALayer *previewLayer;
@end
UIImage *prevImage; //一コマ前の画像
@implementation ViewController
- (void)viewDidLoad
{
[super viewDidLoad];
// Do any additional setup after loading the view, typically from a nib.
self.previewLayer = [CALayer layer];
self.previewLayer.frame = self.view.bounds;
[self.view.layer addSublayer:self.previewLayer];
}
- (void)viewWillAppear:(BOOL)animated
{
[super viewWillAppear:animated];
[self setupAVCapture];
}
- (void)viewDidDisappear:(BOOL)animated
{
[super viewDidDisappear:animated];
[self teardownAVCapture];
}
- (void)didReceiveMemoryWarning
{
[super didReceiveMemoryWarning];
// Dispose of any resources that can be recreated.
}
- (void)setupAVCapture
{
NSError *error = nil;
// 入力と出力からキャプチャーセッションを作成
self.session = [[AVCaptureSession alloc] init];
self.session.sessionPreset = AVCaptureSessionPresetLow;
// カメラからの入力を作成
AVCaptureDevice *camera = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
// キャプチャーセッションに追加
self.videoInput = [AVCaptureDeviceInput deviceInputWithDevice:camera error:&error];
[self.session addInput:self.videoInput];
// 画像への出力を作成
self.videoDataOutput = [[AVCaptureVideoDataOutput alloc] init];
// キャプチャーセッションに追加
[self.session addOutput:self.videoDataOutput];
// ビデオ出力のキャプチャの画像情報のキューを設定
self.videoDataOutputQueue = dispatch_queue_create("myQueue", NULL);
[self.videoDataOutput setAlwaysDiscardsLateVideoFrames:TRUE];
[self.videoDataOutput setSampleBufferDelegate:self queue:self.videoDataOutputQueue];
// ビデオへの出力の画像は、BGRAで出力
self.videoDataOutput.videoSettings = @{
(id)kCVPixelBufferPixelFormatTypeKey : [NSNumber numberWithInt:kCVPixelFormatType_32BGRA]
};
// ビデオ入力のAVCaptureConnectionを取得
AVCaptureConnection *videoConnection = [self.videoDataOutput connectionWithMediaType:AVMediaTypeVideo];
if ([videoConnection isVideoOrientationSupported]) {
AVCaptureVideoOrientation orientation = AVCaptureVideoOrientationPortrait;[videoConnection setVideoOrientation:orientation];
}
// 1秒あたり8回画像をキャプチャ
if ( YES == [camera lockForConfiguration:NULL] )
{
[camera setActiveVideoMinFrameDuration:CMTimeMake(1,16)];
[camera setActiveVideoMaxFrameDuration:CMTimeMake(1,8)];
}
// 開始
[self.session startRunning];
}
- (void)teardownAVCapture
{
self.videoDataOutput = nil;
if (self.videoDataOutputQueue) {
#if __IPHONE_OS_VERSION_MIN_REQUIRED < 60000
dispatch_release(self.videoDataOutputQueue);
#endif
}
}
// AVCaptureVideoDataOutputSampleBufferDelegateプロトコルのメソッド。
// 新しいキャプチャの情報が追加されたときに呼び出される。
- (void)captureOutput:(AVCaptureOutput *)captureOutput
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
fromConnection:(AVCaptureConnection *)connection
{
// キャプチャしたフレームからCGImageを作成
UIImage *image = [self imageFromSampleBuffer:sampleBuffer];
if(prevImage==nil)prevImage = [[UIImage alloc] initWithCGImage:image.CGImage];
// 画像を画面に表示
dispatch_async(dispatch_get_main_queue(), ^{
[self process:image];
});
}
// 画像処理
- (void)process:(UIImage *)image
{
// オプティカルフロー
UIImage *processedImage = [self optFlow:image];
// 加工した画像をプレビューレイヤーに追加
self.previewLayer.contents = (__bridge id)(processedImage.CGImage);
// imageをprevImageにセット
prevImage = [[UIImage alloc] initWithCGImage:image.CGImage];
}
- (UIImage *)optFlow:(UIImage *)image
{
// CGImageからIplImageを作成
IplImage *srcImage = [self IplImageFromUIImage:image];
IplImage *graySrcImage = cvCreateImage(cvGetSize(srcImage), IPL_DEPTH_8U, 1);
cvCvtColor(srcImage, graySrcImage, CV_BGR2GRAY);
IplImage *prvImage = [self IplImageFromUIImage:prevImage];
IplImage *grayPrvImage = cvCreateImage(cvGetSize(prvImage), IPL_DEPTH_8U, 1);
cvCvtColor(prvImage, grayPrvImage, CV_BGR2GRAY);
CvMat *flow = cvCreateMat(cvGetSize(srcImage).height, cvGetSize(srcImage).width, CV_32FC2);
IplImage *grayScaleImage = cvCreateImage(cvGetSize(srcImage), IPL_DEPTH_8U, 1);
cvCalcOpticalFlowFarneback(grayPrvImage, graySrcImage, flow, 0.5, 3, 15, 3, 5, 1.1, 0);
for ( int i=0;i < cvGetSize(srcImage).height;i+=10 ){
for ( int j=0;j < cvGetSize(srcImage).width;j+=10 ){
cv::Point pt1(j, i);
int jj = flow->data.fl[(i*cvGetSize(srcImage).width+j)*2];
int ii = flow->data.fl[(i*cvGetSize(srcImage).width+j)*2]+1;
cv::Point pt2(j + jj*2, i + ii*2);
cv::Mat matImage(srcImage);
cv::line(matImage,
pt1,
pt2,
cv::Scalar(255,255,0,255), 2, 8, 0);
}
}
UIImage *effectedImage = [self UIImageFromIplImage:srcImage];
cvReleaseImage(&srcImage);
cvReleaseImage(&graySrcImage);
cvReleaseImage(&prvImage);
cvReleaseImage(&grayPrvImage);
cvReleaseImage(&grayScaleImage);
cvReleaseMat(&flow);
return effectedImage;
}
- (IplImage *)IplImageFromUIImage:(UIImage *)image
{
CGImageRef imageRef = image.CGImage;
// RGB色空間を作成
CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
// 一時的なIplImageを作成
IplImage *iplimage = cvCreateImage(cvSize(image.size.width,image.size.height), IPL_DEPTH_8U, 4);
// CGBitmapContextをIplImageのビットマップデータのポインタから作成
CGContextRef contextRef = CGBitmapContextCreate(
iplimage->imageData,
iplimage->width,
iplimage->height,
iplimage->depth,
iplimage->widthStep,
colorSpace,
kCGImageAlphaPremultipliedLast|kCGBitmapByteOrderDefault);
// CGImageをCGBitmapContextに描画
CGContextDrawImage(contextRef,
CGRectMake(0, 0, image.size.width, image.size.height),
imageRef);
// ビットマップコンテキストと色空間を解放
CGContextRelease(contextRef);
CGColorSpaceRelease(colorSpace);
// 最終的なIplImageを作成
IplImage *ret = cvCreateImage(cvGetSize(iplimage), IPL_DEPTH_8U, 3);
// 一時的なIplImageを解放
cvCvtColor(iplimage, ret, CV_RGBA2BGR);
cvReleaseImage(&iplimage);
return ret;
}
- (UIImage *)UIImageFromIplImage:(IplImage*)image
{
CGColorSpaceRef colorSpace;
if (image->nChannels == 1) {
colorSpace = CGColorSpaceCreateDeviceGray();
} else {
colorSpace = CGColorSpaceCreateDeviceRGB();
//BGRになっているのでRGBに変換
cvCvtColor(image, image, CV_BGR2RGB);
}
// IplImageのビットマップデータのポインタアドレスからNSDataを作成
NSData *data = [NSData dataWithBytes:image->imageData length:image->imageSize];
CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)data);
// CGImageを作成
CGImageRef imageRef = CGImageCreate(image->width,
image->height,
image->depth,
image->depth * image->nChannels,
image->widthStep,
colorSpace,
kCGImageAlphaNone|kCGBitmapByteOrderDefault,
provider,
NULL,
false,
kCGRenderingIntentDefault
);
// UIImageを生成
UIImage *ret = [UIImage imageWithCGImage:imageRef];
CGImageRelease(imageRef);
CGDataProviderRelease(provider);
CGColorSpaceRelease(colorSpace);
return ret;
}
- (UIImage *)imageFromSampleBuffer:(CMSampleBufferRef)sampleBuffer
{
CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
// ピクセルバッファのベースアドレスをロックする
CVPixelBufferLockBaseAddress(imageBuffer, 0);
// Get information of the image
uint8_t *baseAddress = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(imageBuffer, 0);
size_t bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer);
size_t width = CVPixelBufferGetWidth(imageBuffer);
size_t height = CVPixelBufferGetHeight(imageBuffer);
// RGBの色空間
CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
CGContextRef newContext = CGBitmapContextCreate(baseAddress,
width,
height,
8,
bytesPerRow,
colorSpace,
kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst);
CGImageRef imageRef = CGBitmapContextCreateImage(newContext);
UIImage *ret = [UIImage imageWithCGImage:imageRef];
CGImageRelease(imageRef);
CGContextRelease(newContext);
CGColorSpaceRelease(colorSpace);
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
return ret;
}
- (AVCaptureVideoOrientation)videoOrientationFromDeviceOrientation:(UIDeviceOrientation)deviceOrientation
{
AVCaptureVideoOrientation orientation;
switch (deviceOrientation) {
case UIDeviceOrientationUnknown:
orientation = AVCaptureVideoOrientationPortrait;
break;
case UIDeviceOrientationPortrait:
orientation = AVCaptureVideoOrientationPortrait;
break;
case UIDeviceOrientationPortraitUpsideDown:
orientation = AVCaptureVideoOrientationPortraitUpsideDown;
break;
case UIDeviceOrientationLandscapeLeft:
orientation = AVCaptureVideoOrientationLandscapeRight;
break;
case UIDeviceOrientationLandscapeRight:
orientation = AVCaptureVideoOrientationLandscapeLeft;
break;
case UIDeviceOrientationFaceUp:
orientation = AVCaptureVideoOrientationPortrait;
break;
case UIDeviceOrientationFaceDown:
orientation = AVCaptureVideoOrientationPortrait;
break;
}
return orientation;
}
@end
