16 #include "PlusConfigure.h" 24 #include <vtkObjectFactory.h> 27 #include <MediaFoundationVideoCaptureApi.h> 28 #include <FormatReader.h> 35 #include <Mfreadwrite.h> 48 const unsigned int DEFAULT_DEVICE_ID = 0;
49 const FrameSizeType DEFAULT_FRAME_SIZE = { 640, 480, 1 };
50 const double DEFAULT_ACQUISITION_RATE = 30;
51 const std::wstring DEFAULT_PIXEL_TYPE_NAME = L
"YUY2";
52 const GUID DEFAULT_PIXEL_TYPE = MFVideoFormat_YUY2;
53 const std::wstring MF_VIDEO_FORMAT_PREFIX = L
"MFVideoFormat_";
54 const int ERRMSGBUFFERSIZE = 1024;
55 template <
class T>
void SafeRelease(T** ppT)
66 class MmfVideoSourceReader :
public IMFSourceReaderCallback
72 , CaptureSourceReader(NULL)
74 , PlusDevice(plusDevice)
79 STDMETHODIMP QueryInterface(REFIID iid,
void** ppv);
80 STDMETHOD_(
ULONG, AddRef)();
81 STDMETHOD_(
ULONG, Release)();
82 STDMETHODIMP OnEvent(
DWORD, IMFMediaEvent*);
83 STDMETHODIMP OnFlush(
DWORD);
84 STDMETHODIMP OnReadSample(HRESULT hrStatus,
DWORD dwStreamIndex,
DWORD dwStreamFlags, LONGLONG llTimestamp, IMFSample* pSample);
86 IMFMediaSource* CaptureSource;
87 IMFSourceReader* CaptureSourceReader;
94 STDMETHODIMP MmfVideoSourceReader::QueryInterface(REFIID iid,
void** ppv)
96 static const QITAB qit[] =
98 QITABENT(MmfVideoSourceReader, IMFSourceReaderCallback),
101 return QISearch(
this, qit, iid, ppv);
107 LONG uCount = InterlockedIncrement(&RefCount);
110 this->PlusDevice->Register(NULL);
118 ULONG uCount = InterlockedDecrement(&RefCount);
121 LOG_DEBUG(
"vtkPlusMmfVideoSource::Release - unregister");
122 this->PlusDevice->UnRegister(NULL);
128 STDMETHODIMP MmfVideoSourceReader::OnEvent(
DWORD, IMFMediaEvent*)
134 STDMETHODIMP MmfVideoSourceReader::OnFlush(
DWORD)
140 STDMETHODIMP MmfVideoSourceReader::OnReadSample(HRESULT hrStatus,
DWORD dwStreamIndex,
DWORD dwStreamFlags, LONGLONG llTimestamp, IMFSample* pSample)
142 igsioLockGuard<vtkIGSIORecursiveCriticalSection> updateMutexGuardedLock(this->PlusDevice->Mutex);
144 if (!SUCCEEDED(hrStatus))
147 LOG_ERROR(
"Source Reader error: " << std::hex << hrStatus);
151 if (!this->PlusDevice->IsRecording())
156 if (this->CaptureSourceReader == NULL)
165 IMFMediaType* pType = NULL;
166 this->CaptureSourceReader->GetCurrentMediaType(this->PlusDevice->ActiveVideoFormat.StreamIndex, &pType);
169 LOG_ERROR(
"Cannot get current media type");
175 UINT32 actualWidth = 0;
176 UINT32 actualHeight = 0;
177 ::MFGetAttributeSize(pType, MF_MT_FRAME_SIZE, &actualWidth, &actualHeight);
178 if (actualWidth != this->PlusDevice->ActiveVideoFormat.FrameSize[0] ||
179 actualHeight != this->PlusDevice->ActiveVideoFormat.FrameSize[1])
181 LOG_ERROR(
"Unexpected frame size: " << actualWidth <<
"x" << actualHeight <<
" (expected: " << this->PlusDevice->ActiveVideoFormat.FrameSize[0] <<
"x" << this->PlusDevice->ActiveVideoFormat.FrameSize[1] <<
")");
188 GUID videoFormat = DEFAULT_PIXEL_TYPE;
189 pType->GetGUID(MF_MT_SUBTYPE, &videoFormat);
190 std::wstring videoFormatWStr = MfVideoCapture::FormatReader::StringFromGUID(videoFormat);
191 if (igsioCommon::HasSubstrInsensitive(videoFormatWStr, MF_VIDEO_FORMAT_PREFIX))
194 videoFormatWStr.erase(0, MF_VIDEO_FORMAT_PREFIX.size());
197 if (!igsioCommon::IsEqualInsensitive(videoFormatWStr, this->PlusDevice->ActiveVideoFormat.PixelFormatName))
199 LOG_ERROR_W(
"Unexpected video format: " << videoFormatWStr <<
" (expected: " << this->PlusDevice->ActiveVideoFormat.PixelFormatName <<
")");
203 IMFMediaBuffer* aBuffer = NULL;
204 DWORD bufferCount = 0;
205 pSample->GetBufferCount(&bufferCount);
208 LOG_ERROR(
"No buffer available in the sample.");
211 pSample->GetBufferByIndex(0, &aBuffer);
212 BYTE* bufferData = NULL;
214 DWORD currentLength = 0;
216 HRESULT hr = aBuffer->Lock(&bufferData, &maxLength, ¤tLength);
219 this->PlusDevice->AddFrame(bufferData, currentLength);
224 LOG_ERROR(
"Unable to lock the buffer.");
226 SafeRelease(&aBuffer);
229 if (MF_SOURCE_READERF_ENDOFSTREAM & dwStreamFlags)
232 LOG_ERROR(
"End of stream reached. Capture device should never reach end of stream.");
233 this->PlusDevice->Disconnect();
238 this->CaptureSourceReader->ReadSample(this->PlusDevice->ActiveVideoFormat.StreamIndex, 0, NULL, NULL, NULL, NULL);
250 , Mutex(vtkSmartPointer<vtkIGSIORecursiveCriticalSection>::New())
283 os << indent <<
"FrameIndex: " << (this->
FrameIndex ?
"On\n" :
"Off\n");
291 igsioLockGuard<vtkIGSIORecursiveCriticalSection> updateMutexGuardedLock(this->
Mutex);
295 LOG_WARNING(
"There is a reference to this class from a previous connection");
300 GUID pixelFormat = DEFAULT_PIXEL_TYPE;
304 pixelFormat = MfVideoCapture::FormatReader::GUIDFromString(videoFormat);
305 if (pixelFormat == GUID_NULL)
307 LOG_ERROR_W(
"Cannot recognize requested pixel format: " << this->
RequestedVideoFormat.PixelFormatName <<
". Defaulting to \'" << DEFAULT_PIXEL_TYPE_NAME <<
"\'.");
308 pixelFormat = DEFAULT_PIXEL_TYPE;
312 if (!MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().SetupDevice(this->
RequestedVideoFormat.DeviceId, this->RequestedVideoFormat.StreamIndex,
313 this->RequestedVideoFormat.FrameSize[0], this->RequestedVideoFormat.FrameSize[1], this->AcquisitionRate, pixelFormat))
315 LOG_WARNING_W(
"Unable to init capture device with requested details:" 317 <<
", " << this->RequestedVideoFormat.FrameSize[0] <<
"x" << this->RequestedVideoFormat.FrameSize[1]
318 <<
", " << this->AcquisitionRate <<
"Hz, " << this->ActiveVideoFormat.PixelFormatName);
322 if (!MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().SetupDevice(DEFAULT_DEVICE_ID, 0, DEFAULT_FRAME_SIZE[0], DEFAULT_FRAME_SIZE[1], DEFAULT_ACQUISITION_RATE, DEFAULT_PIXEL_TYPE))
324 LOG_ERROR_W(
"Unable to initialize capture device with default details: device ID: " << DEFAULT_DEVICE_ID <<
" (" <<
GetCaptureDeviceName(DEFAULT_DEVICE_ID) <<
") stream 0, " << DEFAULT_FRAME_SIZE[0] <<
"x" << DEFAULT_FRAME_SIZE[1] <<
", " << DEFAULT_ACQUISITION_RATE <<
"Hz, " << DEFAULT_PIXEL_TYPE_NAME);
335 LOG_INFO_W(
"Backing up to connecting with default capture settings:" 337 <<
", " << this->
ActiveVideoFormat.FrameSize[0] <<
"x" << this->ActiveVideoFormat.FrameSize[1]
338 <<
", " << DEFAULT_ACQUISITION_RATE <<
"Hz, " << this->ActiveVideoFormat.PixelFormatName);
344 LOG_ERROR(
"Unable to request capture source from the media foundation library.");
348 unsigned int frameRate = MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().GetFrameRate(this->
ActiveVideoFormat.DeviceId);
349 LOG_DEBUG_W(
"vtkPlusMmfVideoSource connected to device '" <<
GetActiveDeviceName() <<
"' at frame rate of " << frameRate <<
"Hz");
353 LOG_ERROR(
"Failed to change imaging parameters in the device");
365 igsioLockGuard<vtkIGSIORecursiveCriticalSection> updateMutexGuardedLock(this->
Mutex);
367 MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().CloseDevice(this->
ActiveVideoFormat.DeviceId);
376 igsioLockGuard<vtkIGSIORecursiveCriticalSection> updateMutexGuardedLock(this->
Mutex);
382 MFCreateAttributes(&attr, 2);
383 hr = attr->SetUnknown(MF_SOURCE_READER_ASYNC_CALLBACK, this->
MmfSourceReader);
384 hr = attr->SetUINT32(MF_READWRITE_DISABLE_CONVERTERS,
TRUE);
386 hr = MFCreateSourceReaderFromMediaSource(this->
MmfSourceReader->CaptureSource, attr, &this->MmfSourceReader->CaptureSourceReader);
390 LOG_ERROR(
"Unable to create source reader from media source.");
399 IMFMediaType* pDecodeType = NULL;
400 hr = MFCreateMediaType(&pDecodeType);
401 hr = pDecodeType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
402 hr = pDecodeType->SetGUID(MF_MT_SUBTYPE, MfVideoCapture::FormatReader::GUIDFromString(videoFormat));
407 LOG_WARNING_W(
"Unable to set SourceReader output to requested format: " << this->
RequestedVideoFormat.PixelFormatName
408 <<
". Using device default.");
410 SafeRelease(&pDecodeType);
412 MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().StartRecording(this->
ActiveVideoFormat.DeviceId);
417 LOG_ERROR(
"Unable to request IMFMediaSource from the media foundation capture library. Unable to continue.");
427 LOG_DEBUG(
"vtkPlusMmfVideoSource::InternalStopRecording");
429 igsioLockGuard<vtkIGSIORecursiveCriticalSection> updateMutexGuardedLock(this->
Mutex);
431 MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().StopRecording(this->
ActiveVideoFormat.DeviceId);
443 LOG_ERROR(
"No output channels defined for microsoft media foundation video source. Cannot proceed.");
457 for (
int i = 0;
i < numberOfVideoSources; ++
i)
463 if (currentFrameSize[0] != this->
ActiveVideoFormat.FrameSize[0] || currentFrameSize[1] != this->ActiveVideoFormat.FrameSize[1] || currentFrameSize[2] != 1)
470 unsigned int numberOfScalarComponents = (videoSource->
GetImageType() == US_IMG_RGB_COLOR ? 3 : 1);
474 this->
UncompressedVideoFrame.AllocateFrame(currentFrameSize, VTK_UNSIGNED_CHAR, numberOfScalarComponents);
488 if (deviceConfig->GetScalarAttribute(
"CaptureDeviceId", deviceId))
494 if (deviceConfig->GetScalarAttribute(
"CaptureStreamIndex", streamIndex))
499 int requestedFrameSize[2] = { static_cast<int>(DEFAULT_FRAME_SIZE[0]), static_cast<int>(DEFAULT_FRAME_SIZE[1]) };
500 if (deviceConfig->GetVectorAttribute(
"FrameSize", 2, requestedFrameSize))
502 if (requestedFrameSize[0] < 0 || requestedFrameSize[1] < 0)
504 LOG_ERROR(
"Negative frame size defined in config file. Cannot continue.");
513 if (deviceConfig->GetAttribute(
"VideoFormat") != NULL)
515 auto attr = std::string(deviceConfig->GetAttribute(
"VideoFormat"));
520 if (cameraParameters != NULL)
540 XML_REMOVE_ATTRIBUTE(
"CaptureStreamIndex", deviceConfig);
543 deviceConfig->SetVectorAttribute(
"FrameSize", 2, frameSize);
545 deviceConfig->SetAttribute(
"VideoFormat", attr.c_str());
579 MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().GetDeviceNames(deviceNames);
586 unsigned int numberOfStreams = MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().GetNumberOfStreams(deviceId);
587 for (
unsigned int streamIndex = 0; streamIndex < numberOfStreams; ++streamIndex)
589 unsigned int numberOfVideoFormats = MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().GetNumberOfFormats(deviceId, streamIndex);
590 for (
unsigned int formatIndex = 0; formatIndex < numberOfVideoFormats; formatIndex++)
592 MfVideoCapture::MediaType type = MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().GetFormat(deviceId, streamIndex, formatIndex);
593 std::wstring pixelType(type.MF_MT_SUBTYPEName.begin(), type.MF_MT_SUBTYPEName.end());
594 if (igsioCommon::HasSubstrInsensitive(pixelType, MF_VIDEO_FORMAT_PREFIX))
597 pixelType.erase(0, MF_VIDEO_FORMAT_PREFIX.size());
599 std::wostringstream strFriendlyName;
600 strFriendlyName <<
"Stream index " << streamIndex;
601 strFriendlyName <<
" - Frame size: " << type.width <<
"x" << type.height;
602 strFriendlyName <<
", video format: " << pixelType;
603 strFriendlyName <<
", frame rate: " << type.MF_MT_FRAME_RATE;
604 videoModes.push_back(strFriendlyName.str());
626 LOG_ERROR(
"vtkPlusMmfVideoSource::AddFrame skipped, not recording anymore");
632 for (
int i = 0;
i < numberOfVideoSources; ++
i)
645 if (igsioCommon::IsEqualInsensitive(this->
ActiveVideoFormat.PixelFormatName, L
"YUY2"))
647 if (bufferSize < frameSize[0] * frameSize[1] * 2)
649 LOG_ERROR(
"Failed to decode pixel data from YUY2 due to buffer size mismatch");
654 else if (igsioCommon::IsEqualInsensitive(this->
ActiveVideoFormat.PixelFormatName, L
"MJPG"))
658 else if (igsioCommon::IsEqualInsensitive(this->
ActiveVideoFormat.PixelFormatName, L
"RGB24"))
660 if (bufferSize < frameSize[0] * frameSize[1] * 3)
662 LOG_ERROR(
"Failed to decode pixel data from RGB24 due to buffer size mismatch");
669 LOG_ERROR_W(
"Unknown pixel type: " << this->
ActiveVideoFormat.PixelFormatName <<
" (only YUY2, MJPG and RGB24 are supported)");
684 LOG_ERROR(
"Error while decoding the grabbed image");
688 const double maximumFrameTimeVariance = 0.2;
690 double minimumTimeBetweenBetweenRecordedFramesSec = (1.0 - maximumFrameTimeVariance) / this->
GetAcquisitionRate();
691 double lastFrameTimeSec = -1.0;
692 double currentTime = vtkIGSIOAccurateTimer::GetSystemTime();
698 double secondsSinceLastFrame = currentTime - lastFrameTimeSec;
699 if (lastFrameTimeSec > 0 && secondsSinceLastFrame < minimumTimeBetweenBetweenRecordedFramesSec)
711 status = sourceStatus !=
PLUS_SUCCESS ? sourceStatus : status;
721 LOG_INFO_W(
"Supported video formats for Device Id " << deviceId <<
" (" <<
GetCaptureDeviceName(deviceId) <<
")");
722 std::vector<std::wstring> videoModes;
724 for (
auto modeIt = videoModes.begin(); modeIt != videoModes.end(); ++modeIt)
726 LOG_INFO_W(
" " << (*modeIt));
733 LOG_INFO(
"Found capture devices:");
734 std::vector<std::wstring> deviceNames;
737 for (
auto deviceNameIt = deviceNames.begin(); deviceNameIt != deviceNames.end(); ++deviceNameIt,
id++)
739 LOG_INFO_W(
" " <<
id <<
": " << (*deviceNameIt));
746 return MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().GetCaptureDeviceName(deviceId);
754 LOG_ERROR(
"Cannot apply camera control parameter change, capture source is not initialized");
758 MfVideoCapture::CaptureDeviceParameters parameters =
759 MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().GetParameters(this->
ActiveVideoFormat.DeviceId);
766 double panDegrees = 0.0;
769 LOG_ERROR(
"Failed to get pan camera control parameter");
772 parameters.CameraControlParameters[MfVideoCapture::CaptureDeviceParameters::Pan].CurrentValue = panDegrees;
781 double tiltDegrees = 0.0;
784 LOG_ERROR(
"Failed to get tilt camera control parameter");
787 parameters.CameraControlParameters[MfVideoCapture::CaptureDeviceParameters::Tilt].CurrentValue = tiltDegrees;
796 double rollDegrees = 0.0;
799 LOG_ERROR(
"Failed to get roll camera control parameter");
802 parameters.CameraControlParameters[MfVideoCapture::CaptureDeviceParameters::Roll].CurrentValue = rollDegrees;
814 LOG_ERROR(
"Failed to get zoom camera control parameter");
817 parameters.CameraControlParameters[MfVideoCapture::CaptureDeviceParameters::Zoom].CurrentValue = zoomMm;
829 LOG_ERROR(
"Failed to get iris camera control parameter");
832 parameters.CameraControlParameters[MfVideoCapture::CaptureDeviceParameters::Iris].CurrentValue = irisFStop;
842 int exposureLog2Sec = 0;
845 LOG_ERROR(
"Failed to get exposure camera control parameter");
848 parameters.CameraControlParameters[MfVideoCapture::CaptureDeviceParameters::Exposure].CurrentValue = exposureLog2Sec;
857 bool autoExposure =
false;
860 LOG_ERROR(
"Failed to get auto exposure camera control parameter");
863 parameters.CameraControlParameters[MfVideoCapture::CaptureDeviceParameters::Exposure].Flag = autoExposure ? CameraControl_Flags_Auto : CameraControl_Flags_Manual;
872 double focusMm = 0.0;
875 LOG_ERROR(
"Failed to get focus camera control parameter");
878 parameters.CameraControlParameters[MfVideoCapture::CaptureDeviceParameters::Focus].CurrentValue = focusMm;
887 bool autoFocus =
false;
890 LOG_ERROR(
"Failed to get auto focus camera control parameter");
893 parameters.CameraControlParameters[MfVideoCapture::CaptureDeviceParameters::Focus].Flag = autoFocus ? CameraControl_Flags_Auto : CameraControl_Flags_Manual;
899 MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().SetParameters(this->
ActiveVideoFormat.DeviceId, parameters);
virtual void PrintSelf(ostream &os, vtkIndent indent) VTK_OVERRIDE
virtual PlusStatus ReadConfiguration(vtkXMLDataElement *xmlElement)
US_IMAGE_TYPE GetImageType()
virtual PlusStatus InternalConnect()
vtkSmartPointer< vtkIGSIORecursiveCriticalSection > Mutex
virtual int GetNumberOfVideoSources() const
virtual void SetRequestedDeviceId(unsigned int deviceId)
#define XML_FIND_DEVICE_ELEMENT_REQUIRED_FOR_WRITING(deviceConfig, rootConfigElement)
PlusStatus GetVideoSourceByIndex(const unsigned int index, vtkPlusDataSource *&aVideoSource)
This class is used to store a configuration of the imaging parameters of an Microsoft Media Foundatio...
PlusStatus SetInputFrameSize(unsigned int x, unsigned int y, unsigned int z)
virtual PlusStatus AddItem(vtkImageData *frame, US_IMAGE_ORIENTATION usImageOrientation, US_IMAGE_TYPE imageType, long frameNumber, double unfilteredTimestamp=UNDEFINED_TIMESTAMP, double filteredTimestamp=UNDEFINED_TIMESTAMP, const igsioFieldMapType *customFields=NULL)
STDMETHODIMP_(ULONG) MmfVideoSourceReader
static PlusStatus ConvertToBGR24(ComponentOrdering outputOrdering, PixelEncoding inputCompression, int width, int height, unsigned char *s, unsigned char *d)
virtual PlusStatus WriteConfiguration(vtkXMLDataElement *xmlElement)
bool RequireImageOrientationInConfiguration
virtual PlusStatus NotifyConfigured()
std::wstring GetRequestedDeviceName()
static const char * KEY_AUTO_EXPOSURE
PlusStatus SetPixelType(igsioCommon::VTKScalarPixelType pixelType)
static const char * KEY_ZOOM_MM
virtual double GetAcquisitionRate() const
static const char * KEY_EXPOSURE_LOG2SECONDS
virtual void SetRequestedVideoFormat(const std::wstring &pixelFormatName)
static const char * CAMERA_CONTROL_XML_ELEMENT_TAG
virtual PlusStatus InternalStartRecording()
VideoFormat ActiveVideoFormat
static const char * KEY_IRIS_FSTOP
void LogListOfCaptureDevices()
static const char * KEY_ROLL_DEGREES
static const char * KEY_AUTO_FOCUS
#define XML_FIND_DEVICE_ELEMENT_REQUIRED_FOR_READING(deviceConfig, rootConfigElement)
static const char * KEY_PAN_DEGREES
double GetUnfilteredTimestamp(double localTimeOffsetSec)
void LogListOfCaptureVideoFormats(unsigned int deviceId)
virtual US_IMAGE_ORIENTATION GetInputImageOrientation()
static const char * KEY_TILT_DEGREES
std::wstring GetCaptureDeviceName(unsigned int deviceId)
vtkStandardNewMacro(vtkPlusMmfVideoSource)
virtual ItemStatus GetLatestStreamBufferItem(StreamBufferItem *bufferItem)
PlusStatus UpdateFrameSize()
static PlusStatus ConvertToGray(int inputCompression, int width, int height, unsigned char *s, unsigned char *d)
virtual void PrintSelf(ostream &os, vtkIndent indent) VTK_OVERRIDE
virtual PlusStatus InternalApplyCameraControlParameterChange()
igsioVideoFrame UncompressedVideoFrame
void GetListOfCaptureVideoFormats(std::vector< std::wstring > &videoModes, unsigned int deviceId)
ChannelContainer OutputChannels
virtual void SetRequestedStreamIndex(unsigned int streamIndex)
virtual void SetRequestedFrameSize(const FrameSizeType &frameSize)
PlusStatus SetNumberOfScalarComponents(unsigned int numberOfScalarComponents)
virtual PlusStatus InternalStopRecording()
static const char * KEY_FOCUS_MM
friend class MmfVideoSourceReader
VideoFormat RequestedVideoFormat
PlusStatus AddFrame(unsigned char *bufferData, DWORD bufferSize)
FrameSizeType GetInputFrameSize() const
Microsoft media foundation video digitizer.
virtual PlusStatus InternalDisconnect()
MmfVideoSourceReader * MmfSourceReader
virtual int GetNumberOfItems()
vtkSmartPointer< vtkPlusCameraControlParameters > CameraControlParameters
std::wstring GetActiveDeviceName()
void GetListOfCaptureDevices(std::vector< std::wstring > &deviceNames)
Interface to a 3D positioning tool, video source, or generalized data stream.