PlusLib  2.9.0
Software library for tracked ultrasound image acquisition, calibration, and processing.
vtkPlusMmfVideoSource.cxx
Go to the documentation of this file.
1 /*=Plus=header=begin======================================================
2 Program: Plus
3 Copyright (c) Laboratory for Percutaneous Surgery. All rights reserved.
4 See License.txt for details.
5 =========================================================Plus=header=end*/
6 
7 /*=========================================================================
8 The following copyright notice is applicable to parts of this file:
9 Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
10 All rights reserved.
11 See Copyright.txt or http://www.kitware.com/Copyright.htm for details.
12 Authors include: Adam Rankin
13 =========================================================================*/
14 
15 // Local includes
16 #include "PlusConfigure.h"
17 #include "PixelCodec.h"
19 #include "vtkPlusChannel.h"
20 #include "vtkPlusDataSource.h"
21 #include "vtkPlusMmfVideoSource.h"
22 
23 // VTK includes
24 #include <vtkObjectFactory.h>
25 
26 // VideoCapture API includes
27 #include <MediaFoundationVideoCaptureApi.h>
28 #include <FormatReader.h>
29 
30 // Media foundation includes - require Microsoft Windows SDK 7.1 or later.
31 // Download from: http://www.microsoft.com/en-us/download/details.aspx?id=8279
32 #include <Mfapi.h>
33 #include <Mferror.h>
34 #include <Mfidl.h>
35 #include <Mfreadwrite.h>
36 
37 // Windows includes
38 #include <lmerr.h>
39 #include <shlwapi.h>
40 #include <strmif.h>
41 #include <tchar.h>
42 #include <windows.h>
43 
44 //----------------------------------------------------------------------------
45 
46 namespace
47 {
48  const unsigned int DEFAULT_DEVICE_ID = 0;
49  const FrameSizeType DEFAULT_FRAME_SIZE = { 640, 480, 1 };
50  const double DEFAULT_ACQUISITION_RATE = 30;
51  const std::wstring DEFAULT_PIXEL_TYPE_NAME = L"YUY2";
52  const GUID DEFAULT_PIXEL_TYPE = MFVideoFormat_YUY2; // see http://msdn.microsoft.com/en-us/library/windows/desktop/aa370819(v=vs.85).aspx
53  const std::wstring MF_VIDEO_FORMAT_PREFIX = L"MFVideoFormat_";
54  const int ERRMSGBUFFERSIZE = 1024;
55  template <class T> void SafeRelease(T** ppT)
56  {
57  if (*ppT)
58  {
59  (*ppT)->Release();
60  *ppT = NULL;
61  }
62  }
63 }
64 
65 //----------------------------------------------------------------------------
66 class MmfVideoSourceReader : public IMFSourceReaderCallback
67 {
68 public:
69 
70  MmfVideoSourceReader(vtkPlusMmfVideoSource* plusDevice)
71  : CaptureSource(NULL)
72  , CaptureSourceReader(NULL)
73  , RefCount(0)
74  , PlusDevice(plusDevice)
75  {
76  };
77 
78  //------- IMFSourceReaderCallback functions ----------------------
79  STDMETHODIMP QueryInterface(REFIID iid, void** ppv);
80  STDMETHOD_(ULONG, AddRef)();
81  STDMETHOD_(ULONG, Release)();
82  STDMETHODIMP OnEvent(DWORD, IMFMediaEvent*);
83  STDMETHODIMP OnFlush(DWORD);
84  STDMETHODIMP OnReadSample(HRESULT hrStatus, DWORD dwStreamIndex, DWORD dwStreamFlags, LONGLONG llTimestamp, IMFSample* pSample);
85 
86  IMFMediaSource* CaptureSource;
87  IMFSourceReader* CaptureSourceReader;
88 
89  long RefCount;
90  vtkPlusMmfVideoSource* PlusDevice;
91 };
92 
93 //----------------------------------------------------------------------------
94 STDMETHODIMP MmfVideoSourceReader::QueryInterface(REFIID iid, void** ppv)
95 {
96  static const QITAB qit[] =
97  {
98  QITABENT(MmfVideoSourceReader, IMFSourceReaderCallback),
99  { 0 },
100  };
101  return QISearch(this, qit, iid, ppv);
102 }
103 
104 //----------------------------------------------------------------------------
105 STDMETHODIMP_(ULONG) MmfVideoSourceReader::AddRef()
106 {
107  LONG uCount = InterlockedIncrement(&RefCount);
108  if (uCount == 1)
109  {
110  this->PlusDevice->Register(NULL);
111  }
112  return uCount;
113 }
114 
115 //----------------------------------------------------------------------------
116 STDMETHODIMP_(ULONG) MmfVideoSourceReader::Release()
117 {
118  ULONG uCount = InterlockedDecrement(&RefCount);
119  if (uCount == 0)
120  {
121  LOG_DEBUG("vtkPlusMmfVideoSource::Release - unregister");
122  this->PlusDevice->UnRegister(NULL);
123  }
124  return uCount;
125 }
126 
127 //----------------------------------------------------------------------------
128 STDMETHODIMP MmfVideoSourceReader::OnEvent(DWORD, IMFMediaEvent*)
129 {
130  return S_OK;
131 }
132 
133 //----------------------------------------------------------------------------
134 STDMETHODIMP MmfVideoSourceReader::OnFlush(DWORD)
135 {
136  return S_OK;
137 }
138 
139 //----------------------------------------------------------------------------
140 STDMETHODIMP MmfVideoSourceReader::OnReadSample(HRESULT hrStatus, DWORD dwStreamIndex, DWORD dwStreamFlags, LONGLONG llTimestamp, IMFSample* pSample)
141 {
142  igsioLockGuard<vtkIGSIORecursiveCriticalSection> updateMutexGuardedLock(this->PlusDevice->Mutex);
143 
144  if (!SUCCEEDED(hrStatus))
145  {
146  // Streaming error
147  LOG_ERROR("Source Reader error: " << std::hex << hrStatus);
148  return S_FALSE;
149  }
150 
151  if (!this->PlusDevice->IsRecording())
152  {
153  return S_OK;
154  }
155 
156  if (this->CaptureSourceReader == NULL)
157  {
158  return S_FALSE;
159  }
160 
161  if (pSample != NULL)
162  {
163 
164  // Get the media type from the stream.
165  IMFMediaType* pType = NULL;
166  this->CaptureSourceReader->GetCurrentMediaType(this->PlusDevice->ActiveVideoFormat.StreamIndex, &pType);
167  if (pType == NULL)
168  {
169  LOG_ERROR("Cannot get current media type");
170  }
171 
172  // Check the image size, as it may be different from what we requested (even if setup does not give any error).
173  // Mostly happens when the native resolution has a different aspect ration (e.g., 640x480 is requested but actually 640x360 is received).
174  // The check has to be done here, the media type is not yet available at InternalConnect time.
175  UINT32 actualWidth = 0;
176  UINT32 actualHeight = 0;
177  ::MFGetAttributeSize(pType, MF_MT_FRAME_SIZE, &actualWidth, &actualHeight);
178  if (actualWidth != this->PlusDevice->ActiveVideoFormat.FrameSize[0] ||
179  actualHeight != this->PlusDevice->ActiveVideoFormat.FrameSize[1])
180  {
181  LOG_ERROR("Unexpected frame size: " << actualWidth << "x" << actualHeight << " (expected: " << this->PlusDevice->ActiveVideoFormat.FrameSize[0] << "x" << this->PlusDevice->ActiveVideoFormat.FrameSize[1] << ")");
182  return S_FALSE;
183  }
184 
185  // Check the pixel type, as it may be different from what we requested (even if setup does not give any error).
186  // Mostly happens for larger resolutions (e.g., when requesting webcam feed at 1280x720 with YUY then we get MJPG).
187  // The check has to be done here, the media type is not yet available at InternalConnect time.
188  GUID videoFormat = DEFAULT_PIXEL_TYPE;
189  pType->GetGUID(MF_MT_SUBTYPE, &videoFormat);
190  std::wstring videoFormatWStr = MfVideoCapture::FormatReader::StringFromGUID(videoFormat);
191  if (igsioCommon::HasSubstrInsensitive(videoFormatWStr, MF_VIDEO_FORMAT_PREFIX))
192  {
193  // found standard prefix, remove it
194  videoFormatWStr.erase(0, MF_VIDEO_FORMAT_PREFIX.size());
195  }
196 
197  if (!igsioCommon::IsEqualInsensitive(videoFormatWStr, this->PlusDevice->ActiveVideoFormat.PixelFormatName))
198  {
199  LOG_ERROR_W("Unexpected video format: " << videoFormatWStr << " (expected: " << this->PlusDevice->ActiveVideoFormat.PixelFormatName << ")");
200  return S_FALSE;
201  }
202 
203  IMFMediaBuffer* aBuffer = NULL;
204  DWORD bufferCount = 0;
205  pSample->GetBufferCount(&bufferCount);
206  if (bufferCount < 1)
207  {
208  LOG_ERROR("No buffer available in the sample.");
209  return S_FALSE;
210  }
211  pSample->GetBufferByIndex(0, &aBuffer);
212  BYTE* bufferData = NULL;
213  DWORD maxLength = 0;
214  DWORD currentLength = 0;
215 
216  HRESULT hr = aBuffer->Lock(&bufferData, &maxLength, &currentLength);
217  if (SUCCEEDED(hr))
218  {
219  this->PlusDevice->AddFrame(bufferData, currentLength);
220  aBuffer->Unlock();
221  }
222  else
223  {
224  LOG_ERROR("Unable to lock the buffer.");
225  }
226  SafeRelease(&aBuffer);
227  }
228 
229  if (MF_SOURCE_READERF_ENDOFSTREAM & dwStreamFlags)
230  {
231  // Reached the end of the stream.
232  LOG_ERROR("End of stream reached. Capture device should never reach end of stream.");
233  this->PlusDevice->Disconnect();
234  return S_FALSE;
235  // This should never occur under normal operation.
236  }
237 
238  this->CaptureSourceReader->ReadSample(this->PlusDevice->ActiveVideoFormat.StreamIndex, 0, NULL, NULL, NULL, NULL);
239 
240  return S_OK;
241 }
242 
243 //----------------------------------------------------------------------------
244 
246 
247 //----------------------------------------------------------------------------
249  : FrameIndex(0)
250  , Mutex(vtkSmartPointer<vtkIGSIORecursiveCriticalSection>::New())
251  , CameraControlParameters(vtkSmartPointer<vtkPlusCameraControlParameters>::New())
252 {
253  this->MmfSourceReader = new MmfVideoSourceReader(this);
255 
256  this->AcquisitionRate = DEFAULT_ACQUISITION_RATE;
257 
258  this->RequestedVideoFormat.DeviceId = DEFAULT_DEVICE_ID;
259  this->RequestedVideoFormat.StreamIndex = 0;
260  this->RequestedVideoFormat.FrameSize[0] = DEFAULT_FRAME_SIZE[0];
261  this->RequestedVideoFormat.FrameSize[1] = DEFAULT_FRAME_SIZE[1];
262  this->RequestedVideoFormat.FrameSize[2] = DEFAULT_FRAME_SIZE[2];
263  this->RequestedVideoFormat.PixelFormatName = DEFAULT_PIXEL_TYPE_NAME;
264 
265  this->ActiveVideoFormat.DeviceId = DEFAULT_DEVICE_ID;
266  this->ActiveVideoFormat.FrameSize[0] = 0;
267  this->ActiveVideoFormat.FrameSize[1] = 0;
268  this->ActiveVideoFormat.FrameSize[2] = 0;
269 }
270 
271 //----------------------------------------------------------------------------
273 {
274  delete this->MmfSourceReader;
275  this->MmfSourceReader = NULL;
276 }
277 
278 //----------------------------------------------------------------------------
279 void vtkPlusMmfVideoSource::PrintSelf(ostream& os, vtkIndent indent)
280 {
281  this->Superclass::PrintSelf(os, indent);
282 
283  os << indent << "FrameIndex: " << (this->FrameIndex ? "On\n" : "Off\n");
284 
285  this->CameraControlParameters->PrintSelf(os, indent.GetNextIndent());
286 }
287 
288 //----------------------------------------------------------------------------
290 {
291  igsioLockGuard<vtkIGSIORecursiveCriticalSection> updateMutexGuardedLock(this->Mutex);
292 
293  if (this->MmfSourceReader->RefCount != 0)
294  {
295  LOG_WARNING("There is a reference to this class from a previous connection");
296  }
297 
299 
300  GUID pixelFormat = DEFAULT_PIXEL_TYPE;
301  if (!this->RequestedVideoFormat.PixelFormatName.empty())
302  {
303  std::wstring videoFormat = MF_VIDEO_FORMAT_PREFIX + this->RequestedVideoFormat.PixelFormatName;
304  pixelFormat = MfVideoCapture::FormatReader::GUIDFromString(videoFormat);
305  if (pixelFormat == GUID_NULL)
306  {
307  LOG_ERROR_W("Cannot recognize requested pixel format: " << this->RequestedVideoFormat.PixelFormatName << ". Defaulting to \'" << DEFAULT_PIXEL_TYPE_NAME << "\'.");
308  pixelFormat = DEFAULT_PIXEL_TYPE;
309  }
310  }
311 
312  if (!MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().SetupDevice(this->RequestedVideoFormat.DeviceId, this->RequestedVideoFormat.StreamIndex,
313  this->RequestedVideoFormat.FrameSize[0], this->RequestedVideoFormat.FrameSize[1], this->AcquisitionRate, pixelFormat))
314  {
315  LOG_WARNING_W("Unable to init capture device with requested details:"
316  << " device ID: " << this->RequestedVideoFormat.DeviceId << " (" << GetRequestedDeviceName() << ") stream " << this->RequestedVideoFormat.StreamIndex
317  << ", " << this->RequestedVideoFormat.FrameSize[0] << "x" << this->RequestedVideoFormat.FrameSize[1]
318  << ", " << this->AcquisitionRate << "Hz, " << this->ActiveVideoFormat.PixelFormatName);
319 
321 
322  if (!MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().SetupDevice(DEFAULT_DEVICE_ID, 0, DEFAULT_FRAME_SIZE[0], DEFAULT_FRAME_SIZE[1], DEFAULT_ACQUISITION_RATE, DEFAULT_PIXEL_TYPE))
323  {
324  LOG_ERROR_W("Unable to initialize capture device with default details: device ID: " << DEFAULT_DEVICE_ID << " (" << GetCaptureDeviceName(DEFAULT_DEVICE_ID) << ") stream 0, " << DEFAULT_FRAME_SIZE[0] << "x" << DEFAULT_FRAME_SIZE[1] << ", " << DEFAULT_ACQUISITION_RATE << "Hz, " << DEFAULT_PIXEL_TYPE_NAME);
326  LogListOfCaptureVideoFormats(DEFAULT_DEVICE_ID);
327  return PLUS_FAIL;
328  }
329  this->ActiveVideoFormat.DeviceId = DEFAULT_DEVICE_ID;
330  this->ActiveVideoFormat.FrameSize[0] = DEFAULT_FRAME_SIZE[0];
331  this->ActiveVideoFormat.FrameSize[1] = DEFAULT_FRAME_SIZE[1];
332  this->ActiveVideoFormat.FrameSize[2] = DEFAULT_FRAME_SIZE[2];
333  this->ActiveVideoFormat.PixelFormatName = DEFAULT_PIXEL_TYPE_NAME;
334 
335  LOG_INFO_W("Backing up to connecting with default capture settings:"
336  << " device ID: " << this->ActiveVideoFormat.DeviceId << " (" << GetActiveDeviceName() << ")"
337  << ", " << this->ActiveVideoFormat.FrameSize[0] << "x" << this->ActiveVideoFormat.FrameSize[1]
338  << ", " << DEFAULT_ACQUISITION_RATE << "Hz, " << this->ActiveVideoFormat.PixelFormatName);
339  }
340 
341  this->MmfSourceReader->CaptureSource = MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().GetMediaSource(this->ActiveVideoFormat.DeviceId);
342  if (this->MmfSourceReader->CaptureSource == NULL)
343  {
344  LOG_ERROR("Unable to request capture source from the media foundation library.");
345  return PLUS_FAIL;
346  }
347 
348  unsigned int frameRate = MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().GetFrameRate(this->ActiveVideoFormat.DeviceId);
349  LOG_DEBUG_W("vtkPlusMmfVideoSource connected to device '" << GetActiveDeviceName() << "' at frame rate of " << frameRate << "Hz");
350 
352  {
353  LOG_ERROR("Failed to change imaging parameters in the device");
354  return PLUS_FAIL;
355  }
356 
357  this->FrameIndex = 0;
358 
359  return PLUS_SUCCESS;
360 }
361 
362 //----------------------------------------------------------------------------
364 {
365  igsioLockGuard<vtkIGSIORecursiveCriticalSection> updateMutexGuardedLock(this->Mutex);
366 
367  MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().CloseDevice(this->ActiveVideoFormat.DeviceId);
368  this->MmfSourceReader->CaptureSource = NULL;
369 
370  return PLUS_SUCCESS;
371 }
372 
373 //----------------------------------------------------------------------------
375 {
376  igsioLockGuard<vtkIGSIORecursiveCriticalSection> updateMutexGuardedLock(this->Mutex);
377 
378  HRESULT hr;
379  if (this->MmfSourceReader->CaptureSource != NULL)
380  {
381  IMFAttributes* attr;
382  MFCreateAttributes(&attr, 2);
383  hr = attr->SetUnknown(MF_SOURCE_READER_ASYNC_CALLBACK, this->MmfSourceReader);
384  hr = attr->SetUINT32(MF_READWRITE_DISABLE_CONVERTERS, TRUE);
385 
386  hr = MFCreateSourceReaderFromMediaSource(this->MmfSourceReader->CaptureSource, attr, &this->MmfSourceReader->CaptureSourceReader);
387 
388  if (FAILED(hr))
389  {
390  LOG_ERROR("Unable to create source reader from media source.");
391  return PLUS_FAIL;
392  }
393  this->UpdateFrameSize();
394 
395  attr->Release();
396 
397  std::wstring videoFormat = MF_VIDEO_FORMAT_PREFIX + this->RequestedVideoFormat.PixelFormatName;
398 
399  IMFMediaType* pDecodeType = NULL;
400  hr = MFCreateMediaType(&pDecodeType);
401  hr = pDecodeType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
402  hr = pDecodeType->SetGUID(MF_MT_SUBTYPE, MfVideoCapture::FormatReader::GUIDFromString(videoFormat));
403 
404  hr = this->MmfSourceReader->CaptureSourceReader->SetCurrentMediaType(this->ActiveVideoFormat.StreamIndex, NULL, pDecodeType);
405  if (FAILED(hr))
406  {
407  LOG_WARNING_W("Unable to set SourceReader output to requested format: " << this->RequestedVideoFormat.PixelFormatName
408  << ". Using device default.");
409  }
410  SafeRelease(&pDecodeType);
411 
412  MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().StartRecording(this->ActiveVideoFormat.DeviceId);
413  this->MmfSourceReader->CaptureSourceReader->ReadSample(this->ActiveVideoFormat.StreamIndex, 0, NULL, NULL, NULL, NULL);
414  }
415  else
416  {
417  LOG_ERROR("Unable to request IMFMediaSource from the media foundation capture library. Unable to continue.");
418  return PLUS_FAIL;
419  }
420 
421  return PLUS_SUCCESS;
422 }
423 
424 //----------------------------------------------------------------------------
426 {
427  LOG_DEBUG("vtkPlusMmfVideoSource::InternalStopRecording");
428 
429  igsioLockGuard<vtkIGSIORecursiveCriticalSection> updateMutexGuardedLock(this->Mutex);
430 
431  MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().StopRecording(this->ActiveVideoFormat.DeviceId);
432 
433  this->MmfSourceReader->CaptureSourceReader->Flush(this->ActiveVideoFormat.StreamIndex);
434  SafeRelease(&this->MmfSourceReader->CaptureSourceReader);
435  return PLUS_SUCCESS;
436 }
437 
438 //----------------------------------------------------------------------------
440 {
441  if (this->OutputChannels.size() == 0)
442  {
443  LOG_ERROR("No output channels defined for microsoft media foundation video source. Cannot proceed.");
444  this->CorrectlyConfigured = false;
445  return PLUS_FAIL;
446  }
447 
448  return PLUS_SUCCESS;
449 }
450 
451 //----------------------------------------------------------------------------
453 {
454  if (this->MmfSourceReader->CaptureSourceReader != NULL)
455  {
456  int numberOfVideoSources = this->GetNumberOfVideoSources();
457  for (int i = 0; i < numberOfVideoSources; ++i)
458  {
459  vtkPlusDataSource* videoSource(NULL);
460  this->GetVideoSourceByIndex(i, videoSource);
461 
462  FrameSizeType currentFrameSize = videoSource->GetInputFrameSize();
463  if (currentFrameSize[0] != this->ActiveVideoFormat.FrameSize[0] || currentFrameSize[1] != this->ActiveVideoFormat.FrameSize[1] || currentFrameSize[2] != 1)
464  {
465  currentFrameSize[0] = this->ActiveVideoFormat.FrameSize[0];
466  currentFrameSize[1] = this->ActiveVideoFormat.FrameSize[1];
467  currentFrameSize[2] = this->ActiveVideoFormat.FrameSize[2];
468  videoSource->SetInputFrameSize(currentFrameSize);
469  videoSource->SetPixelType(VTK_UNSIGNED_CHAR);
470  unsigned int numberOfScalarComponents = (videoSource->GetImageType() == US_IMG_RGB_COLOR ? 3 : 1);
471  videoSource->SetNumberOfScalarComponents(numberOfScalarComponents);
472  this->UncompressedVideoFrame.SetImageType(videoSource->GetImageType());
473  this->UncompressedVideoFrame.SetImageOrientation(videoSource->GetInputImageOrientation());
474  this->UncompressedVideoFrame.AllocateFrame(currentFrameSize, VTK_UNSIGNED_CHAR, numberOfScalarComponents);
475  }
476  }
477  }
478 
479  return PLUS_SUCCESS;
480 }
481 
482 //----------------------------------------------------------------------------
483 PlusStatus vtkPlusMmfVideoSource::ReadConfiguration(vtkXMLDataElement* rootConfigElement)
484 {
485  XML_FIND_DEVICE_ELEMENT_REQUIRED_FOR_READING(deviceConfig, rootConfigElement);
486 
487  int deviceId = 0;
488  if (deviceConfig->GetScalarAttribute("CaptureDeviceId", deviceId))
489  {
490  this->RequestedVideoFormat.DeviceId = deviceId;
491  }
492 
493  int streamIndex = 0;
494  if (deviceConfig->GetScalarAttribute("CaptureStreamIndex", streamIndex))
495  {
496  this->RequestedVideoFormat.StreamIndex = (DWORD)streamIndex;
497  }
498 
499  int requestedFrameSize[2] = { static_cast<int>(DEFAULT_FRAME_SIZE[0]), static_cast<int>(DEFAULT_FRAME_SIZE[1]) };
500  if (deviceConfig->GetVectorAttribute("FrameSize", 2, requestedFrameSize))
501  {
502  if (requestedFrameSize[0] < 0 || requestedFrameSize[1] < 0)
503  {
504  LOG_ERROR("Negative frame size defined in config file. Cannot continue.");
505  return PLUS_FAIL;
506  }
507  this->RequestedVideoFormat.FrameSize[0] = static_cast<unsigned int>(requestedFrameSize[0]);
508  this->RequestedVideoFormat.FrameSize[1] = static_cast<unsigned int>(requestedFrameSize[1]);
509  this->RequestedVideoFormat.FrameSize[2] = 1;
510  }
511 
512  this->RequestedVideoFormat.PixelFormatName.clear();
513  if (deviceConfig->GetAttribute("VideoFormat") != NULL)
514  {
515  auto attr = std::string(deviceConfig->GetAttribute("VideoFormat"));
516  this->RequestedVideoFormat.PixelFormatName = std::wstring(attr.begin(), attr.end());
517  }
518 
519  XML_FIND_NESTED_ELEMENT_OPTIONAL(cameraParameters, deviceConfig, vtkPlusCameraControlParameters::CAMERA_CONTROL_XML_ELEMENT_TAG);
520  if (cameraParameters != NULL)
521  {
522  this->CameraControlParameters->ReadConfiguration(deviceConfig);
523  }
524 
525  return PLUS_SUCCESS;
526 }
527 
528 //----------------------------------------------------------------------------
529 PlusStatus vtkPlusMmfVideoSource::WriteConfiguration(vtkXMLDataElement* rootConfigElement)
530 {
531  XML_FIND_DEVICE_ELEMENT_REQUIRED_FOR_WRITING(deviceConfig, rootConfigElement);
532 
533  deviceConfig->SetIntAttribute("CaptureDeviceId", this->RequestedVideoFormat.DeviceId);
534  if (this->RequestedVideoFormat.StreamIndex != 0)
535  {
536  deviceConfig->SetIntAttribute("CaptureStreamIndex", this->RequestedVideoFormat.StreamIndex);
537  }
538  else
539  {
540  XML_REMOVE_ATTRIBUTE("CaptureStreamIndex", deviceConfig);
541  }
542  int frameSize[2] = { static_cast<int>(this->RequestedVideoFormat.FrameSize[0]), static_cast<int>(this->RequestedVideoFormat.FrameSize[1]) };
543  deviceConfig->SetVectorAttribute("FrameSize", 2, frameSize);
544  auto attr = std::string(this->RequestedVideoFormat.PixelFormatName.begin(), this->RequestedVideoFormat.PixelFormatName.end());
545  deviceConfig->SetAttribute("VideoFormat", attr.c_str());
546 
547  this->CameraControlParameters->WriteConfiguration(deviceConfig);
548 
549  return PLUS_SUCCESS;
550 }
551 
552 //----------------------------------------------------------------------------
553 void vtkPlusMmfVideoSource::SetRequestedVideoFormat(const std::wstring& pixelFormatName)
554 {
555  this->RequestedVideoFormat.PixelFormatName = pixelFormatName;
556 }
557 
558 //----------------------------------------------------------------------------
560 {
561  this->RequestedVideoFormat.StreamIndex = (DWORD)streamIndex;
562 }
563 
564 //----------------------------------------------------------------------------
565 void vtkPlusMmfVideoSource::SetRequestedFrameSize(const FrameSizeType& frameSize)
566 {
567  this->RequestedVideoFormat.FrameSize = frameSize;
568 }
569 
570 //----------------------------------------------------------------------------
572 {
573  this->RequestedVideoFormat.DeviceId = deviceId;
574 }
575 
576 //----------------------------------------------------------------------------
577 void vtkPlusMmfVideoSource::GetListOfCaptureDevices(std::vector<std::wstring>& deviceNames)
578 {
579  MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().GetDeviceNames(deviceNames);
580 }
581 
582 //----------------------------------------------------------------------------
583 void vtkPlusMmfVideoSource::GetListOfCaptureVideoFormats(std::vector<std::wstring>& videoModes, unsigned int deviceId)
584 {
585  videoModes.clear();
586  unsigned int numberOfStreams = MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().GetNumberOfStreams(deviceId);
587  for (unsigned int streamIndex = 0; streamIndex < numberOfStreams; ++streamIndex)
588  {
589  unsigned int numberOfVideoFormats = MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().GetNumberOfFormats(deviceId, streamIndex);
590  for (unsigned int formatIndex = 0; formatIndex < numberOfVideoFormats; formatIndex++)
591  {
592  MfVideoCapture::MediaType type = MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().GetFormat(deviceId, streamIndex, formatIndex);
593  std::wstring pixelType(type.MF_MT_SUBTYPEName.begin(), type.MF_MT_SUBTYPEName.end());
594  if (igsioCommon::HasSubstrInsensitive(pixelType, MF_VIDEO_FORMAT_PREFIX))
595  {
596  // found standard prefix, remove it
597  pixelType.erase(0, MF_VIDEO_FORMAT_PREFIX.size());
598  }
599  std::wostringstream strFriendlyName;
600  strFriendlyName << "Stream index " << streamIndex;
601  strFriendlyName << " - Frame size: " << type.width << "x" << type.height;
602  strFriendlyName << ", video format: " << pixelType;
603  strFriendlyName << ", frame rate: " << type.MF_MT_FRAME_RATE;
604  videoModes.push_back(strFriendlyName.str());
605  }
606  }
607 }
608 
609 //----------------------------------------------------------------------------
611 {
612  return GetCaptureDeviceName(this->RequestedVideoFormat.DeviceId);
613 }
614 
615 //----------------------------------------------------------------------------
617 {
618  return GetCaptureDeviceName(this->ActiveVideoFormat.DeviceId);
619 }
620 
621 //----------------------------------------------------------------------------
622 PlusStatus vtkPlusMmfVideoSource::AddFrame(unsigned char* bufferData, DWORD bufferSize)
623 {
624  if (!this->Recording)
625  {
626  LOG_ERROR("vtkPlusMmfVideoSource::AddFrame skipped, not recording anymore");
627  return PLUS_SUCCESS;
628  }
629 
630  int numberOfVideoSources = this->GetNumberOfVideoSources();
631  PlusStatus status = PLUS_SUCCESS;
632  for (int i = 0; i < numberOfVideoSources; ++i)
633  {
634  vtkPlusDataSource* videoSource(NULL);
635  if (this->GetVideoSourceByIndex(i, videoSource) != PLUS_SUCCESS)
636  {
637  return PLUS_FAIL;
638  }
639  this->FrameIndex++;
640 
641  FrameSizeType frameSize = videoSource->GetInputFrameSize();
642 
643  PlusStatus decodingStatus(PLUS_SUCCESS);
645  if (igsioCommon::IsEqualInsensitive(this->ActiveVideoFormat.PixelFormatName, L"YUY2"))
646  {
647  if (bufferSize < frameSize[0] * frameSize[1] * 2)
648  {
649  LOG_ERROR("Failed to decode pixel data from YUY2 due to buffer size mismatch");
650  return PLUS_FAIL;
651  }
653  }
654  else if (igsioCommon::IsEqualInsensitive(this->ActiveVideoFormat.PixelFormatName, L"MJPG"))
655  {
657  }
658  else if (igsioCommon::IsEqualInsensitive(this->ActiveVideoFormat.PixelFormatName, L"RGB24"))
659  {
660  if (bufferSize < frameSize[0] * frameSize[1] * 3)
661  {
662  LOG_ERROR("Failed to decode pixel data from RGB24 due to buffer size mismatch");
663  return PLUS_FAIL;
664  }
666  }
667  else
668  {
669  LOG_ERROR_W("Unknown pixel type: " << this->ActiveVideoFormat.PixelFormatName << " (only YUY2, MJPG and RGB24 are supported)");
670  return PLUS_FAIL;
671  }
672 
673  if (videoSource->GetImageType() == US_IMG_RGB_COLOR)
674  {
675  decodingStatus = PixelCodec::ConvertToBGR24(PixelCodec::ComponentOrder_RGB, encoding, frameSize[0], frameSize[1], bufferData, (unsigned char*)this->UncompressedVideoFrame.GetScalarPointer());
676  }
677  else
678  {
679  decodingStatus = PixelCodec::ConvertToGray(encoding, frameSize[0], frameSize[1], bufferData, (unsigned char*)this->UncompressedVideoFrame.GetScalarPointer());
680  }
681 
682  if (decodingStatus != PLUS_SUCCESS)
683  {
684  LOG_ERROR("Error while decoding the grabbed image");
685  return PLUS_FAIL;
686  }
687 
688  const double maximumFrameTimeVariance = 0.2; // to make sure we don't drop frames because of slight variance in acquisition rate, we allow up to 20% higher frame rate before we start dropping frames
689  double acquisitionRate = this->GetAcquisitionRate();
690  double minimumTimeBetweenBetweenRecordedFramesSec = (1.0 - maximumFrameTimeVariance) / this->GetAcquisitionRate();
691  double lastFrameTimeSec = -1.0;
692  double currentTime = vtkIGSIOAccurateTimer::GetSystemTime();
693 
694  StreamBufferItem latestFrame;
695  if (videoSource->GetNumberOfItems() > 2 && videoSource->GetLatestStreamBufferItem(&latestFrame) == ITEM_OK)
696  {
697  lastFrameTimeSec = latestFrame.GetUnfilteredTimestamp(0.0);
698  double secondsSinceLastFrame = currentTime - lastFrameTimeSec;
699  if (lastFrameTimeSec > 0 && secondsSinceLastFrame < minimumTimeBetweenBetweenRecordedFramesSec)
700  {
701  // For some webcams, the requested acquistion rate may not be availiable (not supported, or error in configuration).
702  // In this case we can artificially limit frames to the requested acquisition rate by ignoring frames.
703 
704  // The required time has not elapsed between frames.
705  // Do not need to record this frame.
706  return PLUS_SUCCESS;
707  }
708  }
709 
710  PlusStatus sourceStatus = videoSource->AddItem(&this->UncompressedVideoFrame, this->FrameIndex, currentTime);
711  status = sourceStatus != PLUS_SUCCESS ? sourceStatus : status;
712  }
713 
714  this->Modified();
715  return status;
716 }
717 
718 //----------------------------------------------------------------------------
720 {
721  LOG_INFO_W("Supported video formats for Device Id " << deviceId << " (" << GetCaptureDeviceName(deviceId) << ")");
722  std::vector<std::wstring> videoModes;
723  GetListOfCaptureVideoFormats(videoModes, deviceId);
724  for (auto modeIt = videoModes.begin(); modeIt != videoModes.end(); ++modeIt)
725  {
726  LOG_INFO_W(" " << (*modeIt));
727  }
728 }
729 
730 //----------------------------------------------------------------------------
732 {
733  LOG_INFO("Found capture devices:");
734  std::vector<std::wstring> deviceNames;
735  GetListOfCaptureDevices(deviceNames);
736  int id = 0;
737  for (auto deviceNameIt = deviceNames.begin(); deviceNameIt != deviceNames.end(); ++deviceNameIt, id++)
738  {
739  LOG_INFO_W(" " << id << ": " << (*deviceNameIt));
740  }
741 }
742 
743 //----------------------------------------------------------------------------
744 std::wstring vtkPlusMmfVideoSource::GetCaptureDeviceName(unsigned int deviceId)
745 {
746  return MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().GetCaptureDeviceName(deviceId);
747 }
748 
749 //----------------------------------------------------------------------------
751 {
752  if (this->MmfSourceReader->CaptureSource == NULL)
753  {
754  LOG_ERROR("Cannot apply camera control parameter change, capture source is not initialized");
755  return PLUS_FAIL;
756  }
757 
758  MfVideoCapture::CaptureDeviceParameters parameters =
759  MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().GetParameters(this->ActiveVideoFormat.DeviceId);
760 
762  // Pan
765  {
766  double panDegrees = 0.0;
767  if (this->CameraControlParameters->GetPanDegrees(panDegrees) != PLUS_SUCCESS)
768  {
769  LOG_ERROR("Failed to get pan camera control parameter");
770  return PLUS_FAIL;
771  }
772  parameters.CameraControlParameters[MfVideoCapture::CaptureDeviceParameters::Pan].CurrentValue = panDegrees;
774  }
775 
777  // Tilt
780  {
781  double tiltDegrees = 0.0;
782  if (this->CameraControlParameters->GetTiltDegrees(tiltDegrees) != PLUS_SUCCESS)
783  {
784  LOG_ERROR("Failed to get tilt camera control parameter");
785  return PLUS_FAIL;
786  }
787  parameters.CameraControlParameters[MfVideoCapture::CaptureDeviceParameters::Tilt].CurrentValue = tiltDegrees;
789  }
790 
792  // Roll
795  {
796  double rollDegrees = 0.0;
797  if (this->CameraControlParameters->GetRollDegrees(rollDegrees) != PLUS_SUCCESS)
798  {
799  LOG_ERROR("Failed to get roll camera control parameter");
800  return PLUS_FAIL;
801  }
802  parameters.CameraControlParameters[MfVideoCapture::CaptureDeviceParameters::Roll].CurrentValue = rollDegrees;
804  }
805 
807  // Zoom
810  {
811  double zoomMm = 0.0;
812  if (this->CameraControlParameters->GetZoomMm(zoomMm) != PLUS_SUCCESS)
813  {
814  LOG_ERROR("Failed to get zoom camera control parameter");
815  return PLUS_FAIL;
816  }
817  parameters.CameraControlParameters[MfVideoCapture::CaptureDeviceParameters::Zoom].CurrentValue = zoomMm;
819  }
820 
822  // Iris
825  {
826  int irisFStop = 0;
827  if (this->CameraControlParameters->GetIrisFStop(irisFStop) != PLUS_SUCCESS)
828  {
829  LOG_ERROR("Failed to get iris camera control parameter");
830  return PLUS_FAIL;
831  }
832  parameters.CameraControlParameters[MfVideoCapture::CaptureDeviceParameters::Iris].CurrentValue = irisFStop;
834  }
835 
836 
838  // Exposure
841  {
842  int exposureLog2Sec = 0;
843  if (this->CameraControlParameters->GetExposureLog2Seconds(exposureLog2Sec) != PLUS_SUCCESS)
844  {
845  LOG_ERROR("Failed to get exposure camera control parameter");
846  return PLUS_FAIL;
847  }
848  parameters.CameraControlParameters[MfVideoCapture::CaptureDeviceParameters::Exposure].CurrentValue = exposureLog2Sec;
850  }
851 
853  // AutoExposure
856  {
857  bool autoExposure = false;
858  if (this->CameraControlParameters->GetAutoExposure(autoExposure) != PLUS_SUCCESS)
859  {
860  LOG_ERROR("Failed to get auto exposure camera control parameter");
861  return PLUS_FAIL;
862  }
863  parameters.CameraControlParameters[MfVideoCapture::CaptureDeviceParameters::Exposure].Flag = autoExposure ? CameraControl_Flags_Auto : CameraControl_Flags_Manual;
865  }
866 
868  // Focus
871  {
872  double focusMm = 0.0;
873  if (this->CameraControlParameters->GetFocusMm(focusMm) != PLUS_SUCCESS)
874  {
875  LOG_ERROR("Failed to get focus camera control parameter");
876  return PLUS_FAIL;
877  }
878  parameters.CameraControlParameters[MfVideoCapture::CaptureDeviceParameters::Focus].CurrentValue = focusMm;
880  }
881 
883  // AutoFocus
886  {
887  bool autoFocus = false;
888  if (this->CameraControlParameters->GetAutoFocus(autoFocus) != PLUS_SUCCESS)
889  {
890  LOG_ERROR("Failed to get auto focus camera control parameter");
891  return PLUS_FAIL;
892  }
893  parameters.CameraControlParameters[MfVideoCapture::CaptureDeviceParameters::Focus].Flag = autoFocus ? CameraControl_Flags_Auto : CameraControl_Flags_Manual;
895  }
896 
898  // Update the parameters
899  MfVideoCapture::MediaFoundationVideoCaptureApi::GetInstance().SetParameters(this->ActiveVideoFormat.DeviceId, parameters);
900 
901  return PLUS_SUCCESS;
902 }
virtual void PrintSelf(ostream &os, vtkIndent indent) VTK_OVERRIDE
virtual PlusStatus ReadConfiguration(vtkXMLDataElement *xmlElement)
US_IMAGE_TYPE GetImageType()
virtual PlusStatus InternalConnect()
vtkSmartPointer< vtkIGSIORecursiveCriticalSection > Mutex
virtual int GetNumberOfVideoSources() const
virtual void SetRequestedDeviceId(unsigned int deviceId)
#define XML_FIND_DEVICE_ELEMENT_REQUIRED_FOR_WRITING(deviceConfig, rootConfigElement)
igsioStatus PlusStatus
Definition: PlusCommon.h:40
PlusStatus GetVideoSourceByIndex(const unsigned int index, vtkPlusDataSource *&aVideoSource)
This class is used to store a configuration of the imaging parameters of an Microsoft Media Foundatio...
PlusStatus SetInputFrameSize(unsigned int x, unsigned int y, unsigned int z)
virtual PlusStatus AddItem(vtkImageData *frame, US_IMAGE_ORIENTATION usImageOrientation, US_IMAGE_TYPE imageType, long frameNumber, double unfilteredTimestamp=UNDEFINED_TIMESTAMP, double filteredTimestamp=UNDEFINED_TIMESTAMP, const igsioFieldMapType *customFields=NULL)
STDMETHODIMP_(ULONG) MmfVideoSourceReader
static PlusStatus ConvertToBGR24(ComponentOrdering outputOrdering, PixelEncoding inputCompression, int width, int height, unsigned char *s, unsigned char *d)
Definition: PixelCodec.h:197
virtual PlusStatus WriteConfiguration(vtkXMLDataElement *xmlElement)
bool RequireImageOrientationInConfiguration
for i
double AcquisitionRate
virtual PlusStatus NotifyConfigured()
#define PLUS_FAIL
Definition: PlusCommon.h:43
PlusStatus SetPixelType(igsioCommon::VTKScalarPixelType pixelType)
virtual double GetAcquisitionRate() const
virtual void SetRequestedVideoFormat(const std::wstring &pixelFormatName)
virtual PlusStatus InternalStartRecording()
unsigned long DWORD
Definition: ATC3DGm.h:451
#define PLUS_SUCCESS
Definition: PlusCommon.h:44
#define XML_FIND_DEVICE_ELEMENT_REQUIRED_FOR_READING(deviceConfig, rootConfigElement)
double GetUnfilteredTimestamp(double localTimeOffsetSec)
void LogListOfCaptureVideoFormats(unsigned int deviceId)
virtual US_IMAGE_ORIENTATION GetInputImageOrientation()
unsigned long ULONG
Definition: ATC3DGm.h:432
std::wstring GetCaptureDeviceName(unsigned int deviceId)
vtkStandardNewMacro(vtkPlusMmfVideoSource)
virtual ItemStatus GetLatestStreamBufferItem(StreamBufferItem *bufferItem)
static PlusStatus ConvertToGray(int inputCompression, int width, int height, unsigned char *s, unsigned char *d)
Definition: PixelCodec.h:146
virtual void PrintSelf(ostream &os, vtkIndent indent) VTK_OVERRIDE
virtual PlusStatus InternalApplyCameraControlParameterChange()
igsioVideoFrame UncompressedVideoFrame
void GetListOfCaptureVideoFormats(std::vector< std::wstring > &videoModes, unsigned int deviceId)
ChannelContainer OutputChannels
virtual void SetRequestedStreamIndex(unsigned int streamIndex)
virtual void SetRequestedFrameSize(const FrameSizeType &frameSize)
PlusStatus SetNumberOfScalarComponents(unsigned int numberOfScalarComponents)
virtual PlusStatus InternalStopRecording()
#define TRUE
Definition: ATC3DGm.h:219
unsigned char BYTE
Definition: ATC3DGm.h:449
PlusStatus AddFrame(unsigned char *bufferData, DWORD bufferSize)
FrameSizeType GetInputFrameSize() const
Microsoft media foundation video digitizer.
virtual PlusStatus InternalDisconnect()
MmfVideoSourceReader * MmfSourceReader
virtual int GetNumberOfItems()
vtkSmartPointer< vtkPlusCameraControlParameters > CameraControlParameters
void GetListOfCaptureDevices(std::vector< std::wstring > &deviceNames)
bool CorrectlyConfigured
Interface to a 3D positioning tool, video source, or generalized data stream.