ADD : Depth and IR Image Stream working

KinectConnection/DepthImageStream.cs

@@ -1,5 +1,11 @@
-using System;
+using Microsoft.Kinect;
+using System;
+using System.ComponentModel;
+using System.Globalization;
+using System.IO;
+using System.Windows;
 using System.Windows.Media;
+using System.Windows.Media.Imaging;
 
 namespace KinectConnection
 {
@@ -8,16 +14,154 @@ namespace KinectConnection
     /// </summary>
     public class DepthImageStream : KinectStream
     {
-        public override ImageSource Source => throw new NotImplementedException();
+        /// <summary>
+        /// Map depth range to byte range
+        /// </summary>
+        private const int MapDepthToByte = 8000 / 256;
+
+        /// <summary>
+        /// Active Kinect sensor
+        /// </summary>
+        private KinectSensor kinectSensor = null;
+
+        /// <summary>
+        /// Reader for depth frames
+        /// </summary>
+        private DepthFrameReader depthFrameReader = null;
+
+        /// <summary>
+        /// Description of the data contained in the depth frame
+        /// </summary>
+        private FrameDescription depthFrameDescription = null;
+
+        /// <summary>
+        /// Bitmap to display
+        /// </summary>
+        private WriteableBitmap depthBitmap = null;
+
+        /// <summary>
+        /// Intermediate storage for frame data converted to color
+        /// </summary>
+        private byte[] depthPixels = null;
+
+        /// <summary>
+        /// Gets the bitmap to display
+        /// </summary>
+        public override ImageSource Source
+        {
+            get
+            {
+                return this.depthBitmap;
+            }
+        }
 
         public override void Start()
         {
-            throw new NotImplementedException();
+            // get FrameDescription from DepthFrameSource
+            this.depthFrameDescription = this.KinectSensor.DepthFrameSource.FrameDescription;
+
+            // allocate space to put the pixels being received and converted
+            this.depthPixels = new byte[this.depthFrameDescription.Width * this.depthFrameDescription.Height];
+
+            // create the bitmap to display
+            this.depthBitmap = new WriteableBitmap(this.depthFrameDescription.Width, this.depthFrameDescription.Height, 96.0, 96.0, PixelFormats.Gray8, null);
+
+            if (this.KinectSensor != null)
+            {
+                // open the reader for the depth frames
+                this.depthFrameReader = this.KinectSensor.DepthFrameSource.OpenReader();
+
+                if (this.depthFrameReader != null)
+                {
+                    // wire handler for frame arrival
+                    this.depthFrameReader.FrameArrived += this.Reader_FrameArrived;
+                }
+            }
         }
 
         public override void Stop()
         {
             throw new NotImplementedException();
         }
+
+        /// <summary>
+        /// Handles the depth frame data arriving from the sensor
+        /// </summary>
+        /// <param name="sender">object sending the event</param>
+        /// <param name="e">event arguments</param>
+        private void Reader_FrameArrived(object sender, DepthFrameArrivedEventArgs e)
+        {
+            bool depthFrameProcessed = false;
+
+            using (DepthFrame depthFrame = e.FrameReference.AcquireFrame())
+            {
+                if (depthFrame != null)
+                {
+                    // the fastest way to process the body index data is to directly access 
+                    // the underlying buffer
+                    using (Microsoft.Kinect.KinectBuffer depthBuffer = depthFrame.LockImageBuffer())
+                    {
+                        // verify data and write the color data to the display bitmap
+                        if (((this.depthFrameDescription.Width * this.depthFrameDescription.Height) == (depthBuffer.Size / this.depthFrameDescription.BytesPerPixel)) &&
+                            (this.depthFrameDescription.Width == this.depthBitmap.PixelWidth) && (this.depthFrameDescription.Height == this.depthBitmap.PixelHeight))
+                        {
+                            // Note: In order to see the full range of depth (including the less reliable far field depth)
+                            // we are setting maxDepth to the extreme potential depth threshold
+                            ushort maxDepth = ushort.MaxValue;
+
+                            // If you wish to filter by reliable depth distance, uncomment the following line:
+                            //// maxDepth = depthFrame.DepthMaxReliableDistance
+
+                            this.ProcessDepthFrameData(depthBuffer.UnderlyingBuffer, depthBuffer.Size, depthFrame.DepthMinReliableDistance, maxDepth);
+                            depthFrameProcessed = true;
+                        }
+                    }
+                }
+            }
+
+            if (depthFrameProcessed)
+            {
+                this.RenderDepthPixels();
+            }
+        }
+
+        /// <summary>
+        /// Directly accesses the underlying image buffer of the DepthFrame to 
+        /// create a displayable bitmap.
+        /// This function requires the /unsafe compiler option as we make use of direct
+        /// access to the native memory pointed to by the depthFrameData pointer.
+        /// </summary>
+        /// <param name="depthFrameData">Pointer to the DepthFrame image data</param>
+        /// <param name="depthFrameDataSize">Size of the DepthFrame image data</param>
+        /// <param name="minDepth">The minimum reliable depth value for the frame</param>
+        /// <param name="maxDepth">The maximum reliable depth value for the frame</param>
+        private unsafe void ProcessDepthFrameData(IntPtr depthFrameData, uint depthFrameDataSize, ushort minDepth, ushort maxDepth)
+        {
+            // depth frame data is a 16 bit value
+            ushort* frameData = (ushort*)depthFrameData;
+
+            // convert depth to a visual representation
+            for (int i = 0; i < (int)(depthFrameDataSize / this.depthFrameDescription.BytesPerPixel); ++i)
+            {
+                // Get the depth for this pixel
+                ushort depth = frameData[i];
+
+                // To convert to a byte, we're mapping the depth value to the byte range.
+                // Values outside the reliable depth range are mapped to 0 (black).
+                this.depthPixels[i] = (byte)(depth >= minDepth && depth <= maxDepth ? (depth / MapDepthToByte) : 0);
+            }
+        }
+
+        /// <summary>
+        /// Renders color pixels into the writeableBitmap.
+        /// </summary>
+        private void RenderDepthPixels()
+        {
+            this.depthBitmap.WritePixels(
+                new Int32Rect(0, 0, this.depthBitmap.PixelWidth, this.depthBitmap.PixelHeight),
+                this.depthPixels,
+                this.depthBitmap.PixelWidth,
+                0);
+        }
     }
 }

KinectConnection/InfraredImageStream.cs

@@ -1,5 +1,11 @@
-using System;
+using Microsoft.Kinect;
+using System;
+using System.ComponentModel;
+using System.Globalization;
+using System.IO;
+using System.Windows;
 using System.Windows.Media;
+using System.Windows.Media.Imaging;
 
 namespace KinectConnection
 {
@@ -8,16 +14,171 @@ namespace KinectConnection
     /// </summary>
     public class InfraredImageStream : KinectStream
     {
-        public override ImageSource Source => throw new NotImplementedException();
+        /// <summary>
+        /// Maximum value (as a float) that can be returned by the InfraredFrame
+        /// </summary>
+        private const float InfraredSourceValueMaximum = (float)ushort.MaxValue;
+
+        /// <summary>
+        /// The value by which the infrared source data will be scaled
+        /// </summary>
+        private const float InfraredSourceScale = 0.75f;
+
+        /// <summary>
+        /// Smallest value to display when the infrared data is normalized
+        /// </summary>
+        private const float InfraredOutputValueMinimum = 0.01f;
+
+        /// <summary>
+        /// Largest value to display when the infrared data is normalized
+        /// </summary>
+        private const float InfraredOutputValueMaximum = 1.0f;
+
+        /// <summary>
+        /// Active Kinect sensor
+        /// </summary>
+        private KinectSensor kinectSensor = null;
+
+        /// <summary>
+        /// Reader for infrared frames
+        /// </summary>
+        private InfraredFrameReader infraredFrameReader = null;
+
+        /// <summary>
+        /// Description (width, height, etc) of the infrared frame data
+        /// </summary>
+        private FrameDescription infraredFrameDescription = null;
+
+        /// <summary>
+        /// Bitmap to display
+        /// </summary>
+        private WriteableBitmap infraredBitmap = null;
+
+        /// <summary>
+        /// Current status text to display
+        /// </summary>
+        private string statusText = null;
+
+        /// <summary>
+        /// Gets the bitmap to display
+        /// </summary>
+        public override ImageSource Source
+        {
+            get
+            {
+                return this.infraredBitmap;
+            }
+        }
+
+        /// <summary>
+        /// Execute shutdown tasks
+        /// </summary>
+        /// <param name="sender">object sending the event</param>
+        /// <param name="e">event arguments</param>
+        private void MainWindow_Closing(object sender, CancelEventArgs e)
+        {
+            if (this.infraredFrameReader != null)
+            {
+                // InfraredFrameReader is IDisposable
+                this.infraredFrameReader.Dispose();
+                this.infraredFrameReader = null;
+            }
+
+            if (this.kinectSensor != null)
+            {
+                this.kinectSensor.Close();
+                this.kinectSensor = null;
+            }
+        }
+
+        public InfraredImageStream()
+        {
+            // get FrameDescription from InfraredFrameSource
+            this.infraredFrameDescription = this.KinectSensor.InfraredFrameSource.FrameDescription;
+        }
 
         public override void Start()
         {
-            throw new NotImplementedException();
+            // create the bitmap to display
+            this.infraredBitmap = new WriteableBitmap(this.infraredFrameDescription.Width, this.infraredFrameDescription.Height, 96.0, 96.0, PixelFormats.Gray32Float, null);
+
+            if (this.KinectSensor != null)
+            {
+                // open the reader for the depth frames
+                this.infraredFrameReader = this.KinectSensor.InfraredFrameSource.OpenReader();
+
+                if (this.infraredFrameReader != null)
+                {
+                    // wire handler for frame arrival
+                    this.infraredFrameReader.FrameArrived += this.Reader_InfraredFrameArrived;
+                }
+            }
         }
 
         public override void Stop()
         {
             throw new NotImplementedException();
         }
+
+        /// <summary>
+        /// Handles the infrared frame data arriving from the sensor
+        /// </summary>
+        /// <param name="sender">object sending the event</param>
+        /// <param name="e">event arguments</param>
+        private void Reader_InfraredFrameArrived(object sender, InfraredFrameArrivedEventArgs e)
+        {
+            // InfraredFrame is IDisposable
+            using (InfraredFrame infraredFrame = e.FrameReference.AcquireFrame())
+            {
+                if (infraredFrame != null)
+                {
+                    // the fastest way to process the infrared frame data is to directly access 
+                    // the underlying buffer
+                    using (Microsoft.Kinect.KinectBuffer infraredBuffer = infraredFrame.LockImageBuffer())
+                    {
+                        // verify data and write the new infrared frame data to the display bitmap
+                        if (((this.infraredFrameDescription.Width * this.infraredFrameDescription.Height) == (infraredBuffer.Size / this.infraredFrameDescription.BytesPerPixel)) &&
+                            (this.infraredFrameDescription.Width == this.infraredBitmap.PixelWidth) && (this.infraredFrameDescription.Height == this.infraredBitmap.PixelHeight))
+                        {
+                            this.ProcessInfraredFrameData(infraredBuffer.UnderlyingBuffer, infraredBuffer.Size);
+                        }
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Directly accesses the underlying image buffer of the InfraredFrame to 
+        /// create a displayable bitmap.
+        /// This function requires the /unsafe compiler option as we make use of direct
+        /// access to the native memory pointed to by the infraredFrameData pointer.
+        /// </summary>
+        /// <param name="infraredFrameData">Pointer to the InfraredFrame image data</param>
+        /// <param name="infraredFrameDataSize">Size of the InfraredFrame image data</param>
+        private unsafe void ProcessInfraredFrameData(IntPtr infraredFrameData, uint infraredFrameDataSize)
+        {
+            // infrared frame data is a 16 bit value
+            ushort* frameData = (ushort*)infraredFrameData;
+
+            // lock the target bitmap
+            this.infraredBitmap.Lock();
+
+            // get the pointer to the bitmap's back buffer
+            float* backBuffer = (float*)this.infraredBitmap.BackBuffer;
+
+            // process the infrared data
+            for (int i = 0; i < (int)(infraredFrameDataSize / this.infraredFrameDescription.BytesPerPixel); ++i)
+            {
+                // since we are displaying the image as a normalized grey scale image, we need to convert from
+                // the ushort data (as provided by the InfraredFrame) to a value from [InfraredOutputValueMinimum, InfraredOutputValueMaximum]
+                backBuffer[i] = Math.Min(InfraredOutputValueMaximum, (((float)frameData[i] / InfraredSourceValueMaximum * InfraredSourceScale) * (1.0f - InfraredOutputValueMinimum)) + InfraredOutputValueMinimum);
+            }
+
+            // mark the entire bitmap as needing to be drawn
+            this.infraredBitmap.AddDirtyRect(new Int32Rect(0, 0, this.infraredBitmap.PixelWidth, this.infraredBitmap.PixelHeight));
+
+            // unlock the bitmap
+            this.infraredBitmap.Unlock();
+        }
     }
 }

KinectConnection/KinectConnection.csproj

@@ -23,6 +23,7 @@
     <DefineConstants>DEBUG;TRACE</DefineConstants>
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
     <DebugType>pdbonly</DebugType>
@@ -31,6 +32,7 @@
     <DefineConstants>TRACE</DefineConstants>
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
   <ItemGroup>
     <Reference Include="CommunityToolkit.Mvvm, Version=8.2.0.0, Culture=neutral, PublicKeyToken=4aff67a105548ee2, processorArchitecture=MSIL">

KinectConnection/KinectStreamsFactory.cs

@@ -21,6 +21,8 @@ namespace KinectConnection
         {
             { KinectStreams.Color, () => new ColorImageStream() },
             { KinectStreams.Body, () => new BodyImageStream() },
+            { KinectStreams.IR, () => new InfraredImageStream() },
+            { KinectStreams.Depth, () => new DepthImageStream() },
             // Other streams ...
         };
         }

KinectSensorStreams/ViewModel/MainWindowVM.cs

@@ -56,7 +56,7 @@ namespace KinectSensorStreams.ViewModel
             // factory
             KinectStreamsFactory = new KinectStreamsFactory(new KinectManager());
             // kinect stream => color stream for now
-            KinectStream = KinectStreamsFactory[KinectStreams.Body];
+            KinectStream = KinectStreamsFactory[KinectStreams.Depth];
 
             StartCommand = new RelayCommand(Start);
             // [Question] : StartCommand ici peut être mieux que BeginInit() dans MainWindow.xaml.cs ?