//------------------------------------------------------------------------------ // // Copyright (c) Microsoft Corporation. All rights reserved. // //------------------------------------------------------------------------------ using System; using System.Collections.Generic; using System.ComponentModel; using Windows.ApplicationModel.Resources; using Windows.Foundation; using Windows.UI; using Windows.UI.Xaml; using Windows.UI.Xaml.Controls; using Windows.UI.Xaml.Media; using Windows.UI.Xaml.Shapes; using WindowsPreview.Kinect; namespace Microsoft.Samples.Kinect.BodyBasics { /// /// Main page for sample /// public sealed partial class MainPage : Page, INotifyPropertyChanged { /// /// Resource loader for string resources /// #if WIN81ORLATER private ResourceLoader resourceLoader = ResourceLoader.GetForCurrentView("Resources"); #else private ResourceLoader resourceLoader = new ResourceLoader("Resources"); #endif /// /// Radius of drawn hand circles /// private const double HighConfidenceHandSize = 40; /// /// Radius of drawn hand circles /// private const double LowConfidenceHandSize = 20; /// /// Thickness of drawn joint lines /// private const double JointThickness = 8.0; /// /// Thickness of seen bone lines /// private const double TrackedBoneThickness = 4.0; /// /// Thickness of inferred joint lines /// private const double InferredBoneThickness = 1.0; /// /// Thickness of clip edge rectangles /// private const double ClipBoundsThickness = 5; /// /// Constant for clamping Z values of camera space points from being negative /// private const float InferredZPositionClamp = 0.1f; /// /// Active Kinect sensor /// private KinectSensor kinectSensor = null; /// /// Coordinate mapper to map one type of point to another /// private CoordinateMapper coordinateMapper = null; /// /// Reader for body frames /// private BodyFrameReader bodyFrameReader = null; /// /// Array for the bodies /// private Body[] bodies = null; /// /// Current status text to display /// private string statusText = null; /// /// Main Canvas that contains all visual objects for all bodies and clipped edges /// private Canvas drawingCanvas; /// /// List of BodyInfo objects for each potential body /// private BodyInfo[] BodyInfos; /// /// List of colors for each body tracked /// private List BodyColors; /// /// Clipped edges rectangles /// private Rectangle LeftClipEdge; private Rectangle RightClipEdge; private Rectangle TopClipEdge; private Rectangle BottomClipEdge; private int BodyCount { set { if (value == 0) { this.BodyInfos = null; return; } // creates instances of BodyInfo objects for potential number of bodies if (this.BodyInfos == null || this.BodyInfos.Length != value) { this.BodyInfos = new BodyInfo[value]; for (int bodyIndex = 0; bodyIndex < this.bodies.Length; bodyIndex++) { this.BodyInfos[bodyIndex] = new BodyInfo(this.BodyColors[bodyIndex]); } } } get { return this.BodyInfos == null ? 0 : this.BodyInfos.Length; } } private float JointSpaceWidth { get; set; } private float JointSpaceHeight { get; set; } /// /// Initializes a new instance of the MainPage class. /// public MainPage() { // one sensor is currently supported this.kinectSensor = KinectSensor.GetDefault(); // get the coordinate mapper this.coordinateMapper = this.kinectSensor.CoordinateMapper; // get the depth (display) extents FrameDescription frameDescription = this.kinectSensor.DepthFrameSource.FrameDescription; // get size of joint space this.JointSpaceWidth = frameDescription.Width; this.JointSpaceHeight = frameDescription.Height; // get total number of bodies from BodyFrameSource this.bodies = new Body[this.kinectSensor.BodyFrameSource.BodyCount]; // open the reader for the body frames this.bodyFrameReader = this.kinectSensor.BodyFrameSource.OpenReader(); // wire handler for frame arrival this.bodyFrameReader.FrameArrived += this.Reader_BodyFrameArrived; // set IsAvailableChanged event notifier this.kinectSensor.IsAvailableChanged += this.Sensor_IsAvailableChanged; // populate body colors, one for each BodyIndex this.BodyColors = new List { Colors.Red, Colors.Orange, Colors.Green, Colors.Blue, Colors.Indigo, Colors.Violet }; // sets total number of possible tracked bodies // create ellipses and lines for drawing bodies this.BodyCount = this.kinectSensor.BodyFrameSource.BodyCount; // Instantiate a new Canvas this.drawingCanvas = new Canvas(); // open the sensor this.kinectSensor.Open(); // set the status text this.StatusText = this.kinectSensor.IsAvailable ? resourceLoader.GetString("RunningStatusText") : resourceLoader.GetString("NoSensorStatusText"); // use the window object as the view model in this simple example this.DataContext = this; // initialize the components (controls) of the window this.InitializeComponent(); // set the clip rectangle to prevent rendering outside the canvas this.drawingCanvas.Clip = new RectangleGeometry(); this.drawingCanvas.Clip.Rect = new Rect(0.0, 0.0, this.DisplayGrid.Width, this.DisplayGrid.Height); // create visual objects for drawing joints, bone lines, and clipped edges this.PopulateVisualObjects(); // add canvas to DisplayGrid this.DisplayGrid.Children.Add(this.drawingCanvas); } /// /// INotifyPropertyChangedPropertyChanged event to allow window controls to bind to changeable data /// public event PropertyChangedEventHandler PropertyChanged; /// /// Gets or sets the current status text to display /// public string StatusText { get { return this.statusText; } set { if (this.statusText != value) { this.statusText = value; // notify any bound elements that the text has changed if (this.PropertyChanged != null) { this.PropertyChanged(this, new PropertyChangedEventArgs("StatusText")); } } } } /// /// Execute shutdown tasks /// /// object sending the event /// event arguments private void MainPage_Unloaded(object sender, RoutedEventArgs e) { if (this.bodyFrameReader != null) { // BodyFrameReder is IDisposable this.bodyFrameReader.Dispose(); this.bodyFrameReader = null; } // Body is IDisposable if (this.bodies != null) { foreach (Body body in this.bodies) { if (body != null) { body.Dispose(); } } } if (this.kinectSensor != null) { this.kinectSensor.Close(); this.kinectSensor = null; } } /// /// Handles the body frame data arriving from the sensor /// /// object sending the event /// event arguments private void Reader_BodyFrameArrived(object sender, BodyFrameArrivedEventArgs e) { bool dataReceived = false; bool hasTrackedBody = false; using (BodyFrame bodyFrame = e.FrameReference.AcquireFrame()) { if (bodyFrame != null) { bodyFrame.GetAndRefreshBodyData(this.bodies); dataReceived = true; } } if (dataReceived) { this.BeginBodiesUpdate(); // iterate through each body for (int bodyIndex = 0; bodyIndex < this.bodies.Length; bodyIndex++) { Body body = this.bodies[bodyIndex]; if (body.IsTracked) { // check if this body clips an edge this.UpdateClippedEdges(body, hasTrackedBody); this.UpdateBody(body, bodyIndex); hasTrackedBody = true; } else { // collapse this body from canvas as it goes out of view this.ClearBody(bodyIndex); } } if (!hasTrackedBody) { // clear clipped edges if no bodies are tracked this.ClearClippedEdges(); } } } /// /// Clear update status of all bodies /// internal void BeginBodiesUpdate() { if (this.BodyInfos != null) { foreach (var bodyInfo in this.BodyInfos) { bodyInfo.Updated = false; } } } /// /// Update body data for each body that is tracked. /// /// body for getting joint info /// index for body we are currently updating internal void UpdateBody(Body body, int bodyIndex) { IReadOnlyDictionary joints = body.Joints; var jointPointsInDepthSpace = new Dictionary(); var bodyInfo = this.BodyInfos[bodyIndex]; CoordinateMapper coordinateMapper = this.kinectSensor.CoordinateMapper; // update all joints foreach (var jointType in body.Joints.Keys) { // sometimes the depth(Z) of an inferred joint may show as negative // clamp down to 0.1f to prevent coordinatemapper from returning (-Infinity, -Infinity) CameraSpacePoint position = body.Joints[jointType].Position; if (position.Z < 0) { position.Z = InferredZPositionClamp; } // map joint position to depth space DepthSpacePoint depthSpacePoint = coordinateMapper.MapCameraPointToDepthSpace(position); jointPointsInDepthSpace[jointType] = new Point(depthSpacePoint.X, depthSpacePoint.Y); // modify the joint's visibility and location this.UpdateJoint(bodyInfo.JointPoints[jointType], joints[jointType], jointPointsInDepthSpace[jointType]); // modify hand ellipse colors based on hand states // modity hand ellipse sizes based on tracking confidences if (jointType == JointType.HandRight) { this.UpdateHand(bodyInfo.HandRightEllipse, body.HandRightState, body.HandRightConfidence, jointPointsInDepthSpace[jointType]); } if (jointType == JointType.HandLeft) { this.UpdateHand(bodyInfo.HandLeftEllipse, body.HandLeftState, body.HandLeftConfidence, jointPointsInDepthSpace[jointType]); } } // update all bones foreach (var bone in bodyInfo.Bones) { this.UpdateBone(bodyInfo.BoneLines[bone], joints[bone.Item1], joints[bone.Item2], jointPointsInDepthSpace[bone.Item1], jointPointsInDepthSpace[bone.Item2]); } } /// /// Collapse the body from the canvas. /// /// private void ClearBody(int bodyIndex) { var bodyInfo = this.BodyInfos[bodyIndex]; // collapse all joint ellipses foreach (var joint in bodyInfo.JointPoints) { joint.Value.Visibility = Windows.UI.Xaml.Visibility.Collapsed; } // collapse all bone lines foreach (var bone in bodyInfo.Bones) { bodyInfo.BoneLines[bone].Visibility = Windows.UI.Xaml.Visibility.Collapsed; } // collapse handstate ellipses bodyInfo.HandLeftEllipse.Visibility = Windows.UI.Xaml.Visibility.Collapsed; bodyInfo.HandRightEllipse.Visibility = Windows.UI.Xaml.Visibility.Collapsed; } /// /// Updates hand state ellipses depending on tracking state and it's confidence. /// /// ellipse representing handstate /// open, closed, or lasso /// confidence of handstate /// location of handjoint private void UpdateHand(Ellipse ellipse, HandState handState, TrackingConfidence trackingConfidence, Point point) { ellipse.Fill = new SolidColorBrush(this.HandStateToColor(handState)); // draw handstate ellipse based on tracking confidence ellipse.Width = ellipse.Height = (trackingConfidence == TrackingConfidence.Low) ? LowConfidenceHandSize : HighConfidenceHandSize; ellipse.Visibility = Windows.UI.Xaml.Visibility.Visible; // don't draw handstate if hand joints are not tracked if (!Double.IsInfinity(point.X) && !Double.IsInfinity(point.Y)) { Canvas.SetLeft(ellipse, point.X - ellipse.Width / 2); Canvas.SetTop(ellipse, point.Y - ellipse.Width / 2); } } /// /// Update a joint. /// /// /// /// private void UpdateJoint(Ellipse ellipse, Joint joint, Point point) { TrackingState trackingState = joint.TrackingState; // only draw if joint is tracked or inferred if (trackingState != TrackingState.NotTracked) { if (trackingState == TrackingState.Tracked) { ellipse.Fill = new SolidColorBrush(Colors.Green); } else { // inferred joints are yellow ellipse.Fill = new SolidColorBrush(Colors.Yellow); } Canvas.SetLeft(ellipse, point.X - JointThickness / 2); Canvas.SetTop(ellipse, point.Y - JointThickness / 2); ellipse.Visibility = Windows.UI.Xaml.Visibility.Visible; } else { ellipse.Visibility = Windows.UI.Xaml.Visibility.Collapsed; } } /// /// Update a bone line. /// /// line representing a bone line /// start joint of bone line /// end joint of bone line /// location of start joint /// location of end joint private void UpdateBone(Line line, Joint startJoint, Joint endJoint, Point startPoint, Point endPoint) { // don't draw if neither joints are tracked if (startJoint.TrackingState == TrackingState.NotTracked || endJoint.TrackingState == TrackingState.NotTracked) { line.Visibility = Windows.UI.Xaml.Visibility.Collapsed; return; } // all lines are inferred thickness unless both joints are tracked line.StrokeThickness = InferredBoneThickness; if (startJoint.TrackingState == TrackingState.Tracked && endJoint.TrackingState == TrackingState.Tracked) { line.StrokeThickness = TrackedBoneThickness; } line.Visibility = Windows.UI.Xaml.Visibility.Visible; line.X1 = startPoint.X; line.Y1 = startPoint.Y; line.X2 = endPoint.X; line.Y2 = endPoint.Y; } /// /// Draws indicators to show which edges are clipping body data. /// /// body to draw clipping information for /// bool to determine if another body is triggering a clipped edge private void UpdateClippedEdges(Body body, bool hasTrackedBody) { // BUG (waiting for confirmation): // Clip dectection works differently for top and right edges compared to left and bottom edges // due to the current joint confidence model. This is an ST issue. // Joints become inferred immediately as they touch the left/bottom edges and clip detection triggers. // Joints squish on the right/top edges and clip detection doesn't trigger until more joints of // the body goes out of view (e.g all hand joints vs only handtip). FrameEdges clippedEdges = body.ClippedEdges; if (clippedEdges.HasFlag(FrameEdges.Left)) { this.LeftClipEdge.Visibility = Windows.UI.Xaml.Visibility.Visible; } else if(!hasTrackedBody) { // don't clear this edge if another body is triggering clipped edge this.LeftClipEdge.Visibility = Windows.UI.Xaml.Visibility.Collapsed; } if (clippedEdges.HasFlag(FrameEdges.Right)) { this.RightClipEdge.Visibility = Windows.UI.Xaml.Visibility.Visible; } else if (!hasTrackedBody) { this.RightClipEdge.Visibility = Windows.UI.Xaml.Visibility.Collapsed; } if (clippedEdges.HasFlag(FrameEdges.Top)) { this.TopClipEdge.Visibility = Windows.UI.Xaml.Visibility.Visible; } else if (!hasTrackedBody) { this.TopClipEdge.Visibility = Windows.UI.Xaml.Visibility.Collapsed; } if (clippedEdges.HasFlag(FrameEdges.Bottom)) { this.BottomClipEdge.Visibility = Windows.UI.Xaml.Visibility.Visible; } else if (!hasTrackedBody) { this.BottomClipEdge.Visibility = Windows.UI.Xaml.Visibility.Collapsed; } } /// /// Clear all clipped edges. /// private void ClearClippedEdges() { this.LeftClipEdge.Visibility = Windows.UI.Xaml.Visibility.Collapsed; this.RightClipEdge.Visibility = Windows.UI.Xaml.Visibility.Collapsed; this.TopClipEdge.Visibility = Windows.UI.Xaml.Visibility.Collapsed; this.BottomClipEdge.Visibility = Windows.UI.Xaml.Visibility.Collapsed; } /// /// Select color of hand state /// /// /// private Color HandStateToColor(HandState handState) { switch (handState) { case HandState.Open: return Colors.Green; case HandState.Closed: return Colors.Red; case HandState.Lasso: return Colors.Blue; } return Colors.Transparent; } /// /// Instantiate new objects for joints, bone lines, and clipped edge rectangles /// private void PopulateVisualObjects() { // create clipped edges and set to collapsed initially this.LeftClipEdge = new Rectangle() { Fill = new SolidColorBrush(Colors.Red), Width = ClipBoundsThickness, Height = this.DisplayGrid.Height, Visibility = Windows.UI.Xaml.Visibility.Collapsed }; this.RightClipEdge = new Rectangle() { Fill = new SolidColorBrush(Colors.Red), Width = ClipBoundsThickness, Height = this.DisplayGrid.Height, Visibility = Windows.UI.Xaml.Visibility.Collapsed }; this.TopClipEdge = new Rectangle() { Fill = new SolidColorBrush(Colors.Red), Width = this.DisplayGrid.Width, Height = ClipBoundsThickness, Visibility = Windows.UI.Xaml.Visibility.Collapsed }; this.BottomClipEdge = new Rectangle() { Fill = new SolidColorBrush(Colors.Red), Width = this.DisplayGrid.Width, Height = ClipBoundsThickness, Visibility = Windows.UI.Xaml.Visibility.Collapsed }; foreach (var bodyInfo in this.BodyInfos) { // add left and right hand ellipses of all bodies to canvas this.drawingCanvas.Children.Add(bodyInfo.HandLeftEllipse); this.drawingCanvas.Children.Add(bodyInfo.HandRightEllipse); // add joint ellipses of all bodies to canvas foreach (var joint in bodyInfo.JointPoints) { this.drawingCanvas.Children.Add(joint.Value); } // add bone lines of all bodies to canvas foreach (var bone in bodyInfo.Bones) { this.drawingCanvas.Children.Add(bodyInfo.BoneLines[bone]); } } // add clipped edges rectanges to main canvas this.drawingCanvas.Children.Add(this.LeftClipEdge); this.drawingCanvas.Children.Add(this.RightClipEdge); this.drawingCanvas.Children.Add(this.TopClipEdge); this.drawingCanvas.Children.Add(this.BottomClipEdge); // position the clipped edges Canvas.SetLeft(this.LeftClipEdge, 0); Canvas.SetTop(this.LeftClipEdge, 0); Canvas.SetLeft(this.RightClipEdge, this.DisplayGrid.Width - ClipBoundsThickness); Canvas.SetTop(this.RightClipEdge, 0); Canvas.SetLeft(this.TopClipEdge, 0); Canvas.SetTop(this.TopClipEdge, 0); Canvas.SetLeft(this.BottomClipEdge, 0); Canvas.SetTop(this.BottomClipEdge, this.DisplayGrid.Height - ClipBoundsThickness); } /// /// BodyInfo class that contains joint ellipses, handstate ellipses, lines for bones between two joints. /// private class BodyInfo { public bool Updated { get; set; } public Color BodyColor { get; set; } // ellipse representing left handstate public Ellipse HandLeftEllipse { get; set; } // ellipse representing right handstate public Ellipse HandRightEllipse { get; set; } // dictionary of all joints in a body public Dictionary JointPoints { get; private set; } // definition of bones public TupleList Bones { get; private set; } // collection of bones associated with the line object public Dictionary, Line> BoneLines { get; private set; } public BodyInfo(Color bodyColor) { this.BodyColor = bodyColor; // create hand state ellipses this.HandLeftEllipse = new Ellipse() { Visibility = Windows.UI.Xaml.Visibility.Collapsed }; this.HandRightEllipse = new Ellipse() { Visibility = Windows.UI.Xaml.Visibility.Collapsed }; // a joint defined as a jointType with a point location in XY space represented by an ellipse this.JointPoints = new Dictionary(); // pre-populate list of joints and set to non-visible initially foreach (JointType jointType in Enum.GetValues(typeof(JointType))) { this.JointPoints.Add(jointType, new Ellipse() { Visibility = Windows.UI.Xaml.Visibility.Collapsed, Fill = new SolidColorBrush(BodyColor), Width = JointThickness, Height = JointThickness }); } // collection of bones this.BoneLines = new Dictionary, Line>(); // a bone defined as a line between two joints this.Bones = new TupleList { // Torso { JointType.Head, JointType.Neck }, { JointType.Neck, JointType.SpineShoulder }, { JointType.SpineShoulder, JointType.SpineMid }, { JointType.SpineMid, JointType.SpineBase }, { JointType.SpineShoulder, JointType.ShoulderRight }, { JointType.SpineShoulder, JointType.ShoulderLeft }, { JointType.SpineBase, JointType.HipRight }, { JointType.SpineBase, JointType.HipLeft }, // Right Arm { JointType.ShoulderRight, JointType.ElbowRight }, { JointType.ElbowRight, JointType.WristRight }, { JointType.WristRight, JointType.HandRight }, { JointType.HandRight, JointType.HandTipRight }, { JointType.WristRight, JointType.ThumbRight }, // Left Arm { JointType.ShoulderLeft, JointType.ElbowLeft }, { JointType.ElbowLeft, JointType.WristLeft }, { JointType.WristLeft, JointType.HandLeft }, { JointType.HandLeft, JointType.HandTipLeft }, { JointType.WristLeft, JointType.ThumbLeft }, // Right Leg { JointType.HipRight, JointType.KneeRight }, { JointType.KneeRight, JointType.AnkleRight }, { JointType.AnkleRight, JointType.FootRight }, // Left Leg { JointType.HipLeft, JointType.KneeLeft }, { JointType.KneeLeft, JointType.AnkleLeft }, { JointType.AnkleLeft, JointType.FootLeft }, }; // pre-populate list of bones that are non-visible initially foreach (var bone in this.Bones) { this.BoneLines.Add(bone, new Line() { Stroke = new SolidColorBrush(BodyColor), Visibility = Visibility.Collapsed }); } } } private class TupleList : List> { public void Add(T1 item, T2 item2) { this.Add(new Tuple(item, item2)); } } /// /// Handles the event which the sensor becomes unavailable (E.g. paused, closed, unplugged). /// /// object sending the event /// event arguments private void Sensor_IsAvailableChanged(object sender, IsAvailableChangedEventArgs e) { // on failure, set the status text if (!this.kinectSensor.IsAvailable) { this.StatusText = resourceLoader.GetString("SensorNotAvailableStatusText"); } else { this.StatusText = resourceLoader.GetString("RunningStatusText"); } } } }