Following on from my previous posts;
- Kinect for Windows V2 SDK- Jumping In…
- Kinect for Windows V2 SDK- Hello (Color) World
- Kinect for Windows V2 SDK- Hello (Skeletal) World for the Console Developer 😉
- Kinect for Windows V2 SDK- Hello (Skeletal) World for the WPF Developer
- Kinect for Windows V2 SDK- Hello (Skeletal) World for the 3D WPF Developer
and, at the risk of repeating myself, I’m not trying to do anything “new” here and the samples in the SDK have much more than this functionality but I’m experimenting for myself and writing that up here as I go along. You can get the official view from the Channel 9 videos;
What I wanted to do was bring my code back together such that I was able to combine my 2D Canvas based code and my 3D Viewport code into one app and have a simple switch for 2D/3D mode.
You can see this running in the video below where I’m running both 2D and 3D mode side-by-side. I hadn’t done this before but it shows one of the features of the V2 SDK in that multiple applications can be pulling data from the Kinect V2 sensor at the same time which is an interesting one.
If you watch the video, you might notice that there’s a performance lag. That seemed to be more related to me trying to record the screen while running these 2 apps. When I took the screen recording away, things ran a lot more smoothly on my long-suffering laptop 
The code that I ended up with here was very much like what I’d had in previous posts with the XAML definition for the UI changing to have both a Canvas and a HelixViewport3D on it at the same time;
<Window x:Class="WpfApplication6.MainWindow"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
Title="MainWindow"
Height="350"
Width="525"
xmlns:helix="http://helixtoolkit.codeplex.com">
<Grid>
<Canvas Background="Silver"
x:Name="canvasBody" />
<helix:HelixViewport3D x:Name="viewport3D" HorizontalAlignment="Stretch"
VerticalAlignment="Stretch"
Background="White"
Visibility="Collapsed">
<helix:HelixViewport3D.DefaultCamera>
<PerspectiveCamera Position="0,0,5"
LookDirection="0,0,-1"
UpDirection="0,1,0" />
</helix:HelixViewport3D.DefaultCamera>
<helix:SunLight />
<ModelVisual3D x:Name="modelVisual" />
</helix:HelixViewport3D>
<StackPanel VerticalAlignment="Bottom"
Orientation="Horizontal"
HorizontalAlignment="Center">
<StackPanel.Resources>
<Style TargetType="Button">
<Setter Property="Margin"
Value="5" />
</Style>
</StackPanel.Resources>
<CheckBox Content="Draw 3D"
Checked="OnDraw3DChanged"
Unchecked="OnDraw3DChanged"
VerticalAlignment="Center"
x:Name="chkDraw3d" />
<Button Content="Get Sensor"
Click="OnGetSensor" />
<Button Content="Open Reader"
Click="OnOpenReader" />
<Button Content="Close Reader"
Click="OnCloseReader" />
<Button Content="Release Sensor"
Click="OnReleaseSensor" />
</StackPanel>
</Grid>
</Window>
and the code behind the UI hardly changing to add a new check box to signify 3D mode (but I don’t support the idea of that changing mid-session);
namespace WpfApplication6
{
using System;
using System.Windows;
public partial class MainWindow : Window
{
KinectControl controller;
public MainWindow()
{
InitializeComponent();
this.draw3D = false;
}
IBodyDrawer DrawerFactory3D()
{
return(new HelixModelVisual3DBodyDrawer(this.modelVisual));
}
IBodyDrawer DrawerFactory2D()
{
return (new CanvasBodyDrawer(this.canvasBody));
}
void ShowHideDrawingSurfaces()
{
this.canvasBody.Visibility = this.draw3D ? Visibility.Collapsed : Visibility.Visible;
this.viewport3D.Visibility = this.draw3D ? Visibility.Visible : Visibility.Collapsed;
}
void OnGetSensor(object sender, RoutedEventArgs e)
{
Func<IBodyDrawer> factory = this.DrawerFactory2D;
if (this.draw3D)
{
factory = this.DrawerFactory3D;
}
this.controller = new KinectControl(factory);
this.controller.GetSensor();
}
void OnOpenReader(object sender, RoutedEventArgs e)
{
this.controller.OpenReader();
}
void OnCloseReader(object sender, RoutedEventArgs e)
{
this.controller.CloseReader();
}
void OnReleaseSensor(object sender, RoutedEventArgs e)
{
this.controller.ReleaseSensor();
}
void OnDraw3DChanged(object sender, RoutedEventArgs e)
{
this.draw3D = (bool)this.chkDraw3d.IsChecked;
this.ShowHideDrawingSurfaces();
}
bool draw3D;
}
}
and, finally, I settled on an interface IBodyDrawer which can abstract the details of drawing 2D and 3D for me;
namespace WpfApplication6
{
using Microsoft.Kinect;
using System.Windows.Media;
interface IBodyDrawer
{
void Init(int bodyIndex, KinectSensor sensor);
void ClearFrame();
void DrawFrame(Body body);
}
}
I chose to pass the sensor through to the Init method here to try and cope with all scenarios where some drawing component might need different pieces of info from the sensor to initialise itself. That drove minor changes to the KinectControl class that I’ve been evolving across these blog posts;
namespace WpfApplication6
{
using Microsoft.Kinect;
using System;
using System.Windows.Media;
class KinectControl
{
public KinectControl(Func<IBodyDrawer> bodyDrawerFactory)
{
this.bodyDrawerFactory = bodyDrawerFactory;
}
public void GetSensor()
{
this.sensor = KinectSensor.GetDefault();
this.sensor.Open();
this.bodyDrawers = new IBodyDrawer[BodyCount];
for (int i = 0; i < this.bodyDrawers.Length; i++)
{
this.bodyDrawers[i] = bodyDrawerFactory();
this.bodyDrawers[i].Init(i, this.sensor);
}
}
public void OpenReader()
{
this.reader = this.sensor.BodyFrameSource.OpenReader();
this.reader.FrameArrived += OnFrameArrived;
}
public void CloseReader()
{
this.reader.FrameArrived -= OnFrameArrived;
this.reader.Dispose();
this.reader = null;
}
void OnFrameArrived(object sender, BodyFrameArrivedEventArgs e)
{
using (var frame = e.FrameReference.AcquireFrame())
{
if ((frame != null) && (frame.BodyCount > 0))
{
if ((this.bodies == null) || (this.bodies.Length != frame.BodyCount))
{
this.bodies = new Body[frame.BodyCount];
}
frame.GetAndRefreshBodyData(this.bodies);
for (int i = 0; i < BodyCount; i++)
{
if (this.bodies[i].IsTracked)
{
this.bodyDrawers[i].DrawFrame(this.bodies[i]);
}
else
{
this.bodyDrawers[i].ClearFrame();
}
}
}
}
}
public void ReleaseSensor()
{
this.sensor.Close();
this.sensor = null;
}
Body[] bodies;
KinectSensor sensor;
BodyFrameReader reader;
IBodyDrawer[] bodyDrawers;
Func<IBodyDrawer> bodyDrawerFactory;
static readonly int BodyCount = 6;
}
}
Having got that in place, I wrote a common base class which implements IBodyDrawer and then provides some bits that can be overridden for my 2 different implementations of a drawer – 2D and 3D. Having taken this approach I then wondered whether this was really the right thing to do in that I think if I was doing it again I’d perhaps not go for a base class with derived implementations but maybe prefer more of a “common processor which accepts a drawing plug-in” model.
Regardless, that base class ended up at;
namespace WpfApplication6
{
using Microsoft.Kinect;
using System.Collections.Generic;
using System.Windows;
using System.Windows.Media;
using System.Windows.Media.Media3D;
abstract class BodyDrawerBase : IBodyDrawer
{
public BodyDrawerBase()
{
this.drawnJointElements = new Dictionary<JointType, DependencyObject>();
this.drawnLineElements = new List<DependencyObject>();
}
protected Brush JointBrush
{
get
{
return(brushes[this.bodyIndex]);
}
}
protected KinectSensor Sensor
{
get;
private set;
}
public virtual void Init(int bodyIndex, KinectSensor sensor)
{
this.bodyIndex = bodyIndex;
this.Sensor = sensor;
}
public void DrawFrame(Body body)
{
this.RemoveLinesFromPreviousFrame();
var jointPositions = this.DrawJoints(body);
this.DrawLines(jointPositions);
}
public void ClearFrame()
{
this.RemoveJointsFromPreviousFrame();
this.RemoveLinesFromPreviousFrame();
}
protected abstract DependencyObject MakeUIElementForJointType(JointType jointType,
bool isTracked);
protected abstract void DrawUIElementCentredAtPosition(
DependencyObject uiElement, Point3D point, bool isTracked);
protected abstract void RemoveUIElementForJoint(DependencyObject element);
protected abstract void RemoveJointConnectingLine(DependencyObject line);
protected abstract DependencyObject DrawConnectingLineBetweenJoints(Point3D point1, Point3D point2);
protected virtual Point3D MapPoint(Point3D point)
{
return (point);
}
void RemoveJointsFromPreviousFrame()
{
foreach (var item in this.drawnJointElements)
{
this.RemoveUIElementForJoint(item.Value);
}
this.drawnJointElements.Clear();
}
void RemoveLinesFromPreviousFrame()
{
foreach (var item in this.drawnLineElements)
{
this.RemoveJointConnectingLine(item);
}
this.drawnLineElements.Clear();
}
protected bool IsJointForDrawing(Joint joint, Point3D p)
{
return (
(joint.TrackingState != TrackingState.NotTracked) &&
(!double.IsInfinity(p.X)) &&
(!double.IsInfinity(p.Y)) &&
(!double.IsInfinity(p.Z)));
}
Dictionary<JointType, Point3D> DrawJoints(Body body)
{
Dictionary<JointType, Point3D> jointPositions = new Dictionary<JointType, Point3D>();
foreach (var item in body.Joints)
{
JointType jointType = item.Key;
Joint joint = item.Value;
Point3D point = joint.Position.ToPoint3D();
point = this.MapPoint(point);
bool draw = IsJointForDrawing(joint, point);
bool isTracked = joint.TrackingState == TrackingState.Tracked;
DependencyObject element = null;
if (draw && !this.drawnJointElements.TryGetValue(jointType, out element))
{
element = MakeUIElementForJointType(jointType, isTracked);
this.drawnJointElements[jointType] = element;
}
if (draw)
{
this.DrawUIElementCentredAtPosition(element, point, isTracked);
jointPositions[jointType] = point;
}
else if (element != null)
{
this.RemoveUIElementForJoint(element);
this.drawnJointElements.Remove(jointType);
}
}
return (jointPositions);
}
void DrawLines(IReadOnlyDictionary<JointType, Point3D> jointPositions)
{
foreach (var jointConnection in jointConnections)
{
// that little data structure either contains a list of joints to work through or
// a start joint and an element count. it's discriminated and shouldn't contain
// both!
jointConnection.ForEachPair(
(j1, j2) =>
{
if (jointPositions.ContainsKey(j1) && jointPositions.ContainsKey(j2))
{
Point3D p1 = jointPositions[j1];
Point3D p2 = jointPositions[j2];
DependencyObject line = DrawConnectingLineBetweenJoints(p1, p2);
this.drawnLineElements.Add(line);
}
}
);
}
}
Dictionary<JointType, DependencyObject> drawnJointElements;
List<DependencyObject> drawnLineElements;
int bodyIndex;
// This is bad really because it depends on the actual enum values for the JointType
// type in the SDK not changing.
// It's easier though than having some massive array of all the connections but that
// would be the right thing to do I think.
static JointConnection[] jointConnections =
{
new JointConnection(JointType.SpineBase, 2),
new JointConnection(JointType.ShoulderLeft, 4),
new JointConnection(JointType.ShoulderRight, 4),
new JointConnection(JointType.HipLeft, 4),
new JointConnection(JointType.HipRight, 4),
new JointConnection(JointType.Neck, 2),
new JointConnection(JointType.SpineMid, JointType.SpineShoulder, JointType.Neck),
new JointConnection(JointType.ShoulderLeft, JointType.SpineShoulder, JointType.ShoulderRight),
new JointConnection(JointType.HipLeft, JointType.SpineBase, JointType.HipRight),
new JointConnection(JointType.HandTipLeft, JointType.HandLeft),
new JointConnection(JointType.HandTipRight, JointType.HandRight),
new JointConnection(JointType.WristLeft, JointType.ThumbLeft),
new JointConnection(JointType.WristRight, JointType.ThumbRight)
};
static protected readonly Brush[] brushes =
{
Brushes.Red,
Brushes.Green,
Brushes.Blue,
Brushes.Yellow,
Brushes.Purple,
Brushes.Orange
};
static protected readonly Brush LineBrush = Brushes.Black;
static protected readonly Brush InferredBrush = Brushes.LightGray;
}
}
and I wrote the 2 derived classes for the 2D and 3D cases;
namespace WpfApplication6
{
using Microsoft.Kinect;
using System.Linq;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Media.Media3D;
using System.Windows.Shapes;
class CanvasBodyDrawer : BodyDrawerBase
{
public CanvasBodyDrawer(Canvas canvas)
{
this.canvas = canvas;
}
public override void Init(int bodyIndex, KinectSensor sensor)
{
base.Init(bodyIndex, sensor);
Int32Rect colourFrameSize = new Int32Rect(
0,
0,
sensor.ColorFrameSource.FrameDescription.Width,
sensor.ColorFrameSource.FrameDescription.Height);
this.canvasCoordMapper = new CanvasCoordMapper(
this.canvas,
this.Sensor.CoordinateMapper,
colourFrameSize);
}
protected override DependencyObject MakeUIElementForJointType(JointType jointType,
bool isTracked)
{
JointType[] leafTypes =
{
JointType.Head,
JointType.FootRight,
JointType.FootLeft
};
bool large = leafTypes.Contains(jointType);
int size = large ? Constants.LargeJointEllipseDiamater : Constants.JointEllipseDiameter;
Ellipse ellipse = new Ellipse()
{
Width = size,
Height = size,
VerticalAlignment = VerticalAlignment.Center,
HorizontalAlignment = HorizontalAlignment.Center
};
this.canvas.Children.Add(ellipse);
return (ellipse);
}
protected override Point3D MapPoint(Point3D point)
{
Point p = this.canvasCoordMapper.MapCameraSpacePoint(point);
point.X = p.X;
point.Y = p.Y;
return (point);
}
protected override void DrawUIElementCentredAtPosition(
DependencyObject uiElement, Point3D point, bool isTracked)
{
Ellipse ellipse = (Ellipse)uiElement;
ellipse.Fill = isTracked ? this.JointBrush : BodyDrawerBase.InferredBrush;
Canvas.SetLeft(ellipse, point.X - (ellipse.Width / 2));
Canvas.SetTop(ellipse, point.Y - (ellipse.Height / 2));
}
protected override DependencyObject DrawConnectingLineBetweenJoints(
Point3D point1, Point3D point2)
{
Line line = new Line()
{
X1 = point1.X,
Y1 = point1.Y,
X2 = point2.X,
Y2 = point2.Y,
Stroke = BodyDrawerBase.LineBrush,
StrokeThickness = Constants.StrokeThickness
};
this.canvas.Children.Add(line);
return (line);
}
protected override void RemoveJointConnectingLine(DependencyObject line)
{
this.RemoveElement((UIElement)line);
}
protected override void RemoveUIElementForJoint(DependencyObject element)
{
this.RemoveElement((UIElement)element);
}
void RemoveElement(UIElement element)
{
this.canvas.Children.Remove(element);
}
Canvas canvas;
CanvasCoordMapper canvasCoordMapper;
static class Constants
{
public static readonly int JointEllipseDiameter = 10;
public static readonly int LargeJointEllipseDiamater = 30;
public static readonly double StrokeThickness = 1.0;
}
}
}
and 3D;
namespace WpfApplication6
{
using HelixToolkit.Wpf;
using Microsoft.Kinect;
using System.Linq;
using System.Windows;
using System.Windows.Media;
using System.Windows.Media.Media3D;
class HelixModelVisual3DBodyDrawer : BodyDrawerBase
{
public HelixModelVisual3DBodyDrawer(ModelVisual3D model3dGroup)
{
this.model3dGroup = model3dGroup;
}
public override void Init(int bodyIndex, KinectSensor sensor)
{
base.Init(bodyIndex, sensor);
this.sphereVisual3d = MakeSphereForBrush(this.JointBrush);
if (inferredSphereVisual3d == null)
{
inferredSphereVisual3d = MakeSphereForBrush(BodyDrawerBase.InferredBrush);
}
}
protected override DependencyObject MakeUIElementForJointType(JointType jointType,
bool isTracked)
{
JointType[] leafTypes =
{
JointType.Head
};
var isLeaf = leafTypes.Contains(jointType);
var modelVisual3d = new ModelVisual3D();
modelVisual3d.Content =
isTracked ? this.sphereVisual3d.Model : inferredSphereVisual3d.Model;
Transform3DGroup transform = new Transform3DGroup();
transform.Children.Add(new ScaleTransform3D()
{
ScaleX = isLeaf ? Constants.LeafScaleSize : Constants.RegularScaleSize,
ScaleY = isLeaf ? Constants.LeafScaleSize : Constants.RegularScaleSize,
ScaleZ = isLeaf ? Constants.LeafScaleSize : Constants.RegularScaleSize
});
transform.Children.Add(new TranslateTransform3D());
modelVisual3d.Transform = transform;
this.model3dGroup.Children.Add(modelVisual3d);
return (modelVisual3d);
}
protected override Point3D MapPoint(Point3D point)
{
point.Z = 1 - point.Z;
return (point);
}
protected override void DrawUIElementCentredAtPosition(
DependencyObject uiElement, Point3D point, bool isTracked)
{
ModelVisual3D modelVisual3d = (ModelVisual3D)uiElement;
modelVisual3d.Content =
isTracked ? this.sphereVisual3d.Model : inferredSphereVisual3d.Model;
Transform3DGroup group = (Transform3DGroup)modelVisual3d.Transform;
TranslateTransform3D translate = (TranslateTransform3D)group.Children[1];
translate.OffsetX = point.X;
translate.OffsetY = point.Y;
translate.OffsetZ = point.Z;
}
protected override DependencyObject DrawConnectingLineBetweenJoints(
Point3D point1, Point3D point2)
{
var tube = new TubeVisual3D();
tube.Diameter = Constants.TubeDiameter;
Point3DCollection points = new Point3DCollection();
points.Add(point1);
points.Add(point2);
tube.Path = points;
tube.Fill = BodyDrawerBase.LineBrush;
this.model3dGroup.Children.Add(tube);
return (tube);
}
protected override void RemoveUIElementForJoint(DependencyObject element)
{
this.RemoveVisual3D((Visual3D)element);
}
protected override void RemoveJointConnectingLine(DependencyObject line)
{
this.RemoveVisual3D((Visual3D)line);
}
void RemoveVisual3D(Visual3D visual3D)
{
this.model3dGroup.Children.Remove(visual3D);
}
static SphereVisual3D MakeSphereForBrush(Brush brush)
{
var sphere = new SphereVisual3D();
sphere.Radius = Constants.SphereRadius;
sphere.Fill = brush;
return(sphere);
}
static class Constants
{
public static readonly double LeafScaleSize = 4.0;
public static readonly double RegularScaleSize = 1.0;
public static readonly double SphereRadius = 0.02;
public static readonly double TubeDiameter = 0.02;
}
SphereVisual3D sphereVisual3d;
ModelVisual3D model3dGroup;
static SphereVisual3D inferredSphereVisual3d;
}
}
The main changes that I made to the previous code in order to get to this point was deciding to represent a point by a Point3D across both the 2D and the 3D drawing code with (naturally) the 2D code just throwing away the Z co-ordinate.
The rest of the code here is pretty much as it was in the previous posts and so I won’t post it here but the whole thing is available for download here.
Mike Taulty 