การวาด Skeleton จาก Kinect ตอนที่ 4: วาดซ้อนทับกับภาพวิดีโอ

ก่อนหน้านี้เราได้ลองวาด Skeleton แบบเต็มตัว และตรวจสอบว่าข้อต่อถูกตรวจจับหรือไม่ ตอนนี้เรามาลองวาด Skeleton ซ้อนทับกับภาพวิดีโอกันดูครับ

หลักการก็ตรงไปตรงมา คือส่วนของ UI ก็จะวาง canvas เป็นพื้นหลัง ทับด้วย Image เพื่อแสดงผลวิดีโอ และ ถูกทับอีกทีด้วย canvas เพื่อวาด skeleton

เมื่อซ้อนกันตามลำดับนี้ ภาพ Skeleton ก็จะอยู่บน ภาพวิดีโอ โดยอัตโนมัติครับ

รูปผลลัพธ์ก็จะคล้ายๆกับแบบนี้ครับ

ที่มา: Kinect for Windows | Human Interface Guidelines v1.8

โค้ด UI

<Window x:Class="SkeletonVideo.MainWindow"
        xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
        xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
        Title="Skeleton and Video" SizeToContent="WidthAndHeight" Loaded="Window_Loaded" Closing="Window_Closing">
    <Canvas Height="480" Width="640">
        <Image Name="kinectVideo" Height="480" Width="640"/>
        <Canvas Name="skeletonCanvas" Height="480" Width="640"/>
    </Canvas>
</Window>

ส่วนของโค้ด ก็จะอ่านทั้งภาพวิดีโอและ skeleton จากนั้นก็วาดลงไปบน UI ที่เกี่ยวข้อง

โค้ด C#

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Data;
using System.Windows.Documents;
using System.Windows.Input;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Navigation;
using System.Windows.Shapes;
using Microsoft.Kinect;

namespace SkeletonVideo
{
    public partial class MainWindow : Window
    {
        KinectSensor myKinect;

        public MainWindow()
        {
            InitializeComponent();
        }

        //======== Window Loaded and Closing ========
        private void Window_Loaded(object sender, RoutedEventArgs e)
        {
            startKinect();
        }

        private void Window_Closing(object sender, System.ComponentModel.CancelEventArgs e)
        {
            stopKinect();
        }
        
        //=============== Start and Stop Kinect =========================
        void startKinect()
        {
            // Check to see if a Kinect is available
            if (KinectSensor.KinectSensors.Count == 0)
            {
                MessageBox.Show("No Kinects detected", "Camera Viewer");
                Application.Current.Shutdown();
                return;
            }

            // Start the Kinect and enable both video and skeleton streams
            try
            {
                // Get the first Kinect on the computer
                myKinect = KinectSensor.KinectSensors[0];
                myKinect.ColorStream.Enable();
                myKinect.SkeletonStream.Enable();
                myKinect.Start();
            }
            catch
            {
                MessageBox.Show("Kinect initialise failed", "Camera Viewer");
                Application.Current.Shutdown();
                return;
            }

            // connect a handler to the event that fires when new frames are available
            myKinect.ColorFrameReady += new EventHandler<ColorImageFrameReadyEventArgs>(myKinect_ColorFrameReady);
            myKinect.SkeletonFrameReady += new EventHandler<SkeletonFrameReadyEventArgs>(myKinect_SkeletonFrameReady);
        }

        void stopKinect()
        {
            if (myKinect != null)
                myKinect.Stop();
        }

        //================================== video =============================
        byte[] colorData = null;
        WriteableBitmap colorImageBitmap = null;

        private void myKinect_ColorFrameReady(object sender, ColorImageFrameReadyEventArgs e)
        {
            using (ColorImageFrame colorFrame = e.OpenColorImageFrame())
            {
                if (colorFrame == null) return;

                if (colorData == null)
                    colorData = new byte[colorFrame.PixelDataLength];

                colorFrame.CopyPixelDataTo(colorData);

                if (colorImageBitmap == null)
                {
                    colorImageBitmap = new WriteableBitmap(
                        colorFrame.Width,
                        colorFrame.Height,
                        96,  // DpiX
                        96,  // DpiY
                        PixelFormats.Bgr32,
                        null);
                }

                colorImageBitmap.WritePixels(
                    new Int32Rect(0, 0, colorFrame.Width, colorFrame.Height),
                    colorData, // video data
                    colorFrame.Width * colorFrame.BytesPerPixel, // stride,
                    0   // offset into the array - start at 0
                    );

                kinectVideo.Source = colorImageBitmap;
            }
        }

        //================================== skeleton ===========================
        //drawing color
        Brush brushRed = new SolidColorBrush(Colors.Red);
        Brush brushGreen = new SolidColorBrush(Colors.Green);

        //method to draw a line between joints
        void drawLine(Joint j1, Joint j2)
        {
            //if any joint is not tracked
            if (j1.TrackingState == JointTrackingState.NotTracked || j2.TrackingState == JointTrackingState.NotTracked)
            {
                //do not draw
                return;
            }

            //line properties
            Line bone = new Line();
            //line thickness            
            bone.StrokeThickness = 5;

            //if any joint is inferred
            if (j1.TrackingState == JointTrackingState.Inferred || j2.TrackingState == JointTrackingState.Inferred)
            {
                //set color to red
                bone.Stroke = brushRed;
            }
            else if (j1.TrackingState == JointTrackingState.Tracked && j2.TrackingState == JointTrackingState.Tracked)
            {
                //both joints are tracked, set color to green
                bone.Stroke = brushGreen;
            }

            //map joint position to display location
            //starting point
            ColorImagePoint j1p = myKinect.CoordinateMapper.MapSkeletonPointToColorPoint(j1.Position, ColorImageFormat.RgbResolution640x480Fps30);
            bone.X1 = j1p.X;
            bone.Y1 = j1p.Y;
            //ending point
            ColorImagePoint j2p = myKinect.CoordinateMapper.MapSkeletonPointToColorPoint(j2.Position, ColorImageFormat.RgbResolution640x480Fps30);
            bone.X2 = j2p.X;
            bone.Y2 = j2p.Y;

            //add line to canvas
            skeletonCanvas.Children.Add(bone);
        }

        private void myKinect_SkeletonFrameReady(object sender, SkeletonFrameReadyEventArgs e)
        {
            // Remove the old skeleton
            skeletonCanvas.Children.Clear();

            Skeleton[] skeletons = null;

            using (SkeletonFrame frame = e.OpenSkeletonFrame())
            {
                if (frame != null)
                {
                    skeletons = new Skeleton[frame.SkeletonArrayLength];
                    frame.CopySkeletonDataTo(skeletons);
                }
            }

            if (skeletons == null)
                return;

            foreach (Skeleton skeleton in skeletons)
            {
                if (skeleton.TrackingState == SkeletonTrackingState.Tracked)
                {
                    //draw head to neck
                    drawLine(skeleton.Joints[JointType.Head], skeleton.Joints[JointType.ShoulderCenter]);
                    //draw neck to spine
                    drawLine(skeleton.Joints[JointType.ShoulderCenter], skeleton.Joints[JointType.Spine]);
                    //draw spine to hip center
                    drawLine(skeleton.Joints[JointType.Spine], skeleton.Joints[JointType.HipCenter]);

                    //draw left arm
                    drawLine(skeleton.Joints[JointType.ShoulderCenter], skeleton.Joints[JointType.ShoulderLeft]);
                    drawLine(skeleton.Joints[JointType.ShoulderLeft], skeleton.Joints[JointType.ElbowLeft]);
                    drawLine(skeleton.Joints[JointType.ElbowLeft], skeleton.Joints[JointType.WristLeft]);
                    drawLine(skeleton.Joints[JointType.WristLeft], skeleton.Joints[JointType.HandLeft]);

                    //draw right arm
                    drawLine(skeleton.Joints[JointType.ShoulderCenter], skeleton.Joints[JointType.ShoulderRight]);
                    drawLine(skeleton.Joints[JointType.ShoulderRight], skeleton.Joints[JointType.ElbowRight]);
                    drawLine(skeleton.Joints[JointType.ElbowRight], skeleton.Joints[JointType.WristRight]);
                    drawLine(skeleton.Joints[JointType.WristRight], skeleton.Joints[JointType.HandRight]);

                    //draw left leg
                    drawLine(skeleton.Joints[JointType.HipCenter], skeleton.Joints[JointType.HipLeft]);
                    drawLine(skeleton.Joints[JointType.HipLeft], skeleton.Joints[JointType.KneeLeft]);
                    drawLine(skeleton.Joints[JointType.KneeLeft], skeleton.Joints[JointType.AnkleLeft]);
                    drawLine(skeleton.Joints[JointType.AnkleLeft], skeleton.Joints[JointType.FootLeft]);

                    //draw right leg
                    drawLine(skeleton.Joints[JointType.HipCenter], skeleton.Joints[JointType.HipRight]);
                    drawLine(skeleton.Joints[JointType.HipRight], skeleton.Joints[JointType.KneeRight]);
                    drawLine(skeleton.Joints[JointType.KneeRight], skeleton.Joints[JointType.AnkleRight]);
                    drawLine(skeleton.Joints[JointType.AnkleRight], skeleton.Joints[JointType.FootRight]);
                }
            }//end foreach
        }

    }//end class
}