Hi @a.bragin,
I was able to create a reduced version of my program that shows the error; you can find it attached.
The application runs multiple threads as follows:
- A thread runs the loop with the “Nuitrack.WaitUpdate” instruction.
- The GUI thread that runs the “onPaint()” method.
- A temporized thread (created through a System.timers.Timer object) that, each 20 ms, updates the user infos in the bitmap that will be shown in the onPaint() method.
Without the temporized thread the error doesn’t show up, so it seems that this could be the problem. Could this due to the fact that the timer callback code takes more than 20 ms to execute? But even then, I cannot understand why that should lead to an exception on the Nuitrack.WaitUpdate() instruction…
PS: now I am using the 0.36 Nuitrack version.
using System;
using System.Drawing;
using System.Drawing.Imaging;
using System.Windows.Forms;
using System.Runtime.InteropServices;
using System.Threading;
using nuitrack;
using nuitrack.issues;
using System.Threading.Tasks;
using Timer = System.Timers.Timer;
using System.Collections.Generic;
namespace nuitrack
{
public class Program
{
static public void Main()
{
Console.CancelKeyPress += delegate {
Nuitrack.Release();
GC.Collect();
GC.WaitForPendingFinalizers();
};
try
{
Application.Run(new MainForm());
}
catch (Exception exception)
{
Console.WriteLine(exception.ToString());
}
}
}
public class DirectBitmap : IDisposable
{
public Bitmap Bitmap { get; private set; }
public Int32[] Bits { get; private set; }
public bool Disposed { get; private set; }
public int Height { get; private set; }
public int Width { get; private set; }
protected GCHandle BitsHandle { get; private set; }
public DirectBitmap(int width, int height)
{
Width = width;
Height = height;
Bits = new Int32[width * height];
BitsHandle = GCHandle.Alloc(Bits, GCHandleType.Pinned);
Bitmap = new Bitmap(width, height, width * 4, PixelFormat.Format32bppPArgb, BitsHandle.AddrOfPinnedObject());
}
public void SetPixel(int x, int y, Color colour)
{
int index = x + (y * Width);
int col = colour.ToArgb();
Bits[index] = col;
}
public Color GetPixel(int x, int y)
{
int index = x + (y * Width);
int col = Bits[index];
Color result = Color.FromArgb(col);
return result;
}
public void Dispose()
{
if (Disposed)
return;
Disposed = true;
Bitmap.Dispose();
BitsHandle.Free();
}
}
public class MainForm : Form
{
// compact struct representing some information of users
internal struct UserInfo
{
internal UserInfo(Point center, Rectangle bbx, bool isPresent)
{
Center = center;
Bbx = bbx;
IsPresent = isPresent;
}
internal Point Center { get; }
internal Rectangle Bbx { get; }
internal bool IsPresent { get; }
}
private DirectBitmap _bitmap;
private bool _visualizeColorImage = false;
private bool _colorStreamEnabled = false;
private DepthSensor _depthSensor;
private ColorSensor _colorSensor;
private UserTracker _userTracker;
private SkeletonTracker _skeletonTracker;
private GestureRecognizer _gestureRecognizer;
private HandTracker _handTracker;
private DepthFrame _depthFrame;
private SkeletonData _skeletonData;
private HandTrackerData _handTrackerData;
private IssuesData _issuesData;
private UserFrame lastUserFrame;
// In order to do a sampling of events. This is independent from the asynchronous events fired from Nuitrack library.
private Timer timer;
private Dictionary<int, UserInfo> usersInfo = new Dictionary<int, UserInfo>();
private Dictionary<int, Color> userColors = new Dictionary<int, Color>();
private DateTime timeStamp;
private DateTime startProgramTime;
// lock per accesso thread safe a dati condivisi
private readonly object lockBitmap = new object();
private readonly object lockHandTrackerData = new object();
private readonly object lockSkeletonData = new object();
private readonly object lockDepthMap = new object();
private readonly object lockRGB = new object();
private readonly object lockIssueData = new object();
private readonly object lockUserData = new object();
private readonly object lockUserColors = new object();
private readonly object lockUserFrame = new object();
private DateTime t_old;
public MainForm()
{
// Initialize Nuitrack. This should be called before using any Nuitrack module.
// By passing the default arguments we specify that Nuitrack must determine
// the location automatically.
try
{
Nuitrack.Init("");
}
catch (Exception exception)
{
Console.WriteLine("Cannot initialize Nuitrack.");
throw exception;
}
startProgramTime = DateTime.Now;
Nuitrack.SetConfigValue("Realsense2Module.Depth.FPS", Convert.ToString(90));
Console.WriteLine("Nuitrack version: " + Nuitrack.GetVersion());
try
{
// Create and setup all required modules
_depthSensor = DepthSensor.Create();
_colorSensor = ColorSensor.Create();
_userTracker = UserTracker.Create();
_skeletonTracker = SkeletonTracker.Create();
_handTracker = HandTracker.Create();
_gestureRecognizer = GestureRecognizer.Create();
}
catch (Exception exception)
{
Console.WriteLine("Cannot create Nuitrack module.");
throw exception;
}
//_depthSensor.SetMirror(false);
// Add event handlers for all modules
_depthSensor.OnUpdateEvent += onDepthSensorUpdate;
_colorSensor.OnUpdateEvent += onColorSensorUpdate;
_userTracker.OnUpdateEvent += onUserTrackerUpdateAsinc;
_userTracker.OnNewUserEvent += onUserTrackerNewUser;
_userTracker.OnLostUserEvent += onUserTrackerLostUser;
_skeletonTracker.OnSkeletonUpdateEvent += onSkeletonUpdate;
_handTracker.OnUpdateEvent += onHandTrackerUpdate;
_gestureRecognizer.OnNewGesturesEvent += onNewGestures;
// Add an event handler for the IssueUpdate event
Nuitrack.onIssueUpdateEvent += onIssueDataUpdate;
// Create and configure the Bitmap object according to the depth sensor output mode
OutputMode mode = _depthSensor.GetOutputMode();
OutputMode colorMode = _colorSensor.GetOutputMode();
if (mode.XRes < colorMode.XRes)
mode.XRes = colorMode.XRes;
if (mode.YRes < colorMode.YRes)
mode.YRes = colorMode.YRes;
_bitmap = new DirectBitmap(mode.XRes, mode.YRes);
for (int y = 0; y < mode.YRes; ++y)
{
for (int x = 0; x < mode.XRes; ++x)
_bitmap.SetPixel(x, y, Color.FromKnownColor(KnownColor.Aqua));
}
// Set fixed form size
this.MinimumSize = this.MaximumSize = new Size(mode.XRes, mode.YRes);
// Disable unnecessary caption bar buttons
this.MinimizeBox = this.MaximizeBox = false;
// Enable double buffering to prevent flicker
this.DoubleBuffered = true;
// temporized thread that draws users info at regular sample time of 20 ms
// set up timer
int t_tracking = 20;
timer = new Timer(t_tracking);
timer.Elapsed += OnTimeElapsed;
timer.AutoReset = true;
// Run Nuitrack. This starts sensor data processing.
try
{
Nuitrack.Run();
}
catch (Exception exception)
{
Console.WriteLine("Cannot start Nuitrack.");
throw exception;
}
// the main nuitrack query loop is in a separate thread
Task.Run(() =>
{
while (true)
{
DateTime t1 = DateTime.Now;
//Console.WriteLine("time elapsed since last iteration of nuitrack update [us]: " + (t1 - t_old).TotalMilliseconds.ToString());
t_old = t1;
// Update Nuitrack data. Data will be synchronized with skeleton time stamps.
try
{
//Nuitrack.Update(_skeletonTracker);
DateTime start = DateTime.Now;
Nuitrack.WaitUpdate(_skeletonTracker);
//Nuitrack.WaitUpdate(_depthSensor);
DateTime end = DateTime.Now;
Console.WriteLine("Thread ID: " + Thread.CurrentThread.ManagedThreadId + ", Nuitrack.WaitUpdate - elapsed time: "
+ (end - start).TotalMilliseconds);
}
catch (LicenseNotAcquiredException exception)
{
Console.WriteLine("LicenseNotAcquired exception. Exception: ", exception);
throw exception;
}
catch (Exception exception)
{
Console.WriteLine("Nuitrack update failed. Exception: ", exception);
throw exception;
}
catch(System.InvalidOperationException e)
{
Console.WriteLine(e.ToString() + ". Total run time: " + (DateTime.Now - startProgramTime).ToString());
throw e;
}
catch(System.Exception e)
{
Console.WriteLine(e.ToString());
throw e;
}
}
});
timer.Start();
this.Show();
}
private void onUserTrackerUpdateAsinc(UserFrame frame)
{
// we just save the user frame for later (it will be used by the "sampling" thread)
lock (lockUserFrame)
{
lastUserFrame = (nuitrack.UserFrame)frame.Clone();
}
}
// method run every 20 ms from the temporized thread
private void OnTimeElapsed(object sender, System.Timers.ElapsedEventArgs e)
{
DateTime now = DateTime.Now;
Console.WriteLine("thread ID: " + Thread.CurrentThread.ManagedThreadId + ", Last time since OnTimeElapsed: "
+ (now - timeStamp).TotalMilliseconds);
timeStamp = now;
onUserTrackerUpdateTimed();
}
~MainForm()
{
_bitmap.Dispose();
}
protected override void OnFormClosing(FormClosingEventArgs e)
{
// Release Nuitrack and remove all modules
try
{
Nuitrack.onIssueUpdateEvent -= onIssueDataUpdate;
_depthSensor.OnUpdateEvent -= onDepthSensorUpdate;
_colorSensor.OnUpdateEvent -= onColorSensorUpdate;
_userTracker.OnUpdateEvent -= onUserTrackerUpdateAsinc;
_skeletonTracker.OnSkeletonUpdateEvent -= onSkeletonUpdate;
_handTracker.OnUpdateEvent -= onHandTrackerUpdate;
_gestureRecognizer.OnNewGesturesEvent -= onNewGestures;
Nuitrack.Release();
}
catch (Exception exception)
{
Console.WriteLine("Nuitrack release failed.");
throw exception;
}
}
// Switch visualization mode on a mouse click
protected override void OnClick(EventArgs args)
{
base.OnClick(args);
_visualizeColorImage = !_visualizeColorImage;
}
protected override void OnPaint(PaintEventArgs args)
{
base.OnPaint(args);
//Console.WriteLine("Thread ID: " + Thread.CurrentThread.ManagedThreadId + ", OnPaint()" + "time: " + DateTime.Now.Millisecond);
lock (lockBitmap)
{
// Draw a bitmap
args.Graphics.DrawImage(_bitmap.Bitmap, new Point(0, 0));
// Draw skeleton joints
lock (lockSkeletonData)
{
if (_skeletonData != null)
{
const int jointSize = 10;
foreach (var skeleton in _skeletonData.Skeletons)
{
SolidBrush brush = new SolidBrush(Color.FromArgb(255 - 40 * skeleton.ID, 0, 0));
foreach (var joint in skeleton.Joints)
{
args.Graphics.FillEllipse(brush, joint.Proj.X * _bitmap.Width - jointSize / 2,
joint.Proj.Y * _bitmap.Height - jointSize / 2, jointSize, jointSize);
}
}
}
}
lock (lockHandTrackerData)
{
// Draw hand pointers
if (_handTrackerData != null)
{
foreach (var userHands in _handTrackerData.UsersHands)
{
if (userHands.LeftHand != null)
{
HandContent hand = userHands.LeftHand.Value;
int size = hand.Click ? 20 : 30;
Brush brush = new SolidBrush(Color.Aquamarine);
args.Graphics.FillEllipse(brush, hand.X * _bitmap.Width - size / 2, hand.Y * _bitmap.Height - size / 2, size, size);
}
if (userHands.RightHand != null)
{
HandContent hand = userHands.RightHand.Value;
int size = hand.Click ? 20 : 30;
Brush brush = new SolidBrush(Color.DarkBlue);
args.Graphics.FillEllipse(brush, hand.X * _bitmap.Width - size / 2, hand.Y * _bitmap.Height - size / 2, size, size);
}
}
}
}
if (!_visualizeColorImage)
{
// we also draw users' centers and bounding boxes
lock (lockUserData)
{
lock (lockUserColors)
{
foreach (int user_ID in usersInfo.Keys)
{
var user = usersInfo[user_ID];
if (user.IsPresent)
{
#if DEBUG
//Console.WriteLine("onPaint. Thread ID: " + threadID + ", disegno user: " + user_ID);
//Console.WriteLine("info utente: " + user.ToString());
#endif
// retrieve user's center position
float x_center = user.Center.X;
float y_center = user.Center.Y;
Brush brush = new SolidBrush(userColors[user_ID]);
Pen pen = new Pen(brush, 5.0f);
args.Graphics.DrawLine(pen, new Point((int)x_center - 50, (int)y_center), new Point((int)x_center + 50, (int)y_center));
args.Graphics.DrawLine(pen, new Point((int)x_center, (int)y_center - 50), new Point((int)x_center, (int)y_center + 50));
args.Graphics.DrawRectangle(pen, user.Bbx);
}
}
}
}
}
}
// Update Form
this.Invalidate();
}
private void onIssueDataUpdate(IssuesData issuesData)
{
lock (lockIssueData)
{
if (_issuesData != null)
_issuesData.Dispose();
_issuesData = (IssuesData)issuesData.Clone();
}
}
// Event handler for the DepthSensorUpdate event
private void onDepthSensorUpdate(DepthFrame depthFrame)
{
Console.WriteLine("onDepthSensorUpdate. Thread ID: " + Thread.CurrentThread.ManagedThreadId);
lock (lockDepthMap)
{
if (_depthFrame != null)
_depthFrame.Dispose();
_depthFrame = (DepthFrame)depthFrame.Clone();
}
}
// Event handler for the ColorSensorUpdate event
private void onColorSensorUpdate(ColorFrame colorFrame)
{
if (!_visualizeColorImage)
return;
_colorStreamEnabled = true;
lock (lockBitmap)
{
float wStep = (float)_bitmap.Width / colorFrame.Cols;
float hStep = (float)_bitmap.Height / colorFrame.Rows;
float nextVerticalBorder = hStep;
int colorPtr = 0;
int bitmapPtr = 0;
const int elemSizeInBytes = 3;
unsafe
{
byte* data = (byte*)colorFrame.Data.ToPointer();
for (int i = 0; i < _bitmap.Height; ++i)
{
if (i == (int)nextVerticalBorder)
{
colorPtr += colorFrame.Cols * elemSizeInBytes;
nextVerticalBorder += hStep;
}
int offset = 0;
int argb = data[colorPtr]
| (data[colorPtr + 1] << 8)
| (data[colorPtr + 2] << 16)
| (0xFF << 24);
float nextHorizontalBorder = wStep;
for (int j = 0; j < _bitmap.Width; ++j)
{
if (j == (int)nextHorizontalBorder)
{
offset += elemSizeInBytes;
argb = data[colorPtr + offset]
| (data[colorPtr + offset + 1] << 8)
| (data[colorPtr + offset + 2] << 16)
| (0xFF << 24);
nextHorizontalBorder += wStep;
}
_bitmap.Bits[bitmapPtr++] = argb;
}
}
}
}
}
// Event handler for the UserTrackerUpdate event
private void onUserTrackerUpdateTimed()
{
Console.WriteLine("onUserTrackerUpdate. threadID: " + Thread.CurrentThread.ManagedThreadId);
UserFrame _userFrame = null;
if(lastUserFrame == null)
{
return;
}
// we use the last saved UserFrame object
lock (lockUserFrame)
{
_userFrame = (UserFrame)lastUserFrame.Clone();
}
using (_userFrame)
{
if (_visualizeColorImage && _colorStreamEnabled)
return;
lock (lockDepthMap)
{
if (_depthFrame == null)
return;
}
const int MAX_LABELS = 7;
bool[] labelIssueState = new bool[MAX_LABELS];
lock (lockIssueData)
{
for (UInt16 label = 0; label < MAX_LABELS; ++label)
{
labelIssueState[label] = false;
if (_issuesData != null)
{
FrameBorderIssue frameBorderIssue = _issuesData.GetUserIssue<FrameBorderIssue>(label);
labelIssueState[label] = (frameBorderIssue != null);
}
}
}
lock (lockBitmap)
{
lock (lockDepthMap)
{
float wStep = (float)_bitmap.Width / _depthFrame.Cols;
float hStep = (float)_bitmap.Height / _depthFrame.Rows;
float nextVerticalBorder = hStep;
unsafe
{
byte* dataDepth = (byte*)_depthFrame.Data.ToPointer();
byte* dataUser = (byte*)_userFrame.Data.ToPointer();
int dataPtr = 0;
int bitmapPtr = 0;
const int elemSizeInBytes = 2;
for (int i = 0; i < _bitmap.Height; ++i)
{
if (i == (int)nextVerticalBorder)
{
dataPtr += _depthFrame.Cols * elemSizeInBytes;
nextVerticalBorder += hStep;
}
int offset = 0;
int argb = 0;
int label = dataUser[dataPtr] | dataUser[dataPtr + 1] << 8;
int depth = Math.Min(255, (dataDepth[dataPtr] | dataDepth[dataPtr + 1] << 8) / 32);
float nextHorizontalBorder = wStep;
for (int j = 0; j < _bitmap.Width; ++j)
{
if (j == (int)nextHorizontalBorder)
{
offset += elemSizeInBytes;
label = dataUser[dataPtr + offset] | dataUser[dataPtr + offset + 1] << 8;
if (label == 0)
depth = Math.Min(255, (dataDepth[dataPtr + offset] | dataDepth[dataPtr + offset + 1] << 8) / 32);
nextHorizontalBorder += wStep;
}
if (label > 0)
{
int user = label * 40;
if (!labelIssueState[label])
user += 40;
argb = 0 | (user << 8) | (0 << 16) | (0xFF << 24);
}
else
{
argb = depth | (depth << 8) | (depth << 16) | (0xFF << 24);
}
_bitmap.Bits[bitmapPtr++] = argb;
}
}
}
}
}
var _users = _userFrame.Users;
foreach (var user in _users)
{
if (!usersInfo.ContainsKey(user.ID))
{
lock (lockUserData)
{
usersInfo[user.ID] = new UserInfo(new Point(0, 0), new Rectangle(), false);
}
}
else
{
var bbx = user.Box;
lock (lockUserData)
{
usersInfo[user.ID] = new UserInfo(
new Point((int)(user.Proj.X * _bitmap.Width), (int)(user.Proj.Y * _bitmap.Height)),
new Rectangle((int)(bbx.Left * _bitmap.Width), (int)(bbx.Top * _bitmap.Height),
(int)((bbx.Right - bbx.Left) * _bitmap.Width), (int)((bbx.Bottom - bbx.Top) * _bitmap.Height)),
true
);
}
}
}
}
}
// Event handler for the NewUser event
private void onUserTrackerNewUser(int id)
{
Console.WriteLine("New User {0}", id);
// create new color for each new user.
var rnd = new Random();
lock (lockUserColors)
{
if (!userColors.ContainsKey(id))
{
userColors[id] = Color.FromArgb(rnd.Next(256), rnd.Next(256), rnd.Next(256));
}
}
}
// Event handler for the LostUser event
private void onUserTrackerLostUser(int id)
{
Console.WriteLine("Lost User {0}", id);
}
// Event handler for the SkeletonUpdate event
private void onSkeletonUpdate(SkeletonData skeletonData)
{
lock (lockSkeletonData)
{
if (_skeletonData != null)
_skeletonData.Dispose();
_skeletonData = (SkeletonData)skeletonData.Clone();
}
}
// Event handler for the HandTrackerUpdate event
private void onHandTrackerUpdate(HandTrackerData handTrackerData)
{
lock (lockHandTrackerData)
{
if (_handTrackerData != null)
_handTrackerData.Dispose();
_handTrackerData = (HandTrackerData)handTrackerData.Clone();
}
}
// Event handler for the gesture detection event
private void onNewGestures(GestureData gestureData)
{
// Display the information about detected gestures in the console
foreach (var gesture in gestureData.Gestures)
Console.WriteLine("Recognized {0} from user {1}", gesture.Type.ToString(), gesture.UserID);
}
}
}