What would be the best way to have the Hand Tracker acting as a mouse. As in so that I can move it around the screen and with the grab gesture it “clicks” a Unity UI “button”.
I have the Hand Tracker tutorial working up to the point before the Gallery is added and have the grab gesture returning to the console as pressed. I’m not sure where to go from here as the rest of the gallery tutorial seems irrelevant. I just want it to click a button and perform that button’s function.
I was wondering if you found a way to do this cause I’m also having trouble with clicking buttons on the SDK. i basically started the same tutorial and am where you are.
There is a possible solution for Windows. Drag-and-drop the script below to any object on the scene and turn on tracking of hands in NuitrackManager. The cursor will move along the screen and you’ll be able to push any buttons using this cursor.
using UnityEngine;
using System.Runtime.InteropServices;
using System;
public class HandCursor : MonoBehaviour
{
bool active = false;
bool press = false;
private void Start()
{
Application.runInBackground = true;
NuitrackManager.onHandsTrackerUpdate += NuitrackManager_onHandsTrackerUpdate;
//Cursor.visible = false;
}
void OnGUI()
{
if (GUI.Button(new Rect(10, 10, 100, 100), "BUTTON"))
Debug.Log("clicked button");
}
private void OnDestroy()
{
NuitrackManager.onHandsTrackerUpdate -= NuitrackManager_onHandsTrackerUpdate;
}
bool pressed = false;
private void NuitrackManager_onHandsTrackerUpdate(nuitrack.HandTrackerData handTrackerData)
{
active = false;
press = false;
if (handTrackerData != null)
{
nuitrack.UserHands userHands = handTrackerData.GetUserHandsByID(CurrentUserTracker.CurrentUser);
if (userHands != null)
{
if (userHands.RightHand != null)
{
Vector2 curpos = new Vector2(userHands.RightHand.Value.X * Screen.currentResolution.width, userHands.RightHand.Value.Y * Screen.currentResolution.height);
MouseOperations.SetCursorPosition((int)(curpos.x), (int)(curpos.y));
active = true;
press = userHands.RightHand.Value.Click;
if (pressed != press)
{
pressed = press;
if (pressed)
{
MouseOperations.MouseEvent(MouseOperations.MouseEventFlags.LeftUp | MouseOperations.MouseEventFlags.LeftDown);
}
}
}
}
}
}
}
public class MouseOperations
{
[Flags]
public enum MouseEventFlags
{
LeftDown = 0x00000002,
LeftUp = 0x00000004,
MiddleDown = 0x00000020,
MiddleUp = 0x00000040,
Move = 0x00000001,
Absolute = 0x00008000,
RightDown = 0x00000008,
RightUp = 0x00000010
}
[DllImport("user32.dll", EntryPoint = "SetCursorPos")]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool SetCursorPos(int X, int Y);
[DllImport("user32.dll")]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool GetCursorPos(out MousePoint lpMousePoint);
[DllImport("user32.dll")]
private static extern void mouse_event(int dwFlags, int dx, int dy, int dwData, int dwExtraInfo);
public static void SetCursorPosition(int X, int Y)
{
SetCursorPos(X, Y);
}
public static void SetCursorPosition(MousePoint point)
{
SetCursorPos(point.X, point.Y);
}
public static MousePoint GetCursorPosition()
{
MousePoint currentMousePoint;
var gotPoint = GetCursorPos(out currentMousePoint);
if (!gotPoint) { currentMousePoint = new MousePoint(0, 0); }
return currentMousePoint;
}
public static void MouseEvent(MouseEventFlags value)
{
MousePoint position = GetCursorPosition();
mouse_event
((int)value,
position.X,
position.Y,
0,
0)
;
}
[StructLayout(LayoutKind.Sequential)]
public struct MousePoint
{
public int X;
public int Y;
public MousePoint(int x, int y)
{
X = x;
Y = y;
}
}
}
Hands are moving inside a frame, the size of which is defined in nuitrack.config (%NUITRACK_HOME%/nuitrack/data). The bigger this frame is, the lower the sensitivity is.
You can use one of these options:
Manually change the values in nuitrack.config: HandTracker > HandPointerFrame > Width & Height
Add the following method to Nuitrack Manager and call it after nuitrack.Nuitrack.Init();
Is there a way to allow a user to change sensitivity during runtime? Right now, my project allows users to change sensitivity but they must restart the program to allow the changes to take effect. I want to allow users to change sensitivity and be able to test it right away without closing the program.
In order for the changes to take effect without restarting the application, it is enough to change the sensitivity of nuitrack.Nuitrack.SetConfigValue(), and then call StopNuitrack() and StartNuitrack() in NuitrackManager.cs.
Unfortunately, there are no changes to the config parameters at runtime.
Please let me know if you have any other questions.