Nuitrack Skeletonization Type CNN_HPE Crashes

Hi.

I am running Nuitrack version 0.36.9, on a Ubuntu system (v20.04.5) with sensor Intel RealSense D435.
And when i try running tracking with Skeletonization Type CNN_HPE i.e AI Skeleton Tracking it crashes with this output:
Custom segmentation (box cutter): ON
bad err=45 in Xbyak::Error

It does not crash if i use RegressionSkeletonization type.
It has work before, on other computers with the same operating system, the same sensor and the same nuitrack version.

Any help would be appreciated

Hi @PhilipOryx,

–°ould you please send us your nuitrack.config file (and any setConfig calls from your code, if there are any)?
Thanks.

Hi @vadim.bazhin,

The only value we set from code is:

tdv::nuitrack::Nuitrack::setConfigValue("Reidentification.ToUse", "true");

here is the nuitrack.config file we use for AI Skeleton Tracking :

{
    "Network": {
        "ProxyUrl": ""
    },
    "KinectV2": {
        "Color": {
            "RawHeight": 1080,
            "RawWidth": 1920
        },
        "Depth": {
            "RawHeight": 424,
            "RawWidth": 512
        }
    },
    "GestureRecognition": {
        "WavingEnabled": true
    },
    "StructureDepthProvider": {
        "ColorCameraSettings": {
            "CameraPosition": "back"
        },
        "SensorSettings": {
            "DepthHeight": 480,
            "DepthWidth": 640
        }
    },
    "MovingSensor": {
        "VocabFilePath": "MovingSensor/orb_vocab/orb_vocab.dbow2",
        "ToUse": false
    },
    "Skeletonization": {
        "AutoTracking": true,
        "FeedbackThreshold": 0.1,
        "Type": "CNN_HPE",
        "CnnHPE": {
            "Filter": {
                "ThreshHold": {
                    "MaxDisplacement": 500,
                    "SmoothCameraJointVelocity": false,
                    "MinDisplacement": 20
                }
            }
        },
        "ActiveUsers": 2,
        "MaxDistance": 2000
    },
    "DepthProviderAdapter": {
        "Library": ""
    },
    "StateDumper": {
        "Directory": "",
        "ToUse": false,
        "LengthInSeconds": 15
    },
    "CPU": {
        "MaxLoad": -1.0
    },
    "HandTracker": {
        "WristGrabDetector": {
            "TrainedClassificator": "handtracker/svm_grab.xml"
        },
        "HandMode": "grab3d",
        "HandPointerFrame": {
            "DistanceBetweenFrames": 150.0,
            "Width": 450.0,
            "Height": 200.0
        },
        "GrabDetector": "WristGrabDetector",
        "SkeletonSupport": true,
        "SkeletonForearmTracker": {
            "BottomPlane": {
                "Usage": false,
                "centralPoint": [
                    0,
                    -240,
                    677
                ],
                "maxY": 0,
                "normal": [
                    0,
                    -47,
                    100
                ]
            }
        },
        "CoordsMode": "abs",
        "CNNGrabDetector": {
            "InputShape": [
                96,
                96,
                1
            ],
            "PathToModel": "handtracker/testout12.pb"
        }
    },
    "CnnDetectionModule": {
        "meanPixel": 128.0,
        "ResizeMode": "pad",
        "ToUse": false,
        "InputSize": {
            "width": 640,
            "height": 384
        },
        "per_sample_mode": true,
        "MinConfidence": 0.2,
        "Classes": [
            "bottle",
            "cigarette",
            "phone",
            "laptop",
            "bag",
            "book"
        ],
        "stdPixel": 33.0
    },
    "RGBMarkEstimator": {
        "MarkerWidth": 105,
        "MarkerHeight": 62,
        "Precision": 0.5,
        "DebugLevel": 1
    },
    "FrameDumpDepthProvider": {
        "Directory": "",
        "ToUse": false
    },
    "AstraProPerseeDepthProvider": {
        "POSIX": {
            "PID": "0x0501",
            "VID": "0x2bc5"
        },
        "CameraID": 0
    },
    "AzureKinectDepthProviderModule": {},
    "DepthProvider": {
        "RotateAngle": "0",
        "RGB": "Auto",
        "Depth2ColorRegistration": true,
        "Mirror": false
    },
    "CnnDetectionModel": {
        "keypointsClasses": [
            "human"
        ],
        "keypointsNames": [
            "head",
            "left_shoulder",
            "left_elbow",
            "left_wrist",
            "right_shoulder",
            "right_elbow",
            "right_wrist",
            "left_hip",
            "right_hip",
            "left_knee",
            "right_knee",
            "left_ankle",
            "right_ankle"
        ],
        "InterParallelismThreads": -1,
        "IntraParallelismThreads": -1,
        "PathToModel": "CnnDetector/TFCnnDetector/model_description.bin"
    },
    "Settings": {
        "Multiplexing": true,
        "MapResolutionX": 480,
        "ServerMAC": "",
        "Autocalibration": true,
        "ServerName": "",
        "hfov": 1.3176,
        "DepthMapSmoothing": false,
        "AdvancedConnect": true,
        "LatencyEvents": false,
        "MapResolutionY": 640,
        "Mode": "WF",
        "AdvancedStreaming": true,
        "SkeletonSmoothing": false,
        "PredictionInterval": 0,
        "IPAddress": "192.168.0.16"
    },
    "Segmentation": {
        "BoxCutter":{
            "x":"0",
            "y":"0",
            "z":"0",
            "width":"800",
            "height":"5000",
            "depth":"2500",
            "alpha":"0",
            "beta":"-1.5"
        },
        "skipFrames": 0,
        "ToUse": true,
        "FloorMaxError": 100,
        "Type": "experimental",
        "ZPS": 575.816,
        "median": 0,
        "RotationOY": 0,
        "minHumanTTL": 0,
        "Pixelization": 0,
        "FloorDetector": "AdaptiveFloorDetector",
        "MOTION_THRESHOLD": 1,
        "Background": {
            "BackgroundMode": "dynamic"
        },
        "MAX_DISTANCE": 7000,
        "RotationOX": 0,
        "TOFHacks": false,
        "ScaleFactor": 1.0,
        "MIN_CHECK_DISTANCE": 500
    },
    "Filter": {
        "DummyFilter": {},
        "Kinematic": {
            "SkeletonFile": "skeleton.xml",
            "LimitsBone": "skeleton_limits.xml",
            "CfgFile": "kinematic_filter.xml",
            "Loader": "XMLSkeletonLoader"
        },
        "CombinationFilter": [
            {
                "DummyFilter": {}
            },
            {
                "Kinematic": {
                    "SkeletonFile": "skeleton.xml",
                    "LimitsBone": "skeleton_limits.xml",
                    "CfgFile": "kinematic_filter.xml",
                    "Loader": "XMLSkeletonLoader"
                }
            }
        ],
        "AlphaBeta": {
            "Alpha": 0.5,
            "Beta": 0.5
        },
        "Median": {
            "HistorySize": 5
        },
        "FABRIK": {
            "SkeletonFile": "FABRIKSkeleton.xml"
        },
        "ThreshHold": {
            "MaxDisplacement": 10000,
            "MinDisplacement": 20
        }
    },
    "DynamicModules": {
        "Path": "../middleware/",
        "Modules": [
            "NuitrackModule"
        ],
        "WModules": [
            "NuitrackWirelessModule"
        ]
    },
    "DaemonSettings": {
        "NewHumanEndpoint": "localhost:8081/onNewHuman",
        "LostHumanEndpoint": "localhost:8081/onLostHuman"
    },
    "Faces": {
        "ToUse": false
    },
    "Realsense2Module": {
        "DeviceHardwareReset": false,
        "Depth": {
            "ProcessWidth": 640,
            "ProcessHeight": 480,
            "ProcessMaxDepth": 2100
        },
        "FileRecord": "",
        "RGB": {
            "ProcessWidth": 640,
            "ProcessHeight": 480
        },
        "AlignTo": "none"
    },
    "OrbbecSDKDepthProviderModule": {
        "Depth": {
            "ProcessWidth": 640,
            "ProcessHeight": 480
        },
        "SoftwareFilter": false,
        "RGB": {
            "ProcessWidth": 640,
            "ProcessHeight": 480
        },
        "Exposure": {
            "ToUse": false
        }
    },
    "StaticModuleLoader": [],
    "OpenNIModule": {
        "FileRecord": ""
    },
    "Orientation": {
        "GlobalUp": {},
        "TPose": {}
    },
    "RegressionSkeletonization": {
        "Orientation": {
            "Dummy": {}
        },
        "CNN_RGB": {},
        "Filter": {
            "CombinationFilter": [
                {
                    "ThreshHold": {
                        "MaxDisplacement": 200,
                        "SmoothCameraJointVelocity": false,
                        "MinDisplacement": 15
                    }
                },
                {
                    "FABRIK": {
                        "PalmsAlignment": true,
                        "FixedHeight": 1700,
                        "HeightAlgorithm": "old",
                        "SkeletonFile": "FABRIKSkeleton.xml"
                    }
                },
                {
                    "Confidence": {
                        "ConfidenceThreshold": 0.5,
                        "CorrectHandsConfidence": true
                    }
                }
            ]
        },
        "CNN_RGB_Depth": {
            "Threshold": 0.3
        },
        "PoseOptimization": {
            "OptimizationParams": {
                "RegressionClouds": {
                    "EMOptimizer": {
                        "Tolerance": 0.005,
                        "Iterations": 1
                    },
                    "Solver": {
                        "max_num_iteration": 20,
                        "function_tolerance": 0.008,
                        "gradient_tolerance": 1e-06,
                        "parameter_tolerance": 0.0008
                    }
                }
            },
            "Terms": {
                "DepthEnergyFunction": {
                    "Enable": false,
                    "Iterations": {
                        "To": 2,
                        "From": 1
                    },
                    "Weight": 1000
                },
                "IntersectionEnergyFunction": {
                    "Enable": false,
                    "Iterations": {
                        "To": 4,
                        "From": 2
                    },
                    "Weight": 5000
                },
                "WeightsEnergyFunction": {
                    "Enable": false,
                    "Iterations": {
                        "To": 4,
                        "From": 0
                    },
                    "Weight": 1.0
                },
                "FieldEnergyFunction": {
                    "Enable": false,
                    "Iterations": {
                        "To": 4,
                        "From": 2
                    },
                    "Weight": 1
                },
                "DepthConsistencyEnergyFunction": {
                    "Enable": false,
                    "Iterations": {
                        "To": 4,
                        "From": 2
                    },
                    "Weight": 5000
                },
                "EnergyFunction": {
                    "Enable": true,
                    "Iterations": {
                        "To": 1,
                        "From": 0
                    },
                    "Weight": 1.0
                }
            },
            "SkeletonFile": "poseoptimization/skeleton.xml"
        },
        "Type": "Regression",
        "Regression": {
            "Stat": "trees_stat",
            "Scale": 0.5,
            "MaxHeightScale": 1.4,
            "UseGPU": false,
            "MinHeightScale": 0.85,
            "Tree": "trees.bin",
            "StatBase": "regStat",
            "ClassesCount": 32,
            "Bodyparts": "bodyparts.xml",
            "TreeNormCoef": 4000,
            "MeanHeight": 1700,
            "NormCoef": 4000,
            "PredictType": "base"
        }
    },
    "LicenseFile": "../license.json",
    "Reidentification": {
        "ToUse": false
    },
    "DefaultModules": {
        "DepthProvider": "DepthProviderManager"
    }
}

Hi @PhilipOryx,

Please try to use this config, does it solve your problem?

{
    "Network": {
        "ProxyUrl": ""
    }, 
    "KinectV2": {
        "Color": {
            "RawHeight": 1080, 
            "RawWidth": 1920
        }, 
        "Depth": {
            "RawHeight": 424, 
            "RawWidth": 512
        }
    }, 
    "GestureRecognition": {
        "WavingEnabled": true
    }, 
    "StructureDepthProvider": {
        "ColorCameraSettings": {
            "CameraPosition": "back"
        }, 
        "SensorSettings": {
            "DepthHeight": 480, 
            "DepthWidth": 640
        }
    }, 
    "MovingSensor": {
        "VocabFilePath": "MovingSensor/orb_vocab/orb_vocab.dbow2", 
        "ToUse": false
    }, 
    "Skeletonization": {
        "AutoTracking": true, 
        "FeedbackThreshold": 0.1, 
        "Type": "CNN_HPE", 
        "CnnHPE": {
            "Filter": {
                "ThreshHold": {
                    "MaxDisplacement": 200, 
                    "SmoothCameraJointVelocity": false, 
                    "MinDisplacement": 30
                }
            }
        }, 
        "ActiveUsers": 2, 
        "Debug": false, 
        "MaxDistance": 2000
    }, 
    "DepthProviderAdapter": {
        "Library": ""
    }, 
    "StateDumper": {
        "Directory": "", 
        "ToUse": false, 
        "LengthInSeconds": 15
    }, 
    "CPU": {
        "MaxLoad": -1.0
    }, 
    "HandTracker": {
        "WristGrabDetector": {
            "TrainedClassificator": "handtracker/svm_grab.xml"
        }, 
        "HandMode": "grab3d", 
        "HandPointerFrame": {
            "DistanceBetweenFrames": 150.0, 
            "Width": 450.0, 
            "Height": 200.0
        }, 
        "GrabDetector": "WristGrabDetector", 
        "SkeletonSupport": true, 
        "SkeletonForearmTracker": {
            "BottomPlane": {
                "Usage": false, 
                "centralPoint": [
                    0, 
                    -240, 
                    677
                ], 
                "maxY": 0, 
                "normal": [
                    0, 
                    -47, 
                    100
                ]
            }
        }, 
        "CoordsMode": "abs", 
        "CNNGrabDetector": {
            "InputShape": [
                96, 
                96, 
                1
            ], 
            "PathToModel": "handtracker/testout12.pb"
        }
    }, 
    "CnnDetectionModule": {
        "meanPixel": 128.0, 
        "ResizeMode": "pad", 
        "ToUse": false, 
        "InputSize": {
            "width": 640, 
            "height": 384
        }, 
        "per_sample_mode": true, 
        "MinConfidence": 0.2, 
        "Classes": [
            "bottle", 
            "cigarette", 
            "phone", 
            "laptop", 
            "bag", 
            "book"
        ], 
        "stdPixel": 33.0
    }, 
    "RGBMarkEstimator": {
        "MarkerWidth": 105, 
        "MarkerHeight": 62, 
        "Precision": 0.5, 
        "DebugLevel": 1
    }, 
    "FrameDumpDepthProvider": {
        "Directory": "", 
        "ToUse": false
    }, 
    "AstraProPerseeDepthProvider": {
        "POSIX": {
            "PID": "0x0501", 
            "VID": "0x2bc5"
        }, 
        "CameraID": 0
    }, 
    "AzureKinectDepthProviderModule": {}, 
    "DepthProvider": {
        "RotateAngle": "0", 
        "RGB": "Auto", 
        "Depth2ColorRegistration": true, 
        "Mirror": false
    }, 
    "CnnDetectionModel": {
        "keypointsClasses": [
            "human"
        ], 
        "keypointsNames": [
            "head", 
            "left_shoulder", 
            "left_elbow", 
            "left_wrist", 
            "right_shoulder", 
            "right_elbow", 
            "right_wrist", 
            "left_hip", 
            "right_hip", 
            "left_knee", 
            "right_knee", 
            "left_ankle", 
            "right_ankle"
        ], 
        "InterParallelismThreads": -1, 
        "IntraParallelismThreads": -1, 
        "PathToModel": "CnnDetector/TFCnnDetector/model_description.bin"
    }, 
    "Settings": {
        "Multiplexing": true, 
        "MapResolutionX": 480, 
        "ServerMAC": "", 
        "Autocalibration": true, 
        "ServerName": "", 
        "hfov": 1.3176, 
        "DepthMapSmoothing": false, 
        "AdvancedConnect": true, 
        "LatencyEvents": false, 
        "MapResolutionY": 640, 
        "Mode": "WF", 
        "AdvancedStreaming": true, 
        "SkeletonSmoothing": false, 
        "PredictionInterval": 0, 
        "IPAddress": "192.168.0.16"
    }, 
    "Segmentation": {
        "skipFrames": 0, 
        "ToUse": true, 
        "FloorMaxError": 100, 
        "Type": "experimental", 
        "ZPS": 575.816, 
        "median": 0, 
        "RotationOY": 0, 
        "minHumanTTL": 0, 
        "Pixelization": 0, 
        "FloorDetector": "AdaptiveFloorDetector", 
        "MOTION_THRESHOLD": 1, 
        "Background": {
            "BackgroundMode": "dynamic"
        }, 
        "MAX_DISTANCE": 7000, 
        "RotationOX": 0, 
        "TOFHacks": false, 
        "ScaleFactor": 1.0, 
        "MIN_CHECK_DISTANCE": 500
    }, 
    "Filter": {
        "DummyFilter": {}, 
        "Kinematic": {
            "SkeletonFile": "skeleton.xml", 
            "LimitsBone": "skeleton_limits.xml", 
            "CfgFile": "kinematic_filter.xml", 
            "Loader": "XMLSkeletonLoader"
        }, 
        "CombinationFilter": [
            {
                "DummyFilter": {}
            }, 
            {
                "Kinematic": {
                    "SkeletonFile": "skeleton.xml", 
                    "LimitsBone": "skeleton_limits.xml", 
                    "CfgFile": "kinematic_filter.xml", 
                    "Loader": "XMLSkeletonLoader"
                }
            }
        ], 
        "AlphaBeta": {
            "Alpha": 0.5, 
            "Beta": 0.5
        }, 
        "Median": {
            "HistorySize": 5
        }, 
        "FABRIK": {
            "SkeletonFile": "FABRIKSkeleton.xml"
        }, 
        "ThreshHold": {
            "MaxDisplacement": 10000, 
            "MinDisplacement": 20
        }
    }, 
    "WorkaroundsAPI": {
        "ConvertCoordsWithSensorParams": false
    }, 
    "DynamicModules": {
        "Path": "../middleware/", 
        "Modules": [
            "NuitrackModule"
        ], 
        "WModules": [
            "NuitrackWirelessModule"
        ]
    }, 
    "DaemonSettings": {
        "NewHumanEndpoint": "localhost:8081/onNewHuman", 
        "LostHumanEndpoint": "localhost:8081/onLostHuman"
    }, 
    "Faces": {
        "ToUse": false
    }, 
    "Realsense2Module": {
        "DeviceHardwareReset": false, 
        "Depth": {
            "ProcessWidth": 640, 
            "ProcessHeight": 480, 
            "ProcessMaxDepth": 2100
        }, 
        "FileRecord": "", 
        "RGB": {
            "ProcessWidth": 640, 
            "ProcessHeight": 480
        }, 
        "AlignTo": "none"
    }, 
    "OrbbecSDKDepthProviderModule": {
        "Depth": {
            "ProcessWidth": 640, 
            "ProcessHeight": 480
        }, 
        "SoftwareFilter": false, 
        "RGB": {
            "ProcessWidth": 640, 
            "ProcessHeight": 480
        }, 
        "Exposure": {
            "ToUse": false
        }
    }, 
    "StaticModuleLoader": [], 
    "OpenNIModule": {
        "FileRecord": ""
    }, 
    "Orientation": {
        "GlobalUp": {}, 
        "TPose": {}
    }, 
    "RegressionSkeletonization": {
        "Orientation": {
            "Dummy": {}
        }, 
        "CNN_RGB": {}, 
        "Filter": {
            "CombinationFilter": [
                {
                    "ThreshHold": {
                        "MaxDisplacement": 200, 
                        "SmoothCameraJointVelocity": false, 
                        "MinDisplacement": 15
                    }
                }, 
                {
                    "FABRIK": {
                        "PalmsAlignment": true, 
                        "FixedHeight": 1700, 
                        "HeightAlgorithm": "old", 
                        "SkeletonFile": "FABRIKSkeleton.xml"
                    }
                }, 
                {
                    "Confidence": {
                        "ConfidenceThreshold": 0.5, 
                        "CorrectHandsConfidence": true
                    }
                }
            ]
        }, 
        "CNN_RGB_Depth": {
            "Threshold": 0.3
        }, 
        "PoseOptimization": {
            "OptimizationParams": {
                "RegressionClouds": {
                    "EMOptimizer": {
                        "Tolerance": 0.005, 
                        "Iterations": 1
                    }, 
                    "Solver": {
                        "max_num_iteration": 20, 
                        "function_tolerance": 0.008, 
                        "gradient_tolerance": 1e-06, 
                        "parameter_tolerance": 0.0008
                    }
                }
            }, 
            "Terms": {
                "DepthEnergyFunction": {
                    "Enable": false, 
                    "Iterations": {
                        "To": 2, 
                        "From": 1
                    }, 
                    "Weight": 1000
                }, 
                "IntersectionEnergyFunction": {
                    "Enable": false, 
                    "Iterations": {
                        "To": 4, 
                        "From": 2
                    }, 
                    "Weight": 5000
                }, 
                "WeightsEnergyFunction": {
                    "Enable": false, 
                    "Iterations": {
                        "To": 4, 
                        "From": 0
                    }, 
                    "Weight": 1.0
                }, 
                "FieldEnergyFunction": {
                    "Enable": false, 
                    "Iterations": {
                        "To": 4, 
                        "From": 2
                    }, 
                    "Weight": 1
                }, 
                "DepthConsistencyEnergyFunction": {
                    "Enable": false, 
                    "Iterations": {
                        "To": 4, 
                        "From": 2
                    }, 
                    "Weight": 5000
                }, 
                "EnergyFunction": {
                    "Enable": true, 
                    "Iterations": {
                        "To": 1, 
                        "From": 0
                    }, 
                    "Weight": 1.0
                }
            }, 
            "SkeletonFile": "poseoptimization/skeleton.xml"
        }, 
        "Type": "Regression", 
        "Regression": {
            "Stat": "trees_stat", 
            "Scale": 0.5, 
            "MaxHeightScale": 1.4, 
            "UseGPU": false, 
            "MinHeightScale": 0.85, 
            "Tree": "trees.bin", 
            "StatBase": "regStat", 
            "ClassesCount": 32, 
            "Bodyparts": "bodyparts.xml", 
            "TreeNormCoef": 4000, 
            "MeanHeight": 1700, 
            "NormCoef": 4000, 
            "PredictType": "base"
        }
    }, 
    "LicenseFile": "../license.json", 
    "Reidentification": {
        "ToUse": false
    }, 
    "DefaultModules": {
        "DepthProvider": "DepthProviderManager"
    }, 
    "FusionDebugModule": {
        "ToUse": false
    }
}

Hello @vadim.bazhin, sadly the config you provided did not solve it, I still get the same crash/error.

Hi @PhilipOryx,

Which sensor are you using? Do you have this crash only in your application or also in nuitrack_sample.exe? Could you please reinstall Nuitrack and test again, does this solve your problem?
Thanks.

Hi @vadim.bazhin we are using sensor Intel RealSense D435.
We just tired it with the nuitrack_sample.exe and it crashes with Exit Code:1, when we check box AI: Skeletal Tracking.. We have also reinstall Nuitrack multiple times.

Hi @PhilipOryx,

This problem does not reproduce on our side with Nuitrack 0.36.9 and Intel Realsense D435 and with both of the above config files.

Is the AnyDesk session an option for you? There we can investigate and try to fix it.
Looking forward to your response.

Hi, yes that could be an option for us, when can we scheduled it? We are in Sweden so any time between 8am-5pm CET could works for us next week.

Is Monday at 2pm CET convenient for you?
You can send your AnyDesk ID to my email (vadim@3divi.com) at the appointed time.

Hello, sorry i think I am blocked that time does the same time on Tuesday work?

Yes, the same time on Tuesday is ok for us.