Update to game build 2024r174ex3

Bones oriented fingers tracking
This commit is contained in:
SDraw 2024-03-06 02:31:06 +03:00
parent 60259c94c4
commit b547512b69
No known key found for this signature in database
GPG key ID: BB95B4DAB2BB8BB5
12 changed files with 1677 additions and 1395 deletions

View file

@ -1,11 +1,14 @@
using ABI_RC.Core.Player;
using ABI_RC.Systems.IK;
using RootMotion.FinalIK;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
namespace ml_lme
{
[DisallowMultipleComponent]
[DefaultExecutionOrder(999999)]
class LeapTracked : MonoBehaviour
{
struct IKInfo
@ -18,17 +21,64 @@ namespace ml_lme
public Transform m_rightElbowTarget;
}
struct FingerBoneInfo
{
public LeapHand.FingerBone m_bone;
public Transform m_targetBone;
public Transform m_sourceBone;
public Quaternion m_offset;
}
static readonly Quaternion ms_offsetLeft = Quaternion.Euler(0f, 90f, 0f);
static readonly Quaternion ms_offsetRight = Quaternion.Euler(0f, 270f, 0f);
static readonly (HumanBodyBones, LeapHand.FingerBone, bool)[] ms_fingerBonesLinks =
{
(HumanBodyBones.LeftThumbProximal, LeapHand.FingerBone.ThumbProximal, true),
(HumanBodyBones.LeftThumbIntermediate, LeapHand.FingerBone.ThumbIntermediate, true),
(HumanBodyBones.LeftThumbDistal, LeapHand.FingerBone.ThumbDistal, true),
(HumanBodyBones.LeftIndexProximal, LeapHand.FingerBone.IndexProximal, true),
(HumanBodyBones.LeftIndexIntermediate, LeapHand.FingerBone.IndexIntermediate, true),
(HumanBodyBones.LeftIndexDistal, LeapHand.FingerBone.IndexDistal, true),
(HumanBodyBones.LeftMiddleProximal, LeapHand.FingerBone.MiddleProximal, true),
(HumanBodyBones.LeftMiddleIntermediate, LeapHand.FingerBone.MiddleIntermediate, true),
(HumanBodyBones.LeftMiddleDistal, LeapHand.FingerBone.MiddleDistal, true),
(HumanBodyBones.LeftRingProximal, LeapHand.FingerBone.RingProximal, true),
(HumanBodyBones.LeftRingIntermediate, LeapHand.FingerBone.RingIntermediate, true),
(HumanBodyBones.LeftRingDistal, LeapHand.FingerBone.RingDistal, true),
(HumanBodyBones.LeftLittleProximal, LeapHand.FingerBone.PinkyProximal, true),
(HumanBodyBones.LeftLittleIntermediate, LeapHand.FingerBone.PinkyIntermediate, true),
(HumanBodyBones.LeftLittleDistal, LeapHand.FingerBone.PinkyDistal, true),
(HumanBodyBones.RightThumbProximal, LeapHand.FingerBone.ThumbProximal, false),
(HumanBodyBones.RightThumbIntermediate, LeapHand.FingerBone.ThumbIntermediate, false),
(HumanBodyBones.RightThumbDistal, LeapHand.FingerBone.ThumbDistal, false),
(HumanBodyBones.RightIndexProximal, LeapHand.FingerBone.IndexProximal, false),
(HumanBodyBones.RightIndexIntermediate, LeapHand.FingerBone.IndexIntermediate, false),
(HumanBodyBones.RightIndexDistal, LeapHand.FingerBone.IndexDistal, false),
(HumanBodyBones.RightMiddleProximal, LeapHand.FingerBone.MiddleProximal, false),
(HumanBodyBones.RightMiddleIntermediate, LeapHand.FingerBone.MiddleIntermediate, false),
(HumanBodyBones.RightMiddleDistal, LeapHand.FingerBone.MiddleDistal, false),
(HumanBodyBones.RightRingProximal, LeapHand.FingerBone.RingProximal, false),
(HumanBodyBones.RightRingIntermediate, LeapHand.FingerBone.RingIntermediate, false),
(HumanBodyBones.RightRingDistal, LeapHand.FingerBone.RingDistal, false),
(HumanBodyBones.RightLittleProximal, LeapHand.FingerBone.PinkyProximal, false),
(HumanBodyBones.RightLittleIntermediate, LeapHand.FingerBone.PinkyIntermediate, false),
(HumanBodyBones.RightLittleDistal, LeapHand.FingerBone.PinkyDistal, false),
};
public static readonly float[] ms_lastLeftFingerBones = new float[20];
public static readonly float[] ms_lastRightFingerBones = new float[20];
bool m_inVR = false;
VRIK m_vrIK = null;
Transform m_hips = null;
bool m_enabled = true;
bool m_fingersOnly = false;
bool m_trackElbows = true;
Transform m_leftHand = null;
Transform m_rightHand = null;
IKInfo m_vrIKInfo;
ArmIK m_leftArmIK = null;
ArmIK m_rightArmIK = null;
@ -39,18 +89,30 @@ namespace ml_lme
bool m_leftTargetActive = false; // VRIK only
bool m_rightTargetActive = false; // VRIK only
readonly List<FingerBoneInfo> m_leftFingerBones = null;
readonly List<FingerBoneInfo> m_rightFingerBones = null;
Quaternion m_leftWristOffset;
Quaternion m_rightWristOffset;
internal LeapTracked()
{
m_leftFingerBones = new List<FingerBoneInfo>();
m_rightFingerBones = new List<FingerBoneInfo>();
}
// Unity events
void Start()
{
m_inVR = Utils.IsInVR();
m_leftHandTarget = new GameObject("RotationTarget").transform;
m_leftHandTarget.parent = LeapTracking.Instance.GetLeftHand();
m_leftHandTarget.parent = LeapTracking.Instance.GetLeftHand().GetRoot();
m_leftHandTarget.localPosition = Vector3.zero;
m_leftHandTarget.localRotation = Quaternion.identity;
m_rightHandTarget = new GameObject("RotationTarget").transform;
m_rightHandTarget.parent = LeapTracking.Instance.GetRightHand();
m_rightHandTarget.parent = LeapTracking.Instance.GetRightHand().GetRoot();
m_rightHandTarget.localPosition = Vector3.zero;
m_rightHandTarget.localRotation = Quaternion.identity;
@ -114,19 +176,47 @@ namespace ml_lme
void LateUpdate()
{
if(m_enabled && (m_vrIK == null) && (m_poseHandler != null))
if(m_enabled && (m_poseHandler != null))
{
LeapParser.LeapData l_data = LeapManager.Instance.GetLatestData();
Vector3 l_hipsLocalPos = m_hips.localPosition;
Quaternion l_hipsLocalRot = m_hips.localRotation;
if(l_data.m_leftHand.m_present)
{
Transform l_leapWrist = LeapTracking.Instance.GetLeftHand().GetWrist();
Quaternion l_turnBack = (m_leftHand.rotation * m_leftWristOffset) * Quaternion.Inverse(l_leapWrist.rotation);
foreach(var l_info in m_leftFingerBones)
l_info.m_targetBone.rotation = l_turnBack * (l_info.m_sourceBone.rotation * l_info.m_offset);
}
if(l_data.m_rightHand.m_present)
{
Transform l_leapWrist = LeapTracking.Instance.GetRightHand().GetWrist();
Quaternion l_turnBack = (m_rightHand.rotation * m_rightWristOffset) * Quaternion.Inverse(l_leapWrist.rotation);
foreach(var l_info in m_rightFingerBones)
l_info.m_targetBone.rotation = l_turnBack * (l_info.m_sourceBone.rotation * l_info.m_offset);
}
m_poseHandler.GetHumanPose(ref m_pose);
UpdateFingers(l_data);
m_poseHandler.SetHumanPose(ref m_pose);
m_hips.localPosition = l_hipsLocalPos;
m_hips.localRotation = l_hipsLocalRot;
if(l_data.m_leftHand.m_present)
{
for(int i = 0; i < 5; i++)
{
int l_offset = i * 4;
ms_lastLeftFingerBones[l_offset] = m_pose.muscles[(int)MuscleIndex.LeftThumb1Stretched + l_offset];
ms_lastLeftFingerBones[l_offset + 1] = m_pose.muscles[(int)MuscleIndex.LeftThumb2Stretched + l_offset];
ms_lastLeftFingerBones[l_offset + 2] = m_pose.muscles[(int)MuscleIndex.LeftThumb3Stretched + l_offset];
ms_lastLeftFingerBones[l_offset + 3] = m_pose.muscles[(int)MuscleIndex.LeftThumbSpread + l_offset];
}
}
if(l_data.m_rightHand.m_present)
{
for(int i = 0; i < 5; i++)
{
int l_offset = i * 4;
ms_lastRightFingerBones[l_offset] = m_pose.muscles[(int)MuscleIndex.RightThumb1Stretched + l_offset];
ms_lastRightFingerBones[l_offset + 1] = m_pose.muscles[(int)MuscleIndex.RightThumb2Stretched + l_offset];
ms_lastRightFingerBones[l_offset + 2] = m_pose.muscles[(int)MuscleIndex.RightThumb3Stretched + l_offset];
ms_lastRightFingerBones[l_offset + 3] = m_pose.muscles[(int)MuscleIndex.RightThumbSpread + l_offset];
}
}
}
}
@ -134,7 +224,6 @@ namespace ml_lme
internal void OnAvatarClear()
{
m_vrIK = null;
m_hips = null;
m_leftArmIK = null;
m_rightArmIK = null;
m_leftTargetActive = false;
@ -147,6 +236,14 @@ namespace ml_lme
m_leftHandTarget.localRotation = Quaternion.identity;
m_rightHandTarget.localPosition = Vector3.zero;
m_rightHandTarget.localRotation = Quaternion.identity;
m_leftFingerBones.Clear();
m_rightFingerBones.Clear();
m_leftHand = null;
m_rightHand = null;
m_leftWristOffset = Quaternion.identity;
m_rightWristOffset = Quaternion.identity;
}
internal void OnAvatarSetup()
@ -156,27 +253,31 @@ namespace ml_lme
if(PlayerSetup.Instance._animator.isHuman)
{
m_hips = PlayerSetup.Instance._animator.GetBoneTransform(HumanBodyBones.Hips);
m_poseHandler = new HumanPoseHandler(PlayerSetup.Instance._animator.avatar, PlayerSetup.Instance._animator.transform);
m_poseHandler.GetHumanPose(ref m_pose);
if(!m_inVR)
if(m_inVR)
{
PlayerSetup.Instance._avatar.transform.localPosition = Vector3.zero;
PlayerSetup.Instance._avatar.transform.localRotation = Quaternion.identity;
}
else
PoseHelper.ForceTPose(PlayerSetup.Instance._animator);
Transform l_hand = PlayerSetup.Instance._animator.GetBoneTransform(HumanBodyBones.LeftHand);
if(l_hand != null)
m_leftHandTarget.localRotation = ms_offsetLeft * (PlayerSetup.Instance._avatar.transform.GetMatrix().inverse * l_hand.GetMatrix()).rotation;
m_leftHand = PlayerSetup.Instance._animator.GetBoneTransform(HumanBodyBones.LeftHand);
m_leftHandTarget.localRotation = ms_offsetLeft * (Quaternion.Inverse(PlayerSetup.Instance._avatar.transform.rotation) * m_leftHand.rotation);
l_hand = PlayerSetup.Instance._animator.GetBoneTransform(HumanBodyBones.RightHand);
if(l_hand != null)
m_rightHandTarget.localRotation = ms_offsetRight * (PlayerSetup.Instance._avatar.transform.GetMatrix().inverse * l_hand.GetMatrix()).rotation;
m_rightHand = PlayerSetup.Instance._animator.GetBoneTransform(HumanBodyBones.RightHand);
m_rightHandTarget.localRotation = ms_offsetRight * (Quaternion.Inverse(PlayerSetup.Instance._avatar.transform.rotation) * m_rightHand.rotation);
ParseFingersBones();
if(m_vrIK != null)
{
m_vrIK.onPreSolverUpdate.AddListener(this.OnIKPreUpdate);
m_vrIK.onPostSolverUpdate.AddListener(this.OnIKPostUpdate);
}
else if(!m_inVR)
else
SetupArmIK();
}
}
@ -195,7 +296,7 @@ namespace ml_lme
m_vrIK.onPreSolverUpdate.AddListener(this.OnIKPreUpdate);
m_vrIK.onPostSolverUpdate.AddListener(this.OnIKPostUpdate);
}
else if(!m_inVR)
else
{
PoseHelper.ForceTPose(PlayerSetup.Instance._animator);
SetupArmIK();
@ -350,69 +451,33 @@ namespace ml_lme
}
}
void UpdateFingers(LeapParser.LeapData p_data)
void ParseFingersBones()
{
if(p_data.m_leftHand.m_present)
LeapTracking.Instance.GetLeftHand().Reset();
LeapTracking.Instance.GetLeftHand().GetWrist().rotation = PlayerSetup.Instance.transform.rotation * ms_offsetRight; // Weird, but that's how it works
m_leftWristOffset = Quaternion.Inverse(m_leftHand.rotation) * LeapTracking.Instance.GetLeftHand().GetWrist().rotation;
LeapTracking.Instance.GetRightHand().Reset();
LeapTracking.Instance.GetRightHand().GetWrist().rotation = PlayerSetup.Instance.transform.rotation * ms_offsetLeft; // Weird, but that's how it works
m_rightWristOffset = Quaternion.Inverse(m_rightHand.rotation) * LeapTracking.Instance.GetRightHand().GetWrist().rotation;
foreach(var l_link in ms_fingerBonesLinks)
{
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftThumb1Stretched, -0.5f - p_data.m_leftHand.m_bends[0]);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftThumb2Stretched, 0.7f - p_data.m_leftHand.m_bends[0] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftThumb3Stretched, 0.7f - p_data.m_leftHand.m_bends[0] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftThumbSpread, -p_data.m_leftHand.m_spreads[0]);
Transform l_transform = PlayerSetup.Instance._animator.GetBoneTransform(l_link.Item1);
if(l_transform != null)
{
FingerBoneInfo l_info = new FingerBoneInfo();
l_info.m_bone = l_link.Item2;
l_info.m_targetBone = l_transform;
l_info.m_sourceBone = (l_link.Item3 ? LeapTracking.Instance.GetLeftHand().GetFingersBone(l_link.Item2) : LeapTracking.Instance.GetRightHand().GetFingersBone(l_link.Item2));
l_info.m_offset = Quaternion.Inverse(l_info.m_sourceBone.rotation) * l_info.m_targetBone.rotation;
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftIndex1Stretched, 0.5f - p_data.m_leftHand.m_bends[1]);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftIndex2Stretched, 0.7f - p_data.m_leftHand.m_bends[1] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftIndex3Stretched, 0.7f - p_data.m_leftHand.m_bends[1] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftIndexSpread, p_data.m_leftHand.m_spreads[1]);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftMiddle1Stretched, 0.5f - p_data.m_leftHand.m_bends[2]);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftMiddle2Stretched, 0.7f - p_data.m_leftHand.m_bends[2] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftMiddle3Stretched, 0.7f - p_data.m_leftHand.m_bends[2] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftMiddleSpread, p_data.m_leftHand.m_spreads[2]);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftRing1Stretched, 0.5f - p_data.m_leftHand.m_bends[3]);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftRing2Stretched, 0.7f - p_data.m_leftHand.m_bends[3] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftRing3Stretched, 0.7f - p_data.m_leftHand.m_bends[3] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftRingSpread, -p_data.m_leftHand.m_spreads[3]);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftLittle1Stretched, 0.5f - p_data.m_leftHand.m_bends[4]);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftLittle2Stretched, 0.7f - p_data.m_leftHand.m_bends[4] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftLittle3Stretched, 0.7f - p_data.m_leftHand.m_bends[4] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.LeftLittleSpread, -p_data.m_leftHand.m_spreads[4]);
if(l_link.Item3)
m_leftFingerBones.Add(l_info);
else
m_rightFingerBones.Add(l_info);
}
}
if(p_data.m_rightHand.m_present)
{
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightThumb1Stretched, -0.5f - p_data.m_rightHand.m_bends[0]);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightThumb2Stretched, 0.7f - p_data.m_rightHand.m_bends[0] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightThumb3Stretched, 0.7f - p_data.m_rightHand.m_bends[0] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightThumbSpread, -p_data.m_rightHand.m_spreads[0]);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightIndex1Stretched, 0.5f - p_data.m_rightHand.m_bends[1]);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightIndex2Stretched, 0.7f - p_data.m_rightHand.m_bends[1] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightIndex3Stretched, 0.7f - p_data.m_rightHand.m_bends[1] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightIndexSpread, p_data.m_rightHand.m_spreads[1]);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightMiddle1Stretched, 0.5f - p_data.m_rightHand.m_bends[2]);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightMiddle2Stretched, 0.7f - p_data.m_rightHand.m_bends[2] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightMiddle3Stretched, 0.7f - p_data.m_rightHand.m_bends[2] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightMiddleSpread, p_data.m_rightHand.m_spreads[2]);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightRing1Stretched, 0.5f - p_data.m_rightHand.m_bends[3]);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightRing2Stretched, 0.7f - p_data.m_rightHand.m_bends[3] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightRing3Stretched, 0.7f - p_data.m_rightHand.m_bends[3] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightRingSpread, -p_data.m_rightHand.m_spreads[3]);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightLittle1Stretched, 0.5f - p_data.m_rightHand.m_bends[4]);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightLittle2Stretched, 0.7f - p_data.m_rightHand.m_bends[4] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightLittle3Stretched, 0.7f - p_data.m_rightHand.m_bends[4] * 2f);
UpdatePoseMuscle(ref m_pose, (int)MuscleIndex.RightLittleSpread, -p_data.m_rightHand.m_spreads[4]);
}
}
static void UpdatePoseMuscle(ref HumanPose p_pose, int p_index, float p_value)
{
if(p_pose.muscles.Length > p_index)
p_pose.muscles[p_index] = p_value;
}
}
}