Improved fingers binding

Mechanim filtering
This commit is contained in:
SDraw 2024-04-04 09:00:36 +03:00
parent 5bec2fcdb1
commit 00c92e1913
No known key found for this signature in database
GPG key ID: BB95B4DAB2BB8BB5
19 changed files with 557 additions and 239 deletions

View file

@ -10,6 +10,12 @@ namespace ml_lme
[DefaultExecutionOrder(999999)]
class LeapTracked : MonoBehaviour
{
enum PlaneType
{
OXZ,
OYX
}
struct IKInfo
{
public Vector4 m_armsWeights;
@ -20,50 +26,69 @@ namespace ml_lme
public Transform m_rightElbowTarget;
}
struct FingerBoneInfo
struct RotationOffset
{
public LeapHand.FingerBone m_bone;
public Transform m_targetBone;
public Transform m_sourceBone;
public Transform m_target;
public Transform m_source;
public Quaternion m_offset;
public void Reset()
{
m_source = null;
m_target = null;
m_offset = Quaternion.identity;
}
}
static readonly Quaternion ms_offsetLeft = Quaternion.Euler(0f, 90f, 0f);
static readonly Quaternion ms_offsetRight = Quaternion.Euler(0f, 270f, 0f);
static readonly (HumanBodyBones, LeapHand.FingerBone, bool)[] ms_fingerBonesLinks =
static readonly (HumanBodyBones, bool)[] ms_fingers =
{
(HumanBodyBones.LeftThumbProximal, LeapHand.FingerBone.ThumbProximal, true),
(HumanBodyBones.LeftThumbIntermediate, LeapHand.FingerBone.ThumbIntermediate, true),
(HumanBodyBones.LeftThumbDistal, LeapHand.FingerBone.ThumbDistal, true),
(HumanBodyBones.LeftIndexProximal, LeapHand.FingerBone.IndexProximal, true),
(HumanBodyBones.LeftIndexIntermediate, LeapHand.FingerBone.IndexIntermediate, true),
(HumanBodyBones.LeftIndexDistal, LeapHand.FingerBone.IndexDistal, true),
(HumanBodyBones.LeftMiddleProximal, LeapHand.FingerBone.MiddleProximal, true),
(HumanBodyBones.LeftMiddleIntermediate, LeapHand.FingerBone.MiddleIntermediate, true),
(HumanBodyBones.LeftMiddleDistal, LeapHand.FingerBone.MiddleDistal, true),
(HumanBodyBones.LeftRingProximal, LeapHand.FingerBone.RingProximal, true),
(HumanBodyBones.LeftRingIntermediate, LeapHand.FingerBone.RingIntermediate, true),
(HumanBodyBones.LeftRingDistal, LeapHand.FingerBone.RingDistal, true),
(HumanBodyBones.LeftLittleProximal, LeapHand.FingerBone.PinkyProximal, true),
(HumanBodyBones.LeftLittleIntermediate, LeapHand.FingerBone.PinkyIntermediate, true),
(HumanBodyBones.LeftLittleDistal, LeapHand.FingerBone.PinkyDistal, true),
(HumanBodyBones.LeftThumbProximal, true),
(HumanBodyBones.LeftThumbIntermediate, true),
(HumanBodyBones.LeftThumbDistal, true),
(HumanBodyBones.LeftIndexProximal, true),
(HumanBodyBones.LeftIndexIntermediate, true),
(HumanBodyBones.LeftIndexDistal, true),
(HumanBodyBones.LeftMiddleProximal, true),
(HumanBodyBones.LeftMiddleIntermediate, true),
(HumanBodyBones.LeftMiddleDistal, true),
(HumanBodyBones.LeftRingProximal, true),
(HumanBodyBones.LeftRingIntermediate, true),
(HumanBodyBones.LeftRingDistal, true),
(HumanBodyBones.LeftLittleProximal, true),
(HumanBodyBones.LeftLittleIntermediate, true),
(HumanBodyBones.LeftLittleDistal, true),
(HumanBodyBones.RightThumbProximal, LeapHand.FingerBone.ThumbProximal, false),
(HumanBodyBones.RightThumbIntermediate, LeapHand.FingerBone.ThumbIntermediate, false),
(HumanBodyBones.RightThumbDistal, LeapHand.FingerBone.ThumbDistal, false),
(HumanBodyBones.RightIndexProximal, LeapHand.FingerBone.IndexProximal, false),
(HumanBodyBones.RightIndexIntermediate, LeapHand.FingerBone.IndexIntermediate, false),
(HumanBodyBones.RightIndexDistal, LeapHand.FingerBone.IndexDistal, false),
(HumanBodyBones.RightMiddleProximal, LeapHand.FingerBone.MiddleProximal, false),
(HumanBodyBones.RightMiddleIntermediate, LeapHand.FingerBone.MiddleIntermediate, false),
(HumanBodyBones.RightMiddleDistal, LeapHand.FingerBone.MiddleDistal, false),
(HumanBodyBones.RightRingProximal, LeapHand.FingerBone.RingProximal, false),
(HumanBodyBones.RightRingIntermediate, LeapHand.FingerBone.RingIntermediate, false),
(HumanBodyBones.RightRingDistal, LeapHand.FingerBone.RingDistal, false),
(HumanBodyBones.RightLittleProximal, LeapHand.FingerBone.PinkyProximal, false),
(HumanBodyBones.RightLittleIntermediate, LeapHand.FingerBone.PinkyIntermediate, false),
(HumanBodyBones.RightLittleDistal, LeapHand.FingerBone.PinkyDistal, false),
(HumanBodyBones.RightThumbProximal, false),
(HumanBodyBones.RightThumbIntermediate, false),
(HumanBodyBones.RightThumbDistal, false),
(HumanBodyBones.RightIndexProximal, false),
(HumanBodyBones.RightIndexIntermediate, false),
(HumanBodyBones.RightIndexDistal, false),
(HumanBodyBones.RightMiddleProximal, false),
(HumanBodyBones.RightMiddleIntermediate, false),
(HumanBodyBones.RightMiddleDistal, false),
(HumanBodyBones.RightRingProximal, false),
(HumanBodyBones.RightRingIntermediate, false),
(HumanBodyBones.RightRingDistal, false),
(HumanBodyBones.RightLittleProximal, false),
(HumanBodyBones.RightLittleIntermediate, false),
(HumanBodyBones.RightLittleDistal, false),
};
static readonly (HumanBodyBones, HumanBodyBones, bool)[] ms_rotationFixChains =
{
(HumanBodyBones.LeftThumbProximal,HumanBodyBones.LeftThumbIntermediate,true), (HumanBodyBones.LeftThumbIntermediate, HumanBodyBones.LeftThumbDistal,true),
(HumanBodyBones.LeftIndexProximal,HumanBodyBones.LeftIndexIntermediate,true), (HumanBodyBones.LeftIndexIntermediate, HumanBodyBones.LeftIndexDistal,true),
(HumanBodyBones.LeftMiddleProximal,HumanBodyBones.LeftMiddleIntermediate,true), (HumanBodyBones.LeftMiddleIntermediate, HumanBodyBones.LeftMiddleDistal,true),
(HumanBodyBones.LeftRingProximal,HumanBodyBones.LeftRingIntermediate,true), (HumanBodyBones.LeftRingIntermediate, HumanBodyBones.LeftRingDistal,true),
(HumanBodyBones.LeftLittleProximal,HumanBodyBones.LeftLittleIntermediate,true), (HumanBodyBones.LeftLittleIntermediate, HumanBodyBones.LeftLittleDistal,true),
(HumanBodyBones.RightThumbProximal,HumanBodyBones.RightThumbIntermediate,false), (HumanBodyBones.RightThumbIntermediate, HumanBodyBones.RightThumbDistal,false),
(HumanBodyBones.RightIndexProximal,HumanBodyBones.RightIndexIntermediate,false), (HumanBodyBones.RightIndexIntermediate, HumanBodyBones.RightIndexDistal,false),
(HumanBodyBones.RightMiddleProximal,HumanBodyBones.RightMiddleIntermediate,false), (HumanBodyBones.RightMiddleIntermediate, HumanBodyBones.RightMiddleDistal,false),
(HumanBodyBones.RightRingProximal,HumanBodyBones.RightRingIntermediate,false), (HumanBodyBones.RightRingIntermediate, HumanBodyBones.RightRingDistal,false),
(HumanBodyBones.RightLittleProximal,HumanBodyBones.RightLittleIntermediate,false), (HumanBodyBones.RightLittleIntermediate, HumanBodyBones.RightLittleDistal,false)
};
public static readonly float[] ms_lastLeftFingerBones = new float[20];
@ -71,13 +96,12 @@ namespace ml_lme
bool m_inVR = false;
VRIK m_vrIK = null;
Transform m_hips = null;
bool m_enabled = true;
bool m_fingersOnly = false;
bool m_trackElbows = true;
Transform m_leftHand = null;
Transform m_rightHand = null;
IKInfo m_vrIKInfo;
ArmIK m_leftArmIK = null;
ArmIK m_rightArmIK = null;
@ -88,16 +112,15 @@ namespace ml_lme
bool m_leftTargetActive = false; // VRIK only
bool m_rightTargetActive = false; // VRIK only
readonly List<FingerBoneInfo> m_leftFingerBones = null;
readonly List<FingerBoneInfo> m_rightFingerBones = null;
Quaternion m_leftWristOffset;
Quaternion m_rightWristOffset;
RotationOffset m_leftHandOffset; // From avatar hand to Leap wrist
RotationOffset m_rightHandOffset;
readonly List<RotationOffset> m_leftFingerOffsets = null; // From Leap finger bone to avatar finger bone
readonly List<RotationOffset> m_rightFingerOffsets = null;
internal LeapTracked()
{
m_leftFingerBones = new List<FingerBoneInfo>();
m_rightFingerBones = new List<FingerBoneInfo>();
m_leftFingerOffsets = new List<RotationOffset>();
m_rightFingerOffsets = new List<RotationOffset>();
}
// Unity events
@ -180,17 +203,15 @@ namespace ml_lme
LeapParser.LeapData l_data = LeapManager.Instance.GetLatestData();
if(l_data.m_leftHand.m_present)
{
Transform l_leapWrist = LeapTracking.Instance.GetLeftHand().GetWrist();
Quaternion l_turnBack = (m_leftHand.rotation * m_leftWristOffset) * Quaternion.Inverse(l_leapWrist.rotation);
foreach(var l_info in m_leftFingerBones)
l_info.m_targetBone.rotation = l_turnBack * (l_info.m_sourceBone.rotation * l_info.m_offset);
Quaternion l_turnBack = (m_leftHandOffset.m_source.rotation * m_leftHandOffset.m_offset) * Quaternion.Inverse(m_leftHandOffset.m_target.rotation);
foreach(var l_info in m_leftFingerOffsets)
l_info.m_target.rotation = l_turnBack * (l_info.m_source.rotation * l_info.m_offset);
}
if(l_data.m_rightHand.m_present)
{
Transform l_leapWrist = LeapTracking.Instance.GetRightHand().GetWrist();
Quaternion l_turnBack = (m_rightHand.rotation * m_rightWristOffset) * Quaternion.Inverse(l_leapWrist.rotation);
foreach(var l_info in m_rightFingerBones)
l_info.m_targetBone.rotation = l_turnBack * (l_info.m_sourceBone.rotation * l_info.m_offset);
Quaternion l_turnBack = (m_rightHandOffset.m_source.rotation * m_rightHandOffset.m_offset) * Quaternion.Inverse(m_rightHandOffset.m_target.rotation);
foreach(var l_info in m_rightFingerOffsets)
l_info.m_target.rotation = l_turnBack * (l_info.m_source.rotation * l_info.m_offset);
}
m_poseHandler.GetHumanPose(ref m_pose);
@ -211,6 +232,15 @@ namespace ml_lme
ms_lastRightFingerBones[l_offset + 3] = m_pose.muscles[(int)MuscleIndex.RightThumbSpread + l_offset];
}
}
if(Settings.MechanimFilter && (m_hips != null))
{
// Yoinked from IKSystem.OnPostSolverUpdateGeneral
Vector3 l_pos = m_hips.position;
Quaternion l_rot = m_hips.rotation;
m_poseHandler.SetHumanPose(ref m_pose);
m_hips.SetPositionAndRotation(l_pos, l_rot);
}
}
}
@ -218,6 +248,7 @@ namespace ml_lme
internal void OnAvatarClear()
{
m_vrIK = null;
m_hips = null;
m_leftArmIK = null;
m_rightArmIK = null;
m_leftTargetActive = false;
@ -231,13 +262,11 @@ namespace ml_lme
m_rightHandTarget.localPosition = Vector3.zero;
m_rightHandTarget.localRotation = Quaternion.identity;
m_leftFingerBones.Clear();
m_rightFingerBones.Clear();
m_leftHandOffset.Reset();
m_rightHandOffset.Reset();
m_leftHand = null;
m_rightHand = null;
m_leftWristOffset = Quaternion.identity;
m_rightWristOffset = Quaternion.identity;
m_leftFingerOffsets.Clear();
m_rightFingerOffsets.Clear();
}
internal void OnAvatarSetup()
@ -251,11 +280,13 @@ namespace ml_lme
m_poseHandler = new HumanPoseHandler(PlayerSetup.Instance._animator.avatar, PlayerSetup.Instance._animator.transform);
m_poseHandler.GetHumanPose(ref m_pose);
m_leftHand = PlayerSetup.Instance._animator.GetBoneTransform(HumanBodyBones.LeftHand);
m_leftHandTarget.localRotation = ms_offsetLeft * (Quaternion.Inverse(PlayerSetup.Instance._avatar.transform.rotation) * m_leftHand.rotation);
m_hips = PlayerSetup.Instance._animator.GetBoneTransform(HumanBodyBones.Hips);
m_rightHand = PlayerSetup.Instance._animator.GetBoneTransform(HumanBodyBones.RightHand);
m_rightHandTarget.localRotation = ms_offsetRight * (Quaternion.Inverse(PlayerSetup.Instance._avatar.transform.rotation) * m_rightHand.rotation);
m_leftHandOffset.m_source = PlayerSetup.Instance._animator.GetBoneTransform(HumanBodyBones.LeftHand);
m_leftHandTarget.localRotation = ms_offsetLeft * (Quaternion.Inverse(PlayerSetup.Instance._avatar.transform.rotation) * m_leftHandOffset.m_source.rotation);
m_rightHandOffset.m_source = PlayerSetup.Instance._animator.GetBoneTransform(HumanBodyBones.RightHand);
m_rightHandTarget.localRotation = ms_offsetRight * (Quaternion.Inverse(PlayerSetup.Instance._avatar.transform.rotation) * m_rightHandOffset.m_source.rotation);
ParseFingersBones();
@ -441,31 +472,96 @@ namespace ml_lme
void ParseFingersBones()
{
LeapTracking.Instance.GetLeftHand().Reset();
LeapTracking.Instance.GetLeftHand().GetWrist().rotation = PlayerSetup.Instance.transform.rotation * ms_offsetRight; // Weird, but that's how it works
m_leftWristOffset = Quaternion.Inverse(m_leftHand.rotation) * LeapTracking.Instance.GetLeftHand().GetWrist().rotation;
LeapTracking.Instance.Rebind(PlayerSetup.Instance.transform.rotation);
LeapTracking.Instance.GetRightHand().Reset();
LeapTracking.Instance.GetRightHand().GetWrist().rotation = PlayerSetup.Instance.transform.rotation * ms_offsetLeft; // Weird, but that's how it works
m_rightWristOffset = Quaternion.Inverse(m_rightHand.rotation) * LeapTracking.Instance.GetRightHand().GetWrist().rotation;
// Try to "fix" rotations, slightly inaccurate after 0YX plane rotation
foreach(var l_tuple in ms_rotationFixChains)
{
ReorientateTowards(
PlayerSetup.Instance._animator.GetBoneTransform(l_tuple.Item1),
PlayerSetup.Instance._animator.GetBoneTransform(l_tuple.Item2),
l_tuple.Item3 ? LeapTracking.Instance.GetLeftHand().GetBone(l_tuple.Item1) : LeapTracking.Instance.GetRightHand().GetBone(l_tuple.Item1),
l_tuple.Item3 ? LeapTracking.Instance.GetLeftHand().GetBone(l_tuple.Item2) : LeapTracking.Instance.GetRightHand().GetBone(l_tuple.Item2),
PlaneType.OXZ
);
ReorientateTowards(
PlayerSetup.Instance._animator.GetBoneTransform(l_tuple.Item1),
PlayerSetup.Instance._animator.GetBoneTransform(l_tuple.Item2),
l_tuple.Item3 ? LeapTracking.Instance.GetLeftHand().GetBone(l_tuple.Item1) : LeapTracking.Instance.GetRightHand().GetBone(l_tuple.Item1),
l_tuple.Item3 ? LeapTracking.Instance.GetLeftHand().GetBone(l_tuple.Item2) : LeapTracking.Instance.GetRightHand().GetBone(l_tuple.Item2),
PlaneType.OYX
);
}
foreach(var l_link in ms_fingerBonesLinks)
// Bind
m_leftHandOffset.m_target = LeapTracking.Instance.GetLeftHand().GetBone(HumanBodyBones.LeftHand);
if((m_leftHandOffset.m_source != null) && (m_leftHandOffset.m_target != null))
m_leftHandOffset.m_offset = Quaternion.Inverse(m_leftHandOffset.m_source.rotation) * m_leftHandOffset.m_target.rotation;
m_rightHandOffset.m_target = LeapTracking.Instance.GetRightHand().GetBone(HumanBodyBones.RightHand);
if((m_rightHandOffset.m_source != null) && (m_rightHandOffset.m_target != null))
m_rightHandOffset.m_offset = Quaternion.Inverse(m_rightHandOffset.m_source.rotation) * m_rightHandOffset.m_target.rotation;
foreach(var l_link in ms_fingers)
{
Transform l_transform = PlayerSetup.Instance._animator.GetBoneTransform(l_link.Item1);
if(l_transform != null)
{
FingerBoneInfo l_info = new FingerBoneInfo();
l_info.m_bone = l_link.Item2;
l_info.m_targetBone = l_transform;
l_info.m_sourceBone = (l_link.Item3 ? LeapTracking.Instance.GetLeftHand().GetFingersBone(l_link.Item2) : LeapTracking.Instance.GetRightHand().GetFingersBone(l_link.Item2));
l_info.m_offset = Quaternion.Inverse(l_info.m_sourceBone.rotation) * l_info.m_targetBone.rotation;
RotationOffset l_offset = new RotationOffset();
l_offset.m_target = l_transform;
l_offset.m_source = (l_link.Item2 ? LeapTracking.Instance.GetLeftHand().GetBone(l_link.Item1) : LeapTracking.Instance.GetRightHand().GetBone(l_link.Item1));
l_offset.m_offset = Quaternion.Inverse(l_offset.m_source.rotation) * l_offset.m_target.rotation;
if(l_link.Item3)
m_leftFingerBones.Add(l_info);
if(l_link.Item2)
m_leftFingerOffsets.Add(l_offset);
else
m_rightFingerBones.Add(l_info);
m_rightFingerOffsets.Add(l_offset);
}
}
}
void ReorientateTowards(Transform p_target, Transform p_targetEnd, Transform p_source, Transform p_sourceEnd, PlaneType p_plane)
{
if((p_target != null) && (p_targetEnd != null) && (p_source != null) && (p_sourceEnd != null))
{
Quaternion l_playerInv = Quaternion.Inverse(PlayerSetup.Instance.transform.rotation);
Vector3 l_targetDir = l_playerInv * (p_targetEnd.position - p_target.position);
Vector3 l_sourceDir = l_playerInv * (p_sourceEnd.position - p_source.position);
switch(p_plane)
{
case PlaneType.OXZ:
l_targetDir.y = 0f;
l_sourceDir.y = 0f;
break;
case PlaneType.OYX:
l_targetDir.z = 0f;
l_sourceDir.z = 0f;
break;
}
l_targetDir = Vector3.Normalize(l_targetDir);
l_sourceDir = Vector3.Normalize(l_sourceDir);
Quaternion l_targetRot = Quaternion.identity;
Quaternion l_sourceRot = Quaternion.identity;
switch(p_plane)
{
case PlaneType.OXZ:
l_targetRot = Quaternion.LookRotation(l_targetDir, Vector3.up);
l_sourceRot = Quaternion.LookRotation(l_sourceDir, Vector3.up);
break;
case PlaneType.OYX:
l_targetRot = Quaternion.LookRotation(l_targetDir, Vector3.forward);
l_sourceRot = Quaternion.LookRotation(l_sourceDir, Vector3.forward);
break;
}
Quaternion l_diff = Quaternion.Inverse(l_targetRot) * l_sourceRot;
if(p_plane == PlaneType.OYX)
l_diff = Quaternion.Euler(0f, 0f, l_diff.eulerAngles.y);
Quaternion l_adjusted = l_diff * (l_playerInv * p_target.rotation);
p_target.rotation = PlayerSetup.Instance.transform.rotation * l_adjusted;
}
}
}
}