diff --git a/ml_vet/Main.cs b/ml_vet/Main.cs index 506b4f6..5b95a3a 100644 --- a/ml_vet/Main.cs +++ b/ml_vet/Main.cs @@ -4,6 +4,7 @@ using ABI_RC.Core.Player.EyeMovement; using ABI_RC.Core.Savior; using ABI_RC.Systems.FaceTracking; using ABI_RC.Systems.FaceTracking.Impl; +using ABI_RC.Systems.RuntimeDebug; using System.Reflection; using System.Threading; using UnityEngine; @@ -24,7 +25,17 @@ namespace ml_vet SingleEyeData m_rightEyeData; Vector2 m_openness = Vector2.one; - Vector3 m_gazeDirection = Vector3.forward; + Vector3 m_leftEyeGaze = new Vector3(1f, 0f, 1f).normalized; + Vector3 m_rightEyeGaze = new Vector3(-1f, 0f, 1f).normalized; + Vector3 m_combinedGaze = Vector3.forward; + Vector3 m_leftEyeOrigin = new Vector3(-0.025f, 0f, 0f); + Vector3 m_rightEyeOrigin = new Vector3(0.025f, 0f, 0f); + Vector3 m_cominedGazeOrigin = Vector3.zero; + bool m_leftEyeGazeValid = false; + bool m_rightEyeGazeValid = false; + bool m_combinedGazeValid = false; + Vector3 m_gazePoint = Vector3.forward; + Vector3 m_gazeVelocity = Vector3.zero; bool m_faceTrackingEnabled = false; @@ -42,11 +53,6 @@ namespace ml_vet null, new HarmonyLib.HarmonyMethod(typeof(ViveEyeTracking).GetMethod(nameof(EyeMovementControllerUpdate_Postfix), BindingFlags.NonPublic | BindingFlags.Static)) ); - HarmonyInstance.Patch( - typeof(PlayerSetup).GetMethod("UpdatePlayerAvatarMovementData", BindingFlags.Instance | BindingFlags.NonPublic), - null, - new HarmonyLib.HarmonyMethod(typeof(ViveEyeTracking).GetMethod(nameof(OnPlayerAvatarMovementDataUpdate_Postfix), BindingFlags.Static | BindingFlags.NonPublic)) - ); MelonLoader.MelonCoroutines.Start(WaitForRootLogic(HarmonyInstance)); } @@ -85,15 +91,44 @@ namespace ml_vet Monitor.Exit(m_lock); } - if(m_combinedEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_GAZE_DIRECTION_VALIDITY)) + m_combinedGazeValid = m_combinedEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_GAZE_DIRECTION_VALIDITY); + if(m_combinedGazeValid) { - m_gazeDirection = m_combinedEyeData.gaze_direction_normalized; - m_gazeDirection.x *= -1f; + m_combinedGaze = m_combinedEyeData.gaze_direction_normalized; + m_combinedGaze.x *= -1f; + } + if(m_combinedEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_GAZE_ORIGIN_VALIDITY)) + { + m_cominedGazeOrigin = m_combinedEyeData.gaze_origin_mm * 0.001f; + m_cominedGazeOrigin.x *= -1f; } + + m_leftEyeGazeValid = m_leftEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_GAZE_DIRECTION_VALIDITY); + if(m_leftEyeGazeValid) + { + m_leftEyeGaze = m_leftEyeData.gaze_direction_normalized; + m_leftEyeGaze.x *= -1f; + } + if(m_leftEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_GAZE_ORIGIN_VALIDITY)) + { + m_leftEyeOrigin = m_leftEyeData.gaze_origin_mm * 0.001f; + m_leftEyeOrigin.x *= -1f; + } if(m_leftEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_EYE_OPENNESS_VALIDITY)) m_openness.x = m_leftEyeData.eye_openness; + m_rightEyeGazeValid = m_rightEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_GAZE_DIRECTION_VALIDITY); + if(m_rightEyeGazeValid) + { + m_rightEyeGaze = m_rightEyeData.gaze_direction_normalized; + m_rightEyeGaze.x *= -1f; + } + if(m_rightEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_GAZE_ORIGIN_VALIDITY)) + { + m_rightEyeOrigin = m_rightEyeData.gaze_origin_mm * 0.001f; + m_rightEyeOrigin.x *= -1f; + } if(m_rightEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_EYE_OPENNESS_VALIDITY)) m_openness.y = m_rightEyeData.eye_openness; } @@ -139,27 +174,93 @@ namespace ml_vet if(p_controller.IsLocal && IsReadyForUpdate()) { p_controller.manualBlinking = true; - p_controller.blinkProgress = 1f - Mathf.Clamp01((m_openness.x + m_openness.y) * 0.5f); + p_controller.blinkProgressLeft = 1f - m_openness.x; + p_controller.blinkProgressRight = 1f - m_openness.y; p_controller.manualViewTarget = true; - p_controller.targetViewPosition = p_controller.ViewPointTransform.position + p_controller.ViewPointTransform.rotation * (m_gazeDirection * 5f); - } - } - catch(System.Exception e) - { - MelonLoader.MelonLogger.Error(e); - } - } - static void OnPlayerAvatarMovementDataUpdate_Postfix(ref PlayerSetup __instance, PlayerAvatarMovementData ____playerAvatarMovementData) => ms_instance?.OnPlayerAvatarMovementDataPostUpdate(__instance, ____playerAvatarMovementData); - void OnPlayerAvatarMovementDataPostUpdate(PlayerSetup p_instance, PlayerAvatarMovementData p_data) - { - try - { - if(IsReadyForUpdate() && (p_instance.EyeMovementController != null)) - { - p_data.EyeTrackingBlinkProgressLeft = 1f - Mathf.Clamp01(m_openness.x); - p_data.EyeTrackingBlinkProgressRight = 1f - Mathf.Clamp01(m_openness.y); + Vector3 l_gazePoint = m_gazePoint; + float l_playspaceScale = PlayerSetup.Instance.GetPlaySpaceScale(); + + if(m_leftEyeGazeValid && m_rightEyeGazeValid) + { + // https://en.wikipedia.org/wiki/Line%E2%80%93line_intersection + // Projecting on OXZ plane to intersect eyes + float x1 = m_leftEyeOrigin.x; + float x2 = m_leftEyeOrigin.x + m_leftEyeGaze.x; + float y1 = m_leftEyeOrigin.z; + float y2 = m_leftEyeOrigin.z + m_leftEyeGaze.z; + + float x3 = m_rightEyeOrigin.x; + float x4 = m_rightEyeOrigin.x + m_rightEyeGaze.x; + float y3 = m_rightEyeOrigin.z; + float y4 = m_rightEyeOrigin.z + m_rightEyeGaze.z; + + float l_det = (x1 - x2) * (y3 - y4) - (y1 - y2) * (x3 - x4); + if(!Mathf.Approximately(l_det, 0f)) + { + float l_detZ = (x1 * y2 - y1 * x2) * (y3 - y4) - (y1 - y2) * (x3 * y4 - y3 * x4); + float l_posZ = l_detZ / l_det; + + float l_thetaLeft = (l_posZ - m_leftEyeOrigin.z) / m_leftEyeGaze.z; + Vector3 l_leftEyePoint = m_leftEyeOrigin + l_thetaLeft * m_leftEyeGaze; + + float l_thetaRight = (l_posZ - m_rightEyeOrigin.z) / m_rightEyeGaze.z; + Vector3 l_rightEyePoint = m_rightEyeOrigin + l_thetaRight * m_rightEyeGaze; + + Vector3 l_midPoint = Vector3.Lerp(l_leftEyePoint, l_rightEyePoint, 0.5f); + Vector3 l_combineGazeOrigin = Vector3.Lerp(m_leftEyeOrigin, m_rightEyeOrigin, 0.5f); // m_combinedGazeOrigin is unstable, wow + + if(l_midPoint.z > l_combineGazeOrigin.z) + { + Vector3 l_resultDir = l_midPoint - l_combineGazeOrigin; + l_resultDir = Vector3.ClampMagnitude(l_resultDir, 1f); + l_gazePoint = (l_combineGazeOrigin + l_resultDir) * l_playspaceScale; + } + } + else + { + Vector3 l_combineGazeOrigin = Vector3.Lerp(m_leftEyeOrigin, m_rightEyeOrigin, 0.5f); + l_gazePoint = (l_combineGazeOrigin + m_combinedGaze) * l_playspaceScale; + } + } + else + { + if(m_leftEyeGazeValid) + l_gazePoint = (m_leftEyeOrigin + m_leftEyeGaze) * l_playspaceScale; + if(m_rightEyeGazeValid) + l_gazePoint = (m_rightEyeGaze + m_rightEyeGaze) * l_playspaceScale; + } + + l_gazePoint = Vector3.SmoothDamp(m_gazePoint, l_gazePoint, ref m_gazeVelocity, Settings.Smoothing); + p_controller.targetViewPosition = p_controller.ViewPointTransform.position + p_controller.ViewPointTransform.rotation * l_gazePoint; + + if(m_rightEyeGazeValid || m_leftEyeGazeValid) + m_gazePoint = l_gazePoint; + + if(Settings.Debug) + { + Vector3 l_pos = p_controller.ViewPointTransform.position; + Quaternion l_rot = p_controller.ViewPointTransform.rotation; + + RuntimeGizmos.DrawArrowFromTo( + l_pos + l_rot * (m_leftEyeOrigin * l_playspaceScale), + l_pos + l_rot * ((m_leftEyeOrigin + (m_leftEyeGazeValid ? m_leftEyeGaze : Vector3.forward)) * l_playspaceScale), + 0.01f * l_playspaceScale, Color.blue, CVRLayers.Default, 0.25f + ); + RuntimeGizmos.DrawArrowFromTo( + l_pos + l_rot * (m_rightEyeOrigin * l_playspaceScale), + l_pos + l_rot * ((m_rightEyeOrigin + (m_rightEyeGazeValid ? m_rightEyeGaze : Vector3.forward)) * l_playspaceScale), + 0.01f * l_playspaceScale, Color.blue, CVRLayers.Default, 0.25f + ); + RuntimeGizmos.DrawArrowFromTo( + l_pos + l_rot * (m_cominedGazeOrigin * l_playspaceScale), + l_pos + l_rot * ((m_cominedGazeOrigin + (m_combinedGazeValid ? m_combinedGaze : Vector3.forward)) * l_playspaceScale), + 0.01f * l_playspaceScale, Color.red, CVRLayers.Default, 0.25f + ); + + RuntimeGizmos.DrawSphere(p_controller.targetViewPosition, 0.01f * l_playspaceScale, Color.cyan, CVRLayers.Default, 0.25f); + } } } catch(System.Exception e) diff --git a/ml_vet/Properties/AssemblyInfo.cs b/ml_vet/Properties/AssemblyInfo.cs index a9c4cdd..2414d4e 100644 --- a/ml_vet/Properties/AssemblyInfo.cs +++ b/ml_vet/Properties/AssemblyInfo.cs @@ -1,4 +1,4 @@ -[assembly: MelonLoader.MelonInfo(typeof(ml_vet.ViveEyeTracking), "ViveEyeTracking", "1.0.0", "SDraw", "https://github.com/SDraw/ml_mods_cvr")] +[assembly: MelonLoader.MelonInfo(typeof(ml_vet.ViveEyeTracking), "ViveEyeTracking", "1.0.1", "SDraw", "https://github.com/SDraw/ml_mods_cvr")] [assembly: MelonLoader.MelonGame(null, "ChilloutVR")] [assembly: MelonLoader.MelonPlatform(MelonLoader.MelonPlatformAttribute.CompatiblePlatforms.WINDOWS_X64)] [assembly: MelonLoader.MelonPlatformDomain(MelonLoader.MelonPlatformDomainAttribute.CompatibleDomains.MONO)] diff --git a/ml_vet/README.md b/ml_vet/README.md index 56dcb5d..e76ae74 100644 --- a/ml_vet/README.md +++ b/ml_vet/README.md @@ -6,9 +6,9 @@ This mod complements functionality of in-game SRanipal face tracking module and * Implemented native TobiiXR eye tracking is very unreliable. It freezes main thread with 66% chance at game launch and has limited detection of eyes openness. # Benefits? -* SRanipal API supports ranged eyes openness detection, unlike TobiiXR. You can finally smirk now. +* SRanipal API supports ranged eyes openness detection, unlike TobiiXR. You can finally squint now. * There is no main thread freeze at game launch. -* Semi-ready for future game update with separated eyes blinking. +* Ready for future game update with separated eyes blinking (currently only nightly). # Installation * Install [latest MelonLoader](https://github.com/LavaGang/MelonLoader) @@ -17,5 +17,10 @@ This mod complements functionality of in-game SRanipal face tracking module and # Usage Available mod's settings in `Settings - Implementation - Vive Eye Tracking`: -* **Enable eye tracking:** eye tracking state; `false` by default. +* **Enable eye tracking:** eye tracking state; `true` by default. * Note: If Vive face tracking is already running, enabling requires Vive face tracking restart. +* **Gaze smoothing:** smoothing of gaze point; 5 by default. +* **Debug gizmos:** show debug lines of SRanipal gazes directions and calculated gaze point; `false` by default. + +# Notes +* Made primarily for Vive Pro Eye and SRanipal tracking. diff --git a/ml_vet/Settings.cs b/ml_vet/Settings.cs index 8df996f..6cb85ab 100644 --- a/ml_vet/Settings.cs +++ b/ml_vet/Settings.cs @@ -16,15 +16,21 @@ namespace ml_vet enum ModSetting { - Enabled = 0 + Enabled = 0, + Smoothing, + Debug } - public static bool Enabled { get; private set; } = false; + public static bool Enabled { get; private set; } = true; + public static float Smoothing { get; private set; } = 0f; + public static bool Debug { get; private set; } = false; static MelonLoader.MelonPreferences_Category ms_category = null; static List ms_entries = null; public static readonly SettingEvent OnEnabledChanged = new SettingEvent(); + public static readonly SettingEvent OnSmoothingChanged = new SettingEvent(); + public static readonly SettingEvent OnDebugChanged = new SettingEvent(); internal static void Init() { @@ -33,9 +39,13 @@ namespace ml_vet ms_entries = new List() { ms_category.CreateEntry(ModSetting.Enabled.ToString(), Enabled), + ms_category.CreateEntry(ModSetting.Smoothing.ToString(), (int)(Smoothing * 100f)), + ms_category.CreateEntry(ModSetting.Debug.ToString(), Debug), }; Enabled = (bool)ms_entries[(int)ModSetting.Enabled].BoxedValue; + Smoothing = ((int)ms_entries[(int)ModSetting.Smoothing].BoxedValue) * 0.01f; + Debug = (bool)ms_entries[(int)ModSetting.Debug].BoxedValue; MelonLoader.MelonCoroutines.Start(WaitMainMenuUi()); } @@ -52,6 +62,7 @@ namespace ml_vet ViewManager.Instance.gameMenuView.Listener.ReadyForBindings += () => { ViewManager.Instance.gameMenuView.View.BindCall("OnToggleUpdate_" + ms_category.Identifier, new Action(OnToggleUpdate)); + ViewManager.Instance.gameMenuView.View.BindCall("OnSliderUpdate_" + ms_category.Identifier, new Action(OnSliderUpdate)); }; ViewManager.Instance.gameMenuView.Listener.FinishLoad += (_) => { @@ -59,9 +70,19 @@ namespace ml_vet ViewManager.Instance.gameMenuView.View.ExecuteScript(ResourcesHandler.GetEmbeddedResource("mod_menu.js")); foreach(var l_entry in ms_entries) ViewManager.Instance.gameMenuView.View.TriggerEvent("updateModSetting", ms_category.Identifier, l_entry.DisplayName, l_entry.GetValueAsString()); + + MelonLoader.MelonCoroutines.Start(UpdateMenuSettings()); }; } + static System.Collections.IEnumerator UpdateMenuSettings() + { + yield return new UnityEngine.WaitForEndOfFrame(); + + foreach(var l_entry in ms_entries) + ViewManager.Instance.gameMenuView.View.TriggerEvent("updateModSetting", ms_category.Identifier, l_entry.DisplayName, l_entry.GetValueAsString()); + } + static void OnToggleUpdate(string p_name, string p_value) { try @@ -76,6 +97,37 @@ namespace ml_vet OnEnabledChanged.Invoke(Enabled); } break; + + case ModSetting.Debug: + { + Debug = l_value; + OnDebugChanged.Invoke(Debug); + } break; + } + + ms_entries[(int)l_setting].BoxedValue = l_value; + } + } + catch(Exception e) + { + MelonLoader.MelonLogger.Error(e); + } + } + + static void OnSliderUpdate(string p_name, string p_value) + { + try + { + if(Enum.TryParse(p_name, out ModSetting l_setting) && int.TryParse(p_value, out int l_value)) + { + switch(l_setting) + { + case ModSetting.Smoothing: + { + Smoothing = l_value * 0.01f; + OnSmoothingChanged.Invoke(Smoothing); + } + break; } ms_entries[(int)l_setting].BoxedValue = l_value; diff --git a/ml_vet/ml_vet.csproj b/ml_vet/ml_vet.csproj index a84de2c..ba48299 100644 --- a/ml_vet/ml_vet.csproj +++ b/ml_vet/ml_vet.csproj @@ -4,7 +4,7 @@ netstandard2.1 x64 ViveEyeTracking - 1.0.0 + 1.0.1 SDraw SDraw ViveEyeTracking diff --git a/ml_vet/resources/mod_menu.js b/ml_vet/resources/mod_menu.js index b69fc21..b845796 100644 --- a/ml_vet/resources/mod_menu.js +++ b/ml_vet/resources/mod_menu.js @@ -15,10 +15,28 @@

Requires Vive Face tracking restart at first.

+ +
+
Gaze smoothing:
+
+
+
+
+ +
+
Debug gizmos:
+
+
+
+
`; document.getElementById('settings-implementation').appendChild(l_block); // Toggles for (let l_toggle of l_block.querySelectorAll('.inp_toggle')) modsExtension.addSetting('VET', l_toggle.id, modsExtension.createToggle(l_toggle, 'OnToggleUpdate_VET')); + + // Sliders + for (let l_slider of l_block.querySelectorAll('.inp_slider')) + modsExtension.addSetting('VET', l_slider.id, modsExtension.createSlider(l_slider, 'OnSliderUpdate_VET')); }