New mod: Vive Eye Tracking

2025-09-03 02:19:23 +00:00 · 2025-05-21 02:15:01 +03:00 · 2025-05-21 02:15:01 +03:00 · a72af43f69
commit a72af43f69
parent de463d2236
10 changed files with 432 additions and 0 deletions
--- a/README.md
+++ b/README.md
@ -14,4 +14,5 @@ Merged set of MelonLoader mods for ChilloutVR.
 |[Player Pick Up](/ml_ppu/README.md)|1.0.0 [:arrow_down:](../../releases/latest/download/PlayerPickUp.dll)|
 |[Player Ragdoll Mod](/ml_prm/README.md)|1.2.3 [:arrow_down:](../../releases/latest/download/PlayerRagdollMod.dll)|
 |[Video Player Cookies](/ml_vpc/README.md)|1.0.1 [:arrow_down:](../../releases/latest/download/VideoPlayerCookies.dll)|
+|[Vive Eye Tracking](/ml_vet/README.md)|1.0.0 [:arrow_down:](../../releases/latest/download/ViveEyeTracking.dll)|
 |[Vive Extended Input](/ml_vei/README.md)|1.1.1 [:arrow_down:](../../releases/latest/download/ViveExtendedInput.dll)|
--- a/ml_mods_cvr.sln
+++ b/ml_mods_cvr.sln
@ -30,6 +30,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ml_vpc", "ml_vpc\ml_vpc.csp
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ml_ppu", "ml_ppu\ml_ppu.csproj", "{F16DF16B-D127-4A2A-81FF-2FD80F320E64}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ml_vet", "ml_vet\ml_vet.csproj", "{8DB32590-FC5B-46A8-9747-344E86B18ACF}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@ -77,6 +79,10 @@ Global
 		{F16DF16B-D127-4A2A-81FF-2FD80F320E64}.Debug|x64.Build.0 = Debug|x64
 		{F16DF16B-D127-4A2A-81FF-2FD80F320E64}.Release|x64.ActiveCfg = Release|x64
 		{F16DF16B-D127-4A2A-81FF-2FD80F320E64}.Release|x64.Build.0 = Release|x64
+		{8DB32590-FC5B-46A8-9747-344E86B18ACF}.Debug|x64.ActiveCfg = Debug|x64
+		{8DB32590-FC5B-46A8-9747-344E86B18ACF}.Debug|x64.Build.0 = Debug|x64
+		{8DB32590-FC5B-46A8-9747-344E86B18ACF}.Release|x64.ActiveCfg = Release|x64
+		{8DB32590-FC5B-46A8-9747-344E86B18ACF}.Release|x64.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
--- a/ml_vet/Main.cs
+++ b/ml_vet/Main.cs
@ -0,0 +1,171 @@
+using ABI_RC.Core;
+using ABI_RC.Core.Player;
+using ABI_RC.Core.Player.EyeMovement;
+using ABI_RC.Core.Savior;
+using ABI_RC.Systems.FaceTracking;
+using ABI_RC.Systems.FaceTracking.Impl;
+using System.Reflection;
+using System.Threading;
+using UnityEngine;
+using ViveSR.anipal.Eye;
+
+namespace ml_vet
+{
+    public class ViveEyeTracking : MelonLoader.MelonMod
+    {
+        const string c_gameSettingName = "ImplementationVRViveFaceTracking";
+
+        static ViveEyeTracking ms_instance = null;
+
+        readonly object m_lock = new object();
+        EyeData_v2 m_threadEyeData; // Shared between threads
+        SingleEyeData m_combinedEyeData;
+        SingleEyeData m_leftEyeData;
+        SingleEyeData m_rightEyeData;
+
+        Vector2 m_openness = Vector2.one;
+        Vector3 m_gazeDirection = Vector3.forward;
+
+        bool m_faceTrackingEnabled = false;
+
+        public override void OnInitializeMelon()
+        {
+            ms_instance = this;
+
+            Settings.Init();
+
+            HarmonyInstance.Patch(
+                typeof(SRanipalTrackingModule).GetMethod(nameof(SRanipalTrackingModule.Initialize), BindingFlags.Public | BindingFlags.Instance),
+                new HarmonyLib.HarmonyMethod(typeof(ViveEyeTracking).GetMethod(nameof(SRanipalTrackingModuleUpdate_Prefix), BindingFlags.NonPublic | BindingFlags.Static))
+            );
+            HarmonyInstance.Patch(typeof(EyeMovementController).GetMethod("Update", BindingFlags.NonPublic | BindingFlags.Instance),
+                null,
+                new HarmonyLib.HarmonyMethod(typeof(ViveEyeTracking).GetMethod(nameof(EyeMovementControllerUpdate_Postfix), BindingFlags.NonPublic | BindingFlags.Static))
+            );
+            HarmonyInstance.Patch(
+                typeof(PlayerSetup).GetMethod("UpdatePlayerAvatarMovementData", BindingFlags.Instance | BindingFlags.NonPublic),
+                null,
+                new HarmonyLib.HarmonyMethod(typeof(ViveEyeTracking).GetMethod(nameof(OnPlayerAvatarMovementDataUpdate_Postfix), BindingFlags.Static | BindingFlags.NonPublic))
+            );
+
+            MelonLoader.MelonCoroutines.Start(WaitForRootLogic(HarmonyInstance));
+        }
+
+        System.Collections.IEnumerator WaitForRootLogic(HarmonyLib.Harmony p_harmony)
+        {
+            while(RootLogic.Instance == null)
+                yield return null;
+            while(MetaPort.Instance == null)
+                yield return null;
+
+            p_harmony.Patch(
+                typeof(FaceTrackingManager).GetMethod(nameof(FaceTrackingManager.SubmitNewEyeData), BindingFlags.Public | BindingFlags.Instance),
+                null,
+                new HarmonyLib.HarmonyMethod(typeof(ViveEyeTracking).GetMethod(nameof(FaceTrackingManagerSubmitNewEyeData_Postfix), BindingFlags.Static | BindingFlags.NonPublic))
+            );
+
+            m_faceTrackingEnabled = MetaPort.Instance.settings.GetSettingsBool(c_gameSettingName);
+            MetaPort.Instance.settings.settingBoolChanged.AddListener(this.OnGameSettingBoolChanged);
+        }
+
+        public override void OnDeinitializeMelon()
+        {
+            ms_instance = null;
+        }
+
+        public override void OnUpdate()
+        {
+            if(Settings.Enabled && m_faceTrackingEnabled && Utils.IsInVR())
+            {
+                if(Monitor.TryEnter(m_lock))
+                {
+                    m_combinedEyeData = m_threadEyeData.verbose_data.combined.eye_data;
+                    m_leftEyeData = m_threadEyeData.verbose_data.left;
+                    m_rightEyeData = m_threadEyeData.verbose_data.right;
+                    Monitor.Exit(m_lock);
+                }
+
+                if(m_combinedEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_GAZE_DIRECTION_VALIDITY))
+                {
+                    m_gazeDirection = m_combinedEyeData.gaze_direction_normalized;
+                    m_gazeDirection.x *= -1f;
+                }
+
+                if(m_leftEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_EYE_OPENNESS_VALIDITY))
+                    m_openness.x = m_leftEyeData.eye_openness;
+
+                if(m_rightEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_EYE_OPENNESS_VALIDITY))
+                    m_openness.y = m_rightEyeData.eye_openness;
+            }
+        }
+
+        void OnGameSettingBoolChanged(string p_name, bool p_state)
+        {
+            if(p_name == c_gameSettingName)
+                m_faceTrackingEnabled = p_state;
+        }
+
+        bool IsReadyForUpdate() => (Settings.Enabled && m_faceTrackingEnabled && FaceTrackingManager.Instance.IsEyeDataAvailable());
+
+        // Patches
+        static void SRanipalTrackingModuleUpdate_Prefix(ref bool useEye, bool useLip)
+        {
+            // Hijack face tracking module to be eye tracking module as well, because it is SRanipal too and can initialize without issues
+            if(useLip && Settings.Enabled)
+                useEye = true;
+        }
+
+        static void FaceTrackingManagerSubmitNewEyeData_Postfix(ref EyeData_v2 eyeData) => ms_instance?.OnEyeDataPostSubmit(ref eyeData);
+        void OnEyeDataPostSubmit(ref EyeData_v2 p_data)
+        {
+            // This is called not in main thread
+            try
+            {
+                Monitor.Enter(m_lock);
+                m_threadEyeData = p_data;
+                Monitor.Exit(m_lock);
+            }
+            catch(System.Exception e)
+            {
+                MelonLoader.MelonLogger.Error(e);
+            }
+        }
+
+        static void EyeMovementControllerUpdate_Postfix(ref EyeMovementController __instance) => ms_instance?.OnEyeMovementControllerPostUpdate(__instance);
+        void OnEyeMovementControllerPostUpdate(EyeMovementController p_controller)
+        {
+            try
+            {
+                if(p_controller.IsLocal && IsReadyForUpdate())
+                {
+                    p_controller.manualBlinking = true;
+                    p_controller.blinkProgress = 1f - Mathf.Clamp01((m_openness.x + m_openness.y) * 0.5f);
+
+                    p_controller.manualViewTarget = true;
+                    p_controller.targetViewPosition = p_controller.ViewPointTransform.position + p_controller.ViewPointTransform.rotation * (m_gazeDirection * 5f);
+                }
+            }
+            catch(System.Exception e)
+            {
+                MelonLoader.MelonLogger.Error(e);
+            }
+        }
+
+        static void OnPlayerAvatarMovementDataUpdate_Postfix(ref PlayerSetup __instance, PlayerAvatarMovementData ____playerAvatarMovementData) => ms_instance?.OnPlayerAvatarMovementDataPostUpdate(__instance, ____playerAvatarMovementData);
+        void OnPlayerAvatarMovementDataPostUpdate(PlayerSetup p_instance, PlayerAvatarMovementData p_data)
+        {
+            try
+            {
+                if(IsReadyForUpdate() && (p_instance.EyeMovementController != null))
+                {
+                    p_data.EyeTrackingBlinkProgressLeft = 1f - Mathf.Clamp01(m_openness.x);
+                    p_data.EyeTrackingBlinkProgressRight = 1f - Mathf.Clamp01(m_openness.y);
+                }
+            }
+            catch(System.Exception e)
+            {
+                MelonLoader.MelonLogger.Error(e);
+            }
+        }
+    }
+}
--- a/ml_vet/Properties/AssemblyInfo.cs
+++ b/ml_vet/Properties/AssemblyInfo.cs
@ -0,0 +1,4 @@
+[assembly: MelonLoader.MelonInfo(typeof(ml_vet.ViveEyeTracking), "ViveEyeTracking", "1.0.0", "SDraw", "https://github.com/SDraw/ml_mods_cvr")]
+[assembly: MelonLoader.MelonGame(null, "ChilloutVR")]
+[assembly: MelonLoader.MelonPlatform(MelonLoader.MelonPlatformAttribute.CompatiblePlatforms.WINDOWS_X64)]
+[assembly: MelonLoader.MelonPlatformDomain(MelonLoader.MelonPlatformDomainAttribute.CompatibleDomains.MONO)]
--- a/ml_vet/README.md
+++ b/ml_vet/README.md
@ -0,0 +1,21 @@
+# Vive Eye Tracking
+This mod complements functionality of in-game SRanipal face tracking module and makes it work additionaly as eye tracking module.
+
+# Why?
+* Game has unfinished/unused and forcibly disabled eye tracking code that actually uses SRanipal API instead of TobiiXR.
+* Implemented native TobiiXR eye tracking is very unreliable. It freezes main thread with 66% chance at game launch and has limited detection of eyes openness.
+
+# Benefits?
+* SRanipal API supports ranged eyes openness detection, unlike TobiiXR. You can finally smirk now.
+* There is no main thread freeze at game launch.
+* Semi-ready for future game update with separated eyes blinking.
+
+# Installation
+* Install [latest MelonLoader](https://github.com/LavaGang/MelonLoader)
+* Get [latest release DLL](../../../releases/latest):
+  * Put `ViveEyeTracking.dll` in `Mods` folder of game
+  
+# Usage
+Available mod's settings in `Settings - Implementation - Vive Eye Tracking`:
+* **Enable eye tracking:** eye tracking state; `false` by default.
+  * Note: If Vive face tracking is already running, enabling requires Vive face tracking restart.
--- a/ml_vet/ResourcesHandler.cs
+++ b/ml_vet/ResourcesHandler.cs
@ -0,0 +1,30 @@
+using System;
+using System.IO;
+using System.Reflection;
+
+namespace ml_vet
+{
+    static class ResourcesHandler
+    {
+        readonly static string ms_namespace = typeof(ResourcesHandler).Namespace;
+
+        public static string GetEmbeddedResource(string p_name)
+        {
+            string l_result = "";
+            Assembly l_assembly = Assembly.GetExecutingAssembly();
+
+            try
+            {
+                Stream l_libraryStream = l_assembly.GetManifestResourceStream(ms_namespace + ".resources." + p_name);
+                StreamReader l_streadReader = new StreamReader(l_libraryStream);
+                l_result = l_streadReader.ReadToEnd();
+            }
+            catch(Exception e)
+            {
+                MelonLoader.MelonLogger.Warning(e);
+            }
+
+            return l_result;
+        }
+    }
+}
--- a/ml_vet/Settings.cs
+++ b/ml_vet/Settings.cs
@ -0,0 +1,90 @@
+using ABI_RC.Core.InteractionSystem;
+using System;
+using System.Collections.Generic;
+
+namespace ml_vet
+{
+    static class Settings
+    {
+        internal class SettingEvent<T>
+        {
+            event Action<T> m_action;
+            public void AddListener(Action<T> p_listener) => m_action += p_listener;
+            public void RemoveListener(Action<T> p_listener) => m_action -= p_listener;
+            public void Invoke(T p_value) => m_action?.Invoke(p_value);
+        }
+
+        enum ModSetting
+        {
+            Enabled = 0
+        }
+
+        public static bool Enabled { get; private set; } = false;
+
+        static MelonLoader.MelonPreferences_Category ms_category = null;
+        static List<MelonLoader.MelonPreferences_Entry> ms_entries = null;
+
+        public static readonly SettingEvent<bool> OnEnabledChanged = new SettingEvent<bool>();
+
+        internal static void Init()
+        {
+            ms_category = MelonLoader.MelonPreferences.CreateCategory("VET", null, true);
+
+            ms_entries = new List<MelonLoader.MelonPreferences_Entry>()
+            {
+                ms_category.CreateEntry(ModSetting.Enabled.ToString(), Enabled),
+            };
+
+            Enabled = (bool)ms_entries[(int)ModSetting.Enabled].BoxedValue;
+
+            MelonLoader.MelonCoroutines.Start(WaitMainMenuUi());
+        }
+
+        static System.Collections.IEnumerator WaitMainMenuUi()
+        {
+            while(ViewManager.Instance == null)
+                yield return null;
+            while(ViewManager.Instance.gameMenuView == null)
+                yield return null;
+            while(ViewManager.Instance.gameMenuView.Listener == null)
+                yield return null;
+
+            ViewManager.Instance.gameMenuView.Listener.ReadyForBindings += () =>
+            {
+                ViewManager.Instance.gameMenuView.View.BindCall("OnToggleUpdate_" + ms_category.Identifier, new Action<string, string>(OnToggleUpdate));
+            };
+            ViewManager.Instance.gameMenuView.Listener.FinishLoad += (_) =>
+            {
+                ViewManager.Instance.gameMenuView.View.ExecuteScript(ResourcesHandler.GetEmbeddedResource("mods_extension.js"));
+                ViewManager.Instance.gameMenuView.View.ExecuteScript(ResourcesHandler.GetEmbeddedResource("mod_menu.js"));
+                foreach(var l_entry in ms_entries)
+                    ViewManager.Instance.gameMenuView.View.TriggerEvent("updateModSetting", ms_category.Identifier, l_entry.DisplayName, l_entry.GetValueAsString());
+            };
+        }
+
+        static void OnToggleUpdate(string p_name, string p_value)
+        {
+            try
+            {
+                if(Enum.TryParse(p_name, out ModSetting l_setting) && bool.TryParse(p_value,out bool l_value))
+                {
+                    switch(l_setting)
+                    {
+                        case ModSetting.Enabled:
+                        {
+                            Enabled = l_value;
+                            OnEnabledChanged.Invoke(Enabled);
+                        }
+                        break;
+                    }
+
+                    ms_entries[(int)l_setting].BoxedValue = l_value;
+                }
+            }
+            catch(Exception e)
+            {
+                MelonLoader.MelonLogger.Error(e);
+            }
+        }
+    }
+}
--- a/ml_vet/Utils.cs
+++ b/ml_vet/Utils.cs
@ -0,0 +1,15 @@
+using ABI_RC.Core.Savior;
+using ABI_RC.Core.UI;
+using System.Reflection;
+
+namespace ml_vet
+{
+    public static class Utils
+    {
+        static readonly FieldInfo ms_view = typeof(CohtmlControlledViewWrapper).GetField("_view", BindingFlags.Instance | BindingFlags.NonPublic);
+
+        public static void ExecuteScript(this CohtmlControlledViewWrapper p_instance, string p_script) => (ms_view?.GetValue(p_instance) as cohtml.Net.View)?.ExecuteScript(p_script);
+
+        public static bool IsInVR() => ((MetaPort.Instance != null) && MetaPort.Instance.isUsingVr);
+    }
+}
--- a/ml_vet/ml_vet.csproj
+++ b/ml_vet/ml_vet.csproj
@ -0,0 +1,70 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.1</TargetFramework>
+    <Platforms>x64</Platforms>
+    <PackageId>ViveEyeTracking</PackageId>
+    <Version>1.0.0</Version>
+    <Authors>SDraw</Authors>
+    <Company>SDraw</Company>
+    <Product>ViveEyeTracking</Product>
+    <AssemblyName>ViveEyeTracking</AssemblyName>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <DebugType>embedded</DebugType>
+    <DebugSymbols>true</DebugSymbols>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <None Remove="resources\mod_menu.js" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <EmbeddedResource Include="..\js\mods_extension.js" Link="resources\mods_extension.js" />
+    <EmbeddedResource Include="resources\mod_menu.js" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Reference Include="0Harmony">
+      <HintPath>D:\games\Steam\steamapps\common\ChilloutVR\MelonLoader\net35\0Harmony.dll</HintPath>
+      <Private>false</Private>
+      <SpecificVersion>false</SpecificVersion>
+    </Reference>
+    <Reference Include="Assembly-CSharp">
+      <HintPath>D:\games\Steam\steamapps\common\ChilloutVR\ChilloutVR_Data\Managed\Assembly-CSharp.dll</HintPath>
+      <Private>false</Private>
+      <SpecificVersion>false</SpecificVersion>
+    </Reference>
+    <Reference Include="cohtml.Net">
+      <HintPath>D:\games\Steam\steamapps\common\ChilloutVR\ChilloutVR_Data\Managed\cohtml.Net.dll</HintPath>
+      <Private>false</Private>
+      <SpecificVersion>false</SpecificVersion>
+    </Reference>
+    <Reference Include="Cohtml.Runtime">
+      <HintPath>D:\games\Steam\steamapps\common\ChilloutVR\ChilloutVR_Data\Managed\Cohtml.Runtime.dll</HintPath>
+      <Private>false</Private>
+      <SpecificVersion>false</SpecificVersion>
+    </Reference>
+    <Reference Include="MelonLoader">
+      <HintPath>D:\games\Steam\steamapps\common\ChilloutVR\MelonLoader\net35\MelonLoader.dll</HintPath>
+      <Private>false</Private>
+      <SpecificVersion>false</SpecificVersion>
+    </Reference>
+    <Reference Include="UnityEngine">
+      <HintPath>D:\games\Steam\steamapps\common\ChilloutVR\ChilloutVR_Data\Managed\UnityEngine.dll</HintPath>
+      <Private>false</Private>
+      <SpecificVersion>false</SpecificVersion>
+    </Reference>
+    <Reference Include="UnityEngine.CoreModule">
+      <HintPath>D:\Games\Steam\steamapps\common\ChilloutVR\ChilloutVR_Data\Managed\UnityEngine.CoreModule.dll</HintPath>
+      <Private>false</Private>
+      <SpecificVersion>false</SpecificVersion>
+    </Reference>
+  </ItemGroup>
+
+  <Target Name="PostBuild" AfterTargets="PostBuildEvent">
+    <Exec Command="copy /y &quot;$(TargetPath)&quot; &quot;D:\Games\Steam\steamapps\common\ChilloutVR\Mods\&quot;" />
+  </Target>
+
+</Project>
--- a/ml_vet/resources/mod_menu.js
+++ b/ml_vet/resources/mod_menu.js
@ -0,0 +1,24 @@
+{
+    let l_block = document.createElement('div');
+    l_block.innerHTML = `
+        <div class ="settings-subcategory">
+            <div class ="subcategory-name">Vive Eye Tracking</div>
+            <div class ="subcategory-description"></div>
+        </div>
+
+        <div class ="row-wrapper">
+            <div class ="option-caption">Enable eye tracking: </div>
+            <div class ="option-input">
+                <div id="Enabled" class ="inp_toggle no-scroll" data-current="true"></div>
+            </div>
+        </div>
+        <div class="row-wrapper">
+            <p>Requires Vive Face tracking restart at first.</p>
+        </div>
+    `;
+    document.getElementById('settings-implementation').appendChild(l_block);
+
+    // Toggles
+    for (let l_toggle of l_block.querySelectorAll('.inp_toggle'))
+        modsExtension.addSetting('VET', l_toggle.id, modsExtension.createToggle(l_toggle, 'OnToggleUpdate_VET'));
+}