ml_vet: archive

This commit is contained in:
Hanz 2025-08-15 06:37:19 -05:00
parent 1b3eacd915
commit 2b97399a54
10 changed files with 0 additions and 7 deletions

274
archived/ml_vet/Main.cs Normal file
View file

@ -0,0 +1,274 @@
using ABI_RC.Core;
using ABI_RC.Core.Player;
using ABI_RC.Core.Player.EyeMovement;
using ABI_RC.Core.Savior;
using ABI_RC.Systems.FaceTracking;
using ABI_RC.Systems.FaceTracking.Impl;
using ABI_RC.Systems.RuntimeDebug;
using System.Reflection;
using System.Threading;
using UnityEngine;
using ViveSR.anipal.Eye;
namespace ml_vet
{
public class ViveEyeTracking : MelonLoader.MelonMod
{
const string c_gameSettingName = "ImplementationVRViveFaceTracking";
static ViveEyeTracking ms_instance = null;
readonly object m_lock = new object();
EyeData_v2 m_threadEyeData; // Shared between threads
SingleEyeData m_combinedEyeData;
SingleEyeData m_leftEyeData;
SingleEyeData m_rightEyeData;
Vector2 m_openness = Vector2.one;
Vector3 m_leftEyeGaze = new Vector3(1f, 0f, 1f).normalized;
Vector3 m_rightEyeGaze = new Vector3(-1f, 0f, 1f).normalized;
Vector3 m_combinedGaze = Vector3.forward;
Vector3 m_leftEyeOrigin = new Vector3(-0.025f, 0f, 0f);
Vector3 m_rightEyeOrigin = new Vector3(0.025f, 0f, 0f);
Vector3 m_cominedGazeOrigin = Vector3.zero;
bool m_leftEyeGazeValid = false;
bool m_rightEyeGazeValid = false;
bool m_combinedGazeValid = false;
Vector3 m_gazePoint = Vector3.forward;
Vector3 m_gazeVelocity = Vector3.zero;
bool m_faceTrackingEnabled = false;
public override void OnInitializeMelon()
{
ms_instance = this;
Settings.Init();
HarmonyInstance.Patch(
typeof(SRanipalTrackingModule).GetMethod(nameof(SRanipalTrackingModule.Initialize), BindingFlags.Public | BindingFlags.Instance),
new HarmonyLib.HarmonyMethod(typeof(ViveEyeTracking).GetMethod(nameof(SRanipalTrackingModuleUpdate_Prefix), BindingFlags.NonPublic | BindingFlags.Static))
);
HarmonyInstance.Patch(typeof(EyeMovementController).GetMethod("Update", BindingFlags.NonPublic | BindingFlags.Instance),
null,
new HarmonyLib.HarmonyMethod(typeof(ViveEyeTracking).GetMethod(nameof(EyeMovementControllerUpdate_Postfix), BindingFlags.NonPublic | BindingFlags.Static))
);
MelonLoader.MelonCoroutines.Start(WaitForRootLogic(HarmonyInstance));
}
System.Collections.IEnumerator WaitForRootLogic(HarmonyLib.Harmony p_harmony)
{
while(RootLogic.Instance == null)
yield return null;
while(MetaPort.Instance == null)
yield return null;
p_harmony.Patch(
typeof(FaceTrackingManager).GetMethod(nameof(FaceTrackingManager.SubmitNewEyeData), BindingFlags.Public | BindingFlags.Instance),
null,
new HarmonyLib.HarmonyMethod(typeof(ViveEyeTracking).GetMethod(nameof(FaceTrackingManagerSubmitNewEyeData_Postfix), BindingFlags.Static | BindingFlags.NonPublic))
);
m_faceTrackingEnabled = MetaPort.Instance.settings.GetSettingsBool(c_gameSettingName);
MetaPort.Instance.settings.settingBoolChanged.AddListener(this.OnGameSettingBoolChanged);
}
public override void OnDeinitializeMelon()
{
ms_instance = null;
}
public override void OnUpdate()
{
if(Settings.Enabled && m_faceTrackingEnabled && Utils.IsInVR())
{
if(Monitor.TryEnter(m_lock))
{
m_combinedEyeData = m_threadEyeData.verbose_data.combined.eye_data;
m_leftEyeData = m_threadEyeData.verbose_data.left;
m_rightEyeData = m_threadEyeData.verbose_data.right;
Monitor.Exit(m_lock);
}
m_combinedGazeValid = m_combinedEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_GAZE_DIRECTION_VALIDITY);
if(m_combinedGazeValid)
{
m_combinedGaze = m_combinedEyeData.gaze_direction_normalized;
m_combinedGaze.x *= -1f;
}
if(m_combinedEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_GAZE_ORIGIN_VALIDITY))
{
m_cominedGazeOrigin = m_combinedEyeData.gaze_origin_mm * 0.001f;
m_cominedGazeOrigin.x *= -1f;
}
m_leftEyeGazeValid = m_leftEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_GAZE_DIRECTION_VALIDITY);
if(m_leftEyeGazeValid)
{
m_leftEyeGaze = m_leftEyeData.gaze_direction_normalized;
m_leftEyeGaze.x *= -1f;
}
if(m_leftEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_GAZE_ORIGIN_VALIDITY))
{
m_leftEyeOrigin = m_leftEyeData.gaze_origin_mm * 0.001f;
m_leftEyeOrigin.x *= -1f;
}
if(m_leftEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_EYE_OPENNESS_VALIDITY))
m_openness.x = m_leftEyeData.eye_openness;
m_rightEyeGazeValid = m_rightEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_GAZE_DIRECTION_VALIDITY);
if(m_rightEyeGazeValid)
{
m_rightEyeGaze = m_rightEyeData.gaze_direction_normalized;
m_rightEyeGaze.x *= -1f;
}
if(m_rightEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_GAZE_ORIGIN_VALIDITY))
{
m_rightEyeOrigin = m_rightEyeData.gaze_origin_mm * 0.001f;
m_rightEyeOrigin.x *= -1f;
}
if(m_rightEyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_EYE_OPENNESS_VALIDITY))
m_openness.y = m_rightEyeData.eye_openness;
}
}
void OnGameSettingBoolChanged(string p_name, bool p_state)
{
if(p_name == c_gameSettingName)
m_faceTrackingEnabled = p_state;
}
bool IsReadyForUpdate() => (Settings.Enabled && m_faceTrackingEnabled && FaceTrackingManager.Instance.IsEyeDataAvailable());
// Patches
static void SRanipalTrackingModuleUpdate_Prefix(ref bool useEye, bool useLip)
{
// Hijack face tracking module to be eye tracking module as well, because it is SRanipal too and can initialize without issues
if(useLip && Settings.Enabled)
useEye = true;
}
static void FaceTrackingManagerSubmitNewEyeData_Postfix(ref EyeData_v2 eyeData) => ms_instance?.OnEyeDataPostSubmit(ref eyeData);
void OnEyeDataPostSubmit(ref EyeData_v2 p_data)
{
// This is called not in main thread
try
{
Monitor.Enter(m_lock);
m_threadEyeData = p_data;
Monitor.Exit(m_lock);
}
catch(System.Exception e)
{
MelonLoader.MelonLogger.Error(e);
}
}
static void EyeMovementControllerUpdate_Postfix(ref EyeMovementController __instance) => ms_instance?.OnEyeMovementControllerPostUpdate(__instance);
void OnEyeMovementControllerPostUpdate(EyeMovementController p_controller)
{
try
{
if(p_controller.IsLocal && IsReadyForUpdate())
{
p_controller.manualBlinking = true;
#if NIGHTLY_BUILD
p_controller.blinkProgressLeft = 1f - m_openness.x;
p_controller.blinkProgressRight = 1f - m_openness.y;
#else
p_controller.blinkProgress = 1f - (m_openness.x + m_openness.y) * 0.5f;
#endif
p_controller.manualViewTarget = true;
Vector3 l_gazePoint = m_gazePoint;
float l_playspaceScale = PlayerSetup.Instance.GetPlaySpaceScale();
if(m_leftEyeGazeValid && m_rightEyeGazeValid)
{
// https://en.wikipedia.org/wiki/Line%E2%80%93line_intersection
// Projecting on OXZ plane to intersect eyes
float x1 = m_leftEyeOrigin.x;
float x2 = m_leftEyeOrigin.x + m_leftEyeGaze.x;
float y1 = m_leftEyeOrigin.z;
float y2 = m_leftEyeOrigin.z + m_leftEyeGaze.z;
float x3 = m_rightEyeOrigin.x;
float x4 = m_rightEyeOrigin.x + m_rightEyeGaze.x;
float y3 = m_rightEyeOrigin.z;
float y4 = m_rightEyeOrigin.z + m_rightEyeGaze.z;
float l_det = (x1 - x2) * (y3 - y4) - (y1 - y2) * (x3 - x4);
Vector3 l_combineGazeOrigin = Vector3.Lerp(m_leftEyeOrigin, m_rightEyeOrigin, 0.5f); // SRanipal's combined gaze origin is unstable
if(!Mathf.Approximately(l_det, 0f))
{
float l_detZ = (x1 * y2 - y1 * x2) * (y3 - y4) - (y1 - y2) * (x3 * y4 - y3 * x4);
float l_posZ = l_detZ / l_det;
float l_thetaLeft = (l_posZ - m_leftEyeOrigin.z) / m_leftEyeGaze.z;
Vector3 l_leftEyePoint = m_leftEyeOrigin + l_thetaLeft * m_leftEyeGaze;
float l_thetaRight = (l_posZ - m_rightEyeOrigin.z) / m_rightEyeGaze.z;
Vector3 l_rightEyePoint = m_rightEyeOrigin + l_thetaRight * m_rightEyeGaze;
Vector3 l_midPoint = Vector3.Lerp(l_leftEyePoint, l_rightEyePoint, 0.5f);
if(l_midPoint.z > l_combineGazeOrigin.z)
{
Vector3 l_resultDir = l_midPoint - l_combineGazeOrigin;
l_resultDir = Vector3.ClampMagnitude(l_resultDir, 1f);
l_gazePoint = (l_combineGazeOrigin + l_resultDir) * l_playspaceScale;
}
else
l_gazePoint = (l_combineGazeOrigin + m_combinedGaze) * l_playspaceScale;
}
else
l_gazePoint = (l_combineGazeOrigin + m_combinedGaze) * l_playspaceScale;
}
else
{
if(m_leftEyeGazeValid)
l_gazePoint = (m_leftEyeOrigin + m_leftEyeGaze) * l_playspaceScale;
if(m_rightEyeGazeValid)
l_gazePoint = (m_rightEyeGaze + m_rightEyeGaze) * l_playspaceScale;
}
l_gazePoint = Vector3.SmoothDamp(m_gazePoint, l_gazePoint, ref m_gazeVelocity, Settings.Smoothing);
p_controller.targetViewPosition = p_controller.ViewPointTransform.position + p_controller.ViewPointTransform.rotation * l_gazePoint;
if(m_rightEyeGazeValid || m_leftEyeGazeValid)
m_gazePoint = l_gazePoint;
if(Settings.Debug)
{
Vector3 l_pos = p_controller.ViewPointTransform.position;
Quaternion l_rot = p_controller.ViewPointTransform.rotation;
RuntimeGizmos.DrawArrowFromTo(
l_pos + l_rot * (m_leftEyeOrigin * l_playspaceScale),
l_pos + l_rot * ((m_leftEyeOrigin + (m_leftEyeGazeValid ? m_leftEyeGaze : Vector3.forward)) * l_playspaceScale),
0.01f * l_playspaceScale, Color.blue, CVRLayers.Default, 0.25f
);
RuntimeGizmos.DrawArrowFromTo(
l_pos + l_rot * (m_rightEyeOrigin * l_playspaceScale),
l_pos + l_rot * ((m_rightEyeOrigin + (m_rightEyeGazeValid ? m_rightEyeGaze : Vector3.forward)) * l_playspaceScale),
0.01f * l_playspaceScale, Color.blue, CVRLayers.Default, 0.25f
);
RuntimeGizmos.DrawArrowFromTo(
l_pos + l_rot * (m_cominedGazeOrigin * l_playspaceScale),
l_pos + l_rot * ((m_cominedGazeOrigin + (m_combinedGazeValid ? m_combinedGaze : Vector3.forward)) * l_playspaceScale),
0.01f * l_playspaceScale, Color.red, CVRLayers.Default, 0.25f
);
RuntimeGizmos.DrawSphere(p_controller.targetViewPosition, 0.01f * l_playspaceScale, Color.cyan, CVRLayers.Default, 0.25f);
}
}
}
catch(System.Exception e)
{
MelonLoader.MelonLogger.Error(e);
}
}
}
}

View file

@ -0,0 +1,26 @@
using System.Reflection;
using ml_vet.Properties;
[assembly: AssemblyVersion(AssemblyInfoParams.Version)]
[assembly: AssemblyFileVersion(AssemblyInfoParams.Version)]
[assembly: AssemblyInformationalVersion(AssemblyInfoParams.Version)]
[assembly: AssemblyTitle(nameof(ml_vet.ViveEyeTracking))]
[assembly: AssemblyCompany(AssemblyInfoParams.Author)]
[assembly: AssemblyProduct(nameof(ml_vet.ViveEyeTracking))]
[assembly: MelonLoader.MelonInfo(
typeof(ml_vet.ViveEyeTracking),
nameof(ml_vet.ViveEyeTracking),
AssemblyInfoParams.Version,
AssemblyInfoParams.Author,
"https://github.com/SDraw/ml_mods_cvr"
)]
[assembly: MelonLoader.MelonGame(null, "ChilloutVR")]
[assembly: MelonLoader.MelonPlatform(MelonLoader.MelonPlatformAttribute.CompatiblePlatforms.WINDOWS_X64)]
[assembly: MelonLoader.MelonPlatformDomain(MelonLoader.MelonPlatformDomainAttribute.CompatibleDomains.MONO)]
namespace ml_vet.Properties;
internal static class AssemblyInfoParams {
public const string Version = "1.0.2";
public const string Author = "SDraw";
}

26
archived/ml_vet/README.md Normal file
View file

@ -0,0 +1,26 @@
# Vive Eye Tracking
This mod complements functionality of in-game SRanipal face tracking module and makes it work additionaly as eye tracking module.
# Why?
* Game has unfinished/unused and forcibly disabled eye tracking code that actually uses SRanipal API instead of TobiiXR.
* Implemented native TobiiXR eye tracking is very unreliable. It freezes main thread with 66% chance at game launch and has limited detection of eyes openness.
# Benefits?
* SRanipal API supports ranged eyes openness detection, unlike TobiiXR. You can finally squint now.
* There is no main thread freeze at game launch.
* Ready for future game update with separated eyes blinking (currently only nightly).
# Installation
* Install [latest MelonLoader](https://github.com/LavaGang/MelonLoader)
* Get [latest release DLL](../../../releases/latest):
* Put `ViveEyeTracking.dll` in `Mods` folder of game
# Usage
Available mod's settings in `Settings - Implementation - Vive Eye Tracking`:
* **Enable eye tracking:** eye tracking state; `true` by default.
* Note: If Vive face tracking is already running, enabling requires Vive face tracking restart.
* **Gaze smoothing:** smoothing of gaze point; 5 by default.
* **Debug gizmos:** show debug lines of SRanipal gazes directions and calculated gaze point; `false` by default.
# Notes
* Made primarily for Vive Pro Eye and SRanipal tracking.

View file

@ -0,0 +1,30 @@
using System;
using System.IO;
using System.Reflection;
namespace ml_vet
{
static class ResourcesHandler
{
readonly static string ms_namespace = typeof(ResourcesHandler).Namespace;
public static string GetEmbeddedResource(string p_name)
{
string l_result = "";
Assembly l_assembly = Assembly.GetExecutingAssembly();
try
{
Stream l_libraryStream = l_assembly.GetManifestResourceStream(ms_namespace + ".resources." + p_name);
StreamReader l_streadReader = new StreamReader(l_libraryStream);
l_result = l_streadReader.ReadToEnd();
}
catch(Exception e)
{
MelonLoader.MelonLogger.Warning(e);
}
return l_result;
}
}
}

141
archived/ml_vet/Settings.cs Normal file
View file

@ -0,0 +1,141 @@
using ABI_RC.Core.InteractionSystem;
using System;
using System.Collections.Generic;
namespace ml_vet
{
static class Settings
{
internal class SettingEvent<T>
{
event Action<T> m_action;
public void AddListener(Action<T> p_listener) => m_action += p_listener;
public void RemoveListener(Action<T> p_listener) => m_action -= p_listener;
public void Invoke(T p_value) => m_action?.Invoke(p_value);
}
enum ModSetting
{
Enabled = 0,
Smoothing,
Debug
}
public static bool Enabled { get; private set; } = true;
public static float Smoothing { get; private set; } = 0f;
public static bool Debug { get; private set; } = false;
static MelonLoader.MelonPreferences_Category ms_category = null;
static List<MelonLoader.MelonPreferences_Entry> ms_entries = null;
public static readonly SettingEvent<bool> OnEnabledChanged = new SettingEvent<bool>();
public static readonly SettingEvent<float> OnSmoothingChanged = new SettingEvent<float>();
public static readonly SettingEvent<bool> OnDebugChanged = new SettingEvent<bool>();
internal static void Init()
{
ms_category = MelonLoader.MelonPreferences.CreateCategory("VET", null, true);
ms_entries = new List<MelonLoader.MelonPreferences_Entry>()
{
ms_category.CreateEntry(ModSetting.Enabled.ToString(), Enabled),
ms_category.CreateEntry(ModSetting.Smoothing.ToString(), (int)(Smoothing * 100f)),
ms_category.CreateEntry(ModSetting.Debug.ToString(), Debug),
};
Enabled = (bool)ms_entries[(int)ModSetting.Enabled].BoxedValue;
Smoothing = ((int)ms_entries[(int)ModSetting.Smoothing].BoxedValue) * 0.01f;
Debug = (bool)ms_entries[(int)ModSetting.Debug].BoxedValue;
MelonLoader.MelonCoroutines.Start(WaitMainMenuUi());
}
static System.Collections.IEnumerator WaitMainMenuUi()
{
while(ViewManager.Instance == null)
yield return null;
while(ViewManager.Instance.cohtmlView == null)
yield return null;
while(ViewManager.Instance.cohtmlView.Listener == null)
yield return null;
ViewManager.Instance.cohtmlView.Listener.ReadyForBindings += () =>
{
ViewManager.Instance.cohtmlView.View.BindCall("OnToggleUpdate_" + ms_category.Identifier, new Action<string, string>(OnToggleUpdate));
ViewManager.Instance.cohtmlView.View.BindCall("OnSliderUpdate_" + ms_category.Identifier, new Action<string, string>(OnSliderUpdate));
};
ViewManager.Instance.cohtmlView.Listener.FinishLoad += (_) =>
{
ViewManager.Instance.cohtmlView.View.ExecuteScript(ResourcesHandler.GetEmbeddedResource("mods_extension.js"));
ViewManager.Instance.cohtmlView.View.ExecuteScript(ResourcesHandler.GetEmbeddedResource("mod_menu.js"));
MelonLoader.MelonCoroutines.Start(UpdateMenuSettings());
};
}
static System.Collections.IEnumerator UpdateMenuSettings()
{
while(!ViewManager.Instance.IsReady || !ViewManager.Instance.IsViewShown)
yield return null;
foreach(var l_entry in ms_entries)
ViewManager.Instance.cohtmlView.View.TriggerEvent("updateModSetting", ms_category.Identifier, l_entry.DisplayName, l_entry.GetValueAsString());
}
static void OnToggleUpdate(string p_name, string p_value)
{
try
{
if(Enum.TryParse(p_name, out ModSetting l_setting) && bool.TryParse(p_value, out bool l_value))
{
switch(l_setting)
{
case ModSetting.Enabled:
{
Enabled = l_value;
OnEnabledChanged.Invoke(Enabled);
}
break;
case ModSetting.Debug:
{
Debug = l_value;
OnDebugChanged.Invoke(Debug);
}
break;
}
ms_entries[(int)l_setting].BoxedValue = l_value;
}
}
catch(Exception e)
{
MelonLoader.MelonLogger.Error(e);
}
}
static void OnSliderUpdate(string p_name, string p_value)
{
try
{
if(Enum.TryParse(p_name, out ModSetting l_setting) && int.TryParse(p_value, out int l_value))
{
switch(l_setting)
{
case ModSetting.Smoothing:
{
Smoothing = l_value * 0.01f;
OnSmoothingChanged.Invoke(Smoothing);
}
break;
}
ms_entries[(int)l_setting].BoxedValue = l_value;
}
}
catch(Exception e)
{
MelonLoader.MelonLogger.Error(e);
}
}
}
}

15
archived/ml_vet/Utils.cs Normal file
View file

@ -0,0 +1,15 @@
using ABI_RC.Core.Savior;
using ABI_RC.Core.UI;
using System.Reflection;
namespace ml_vet
{
public static class Utils
{
static readonly FieldInfo ms_view = typeof(CohtmlControlledViewWrapper).GetField("_view", BindingFlags.Instance | BindingFlags.NonPublic);
public static void ExecuteScript(this CohtmlControlledViewWrapper p_instance, string p_script) => (ms_view?.GetValue(p_instance) as cohtml.Net.View)?.ExecuteScript(p_script);
public static bool IsInVR() => ((MetaPort.Instance != null) && MetaPort.Instance.isUsingVr);
}
}

View file

@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<AssemblyName>ViveEyeTracking</AssemblyName>
<PackageId>ViveEyeTracking</PackageId>
<Product>ViveEyeTracking</Product>
</PropertyGroup>
<ItemGroup>
<None Remove="resources/mod_menu.js" />
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="../js/mods_extension.js" Link="resources/mods_extension.js" />
<EmbeddedResource Include="resources/mod_menu.js" />
</ItemGroup>
</Project>

View file

@ -0,0 +1,42 @@
{
let l_block = document.createElement('div');
l_block.innerHTML = `
<div class ="settings-subcategory">
<div class ="subcategory-name">Vive Eye Tracking</div>
<div class ="subcategory-description"></div>
</div>
<div class ="row-wrapper">
<div class ="option-caption">Enable eye tracking: </div>
<div class ="option-input">
<div id="Enabled" class ="inp_toggle no-scroll" data-current="true"></div>
</div>
</div>
<div class="row-wrapper">
<p>Requires Vive Face tracking restart at first.</p>
</div>
<div class ="row-wrapper">
<div class ="option-caption">Gaze smoothing: </div>
<div class ="option-input">
<div id="Smoothing" class ="inp_slider no-scroll" data-min="0" data-max="100" data-current="5"></div>
</div>
</div>
<div class ="row-wrapper">
<div class ="option-caption">Debug gizmos: </div>
<div class ="option-input">
<div id="Debug" class ="inp_toggle no-scroll" data-current="false"></div>
</div>
</div>
`;
document.getElementById('settings-implementation').appendChild(l_block);
// Toggles
for (let l_toggle of l_block.querySelectorAll('.inp_toggle'))
modsExtension.addSetting('VET', l_toggle.id, modsExtension.createToggle(l_toggle, 'OnToggleUpdate_VET'));
// Sliders
for (let l_slider of l_block.querySelectorAll('.inp_slider'))
modsExtension.addSetting('VET', l_slider.id, modsExtension.createSlider(l_slider, 'OnSliderUpdate_VET'));
}