kneesox/sdraw_mods_cvr
1
0
Fork 0
mirror of https://github.com/hanetzer/sdraw_mods_cvr.git synced 2025-09-03 18:39:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Face tracking components override

Browse source
This commit is contained in:
SDraw 2022-09-23 23:27:38 +03:00
parent 65f8992a73
commit b6a4e0cd0a
No known key found for this signature in database
GPG key ID: BB95B4DAB2BB8BB5
8 changed files with 99 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

41
ml_dht/FaceTracked.cs
View file

@ -1,4 +1,5 @@
using ABI_RC.Core.Player;
using ABI.CCK.Components;
using ABI_RC.Core.Player;
using UnityEngine;
namespace ml_dht
@ -8,7 +9,9 @@ namespace ml_dht
bool m_enabled = true;
float m_smoothing = 0.5f;
bool m_mirrored = false;
bool m_faceOverride = true;
CVRAvatar m_avatarDescriptior = null;
RootMotion.FinalIK.LookAtIK m_lookIK = null;
Transform m_camera = null;
Transform m_headBone = null;
@ -61,8 +64,38 @@ namespace ml_dht
}
}
public void OnFaceTrackingUpdate(CVRFaceTracking p_component)
{
if(m_enabled && m_faceOverride)
{
if(m_avatarDescriptior != null)
m_avatarDescriptior.useVisemeLipsync = false;
p_component.BlendShapeValues[(int)ViveSR.anipal.Lip.LipShape_v2.Jaw_Open] = m_mouthShapes.x * 100f;
if(m_mouthShapes.y < 0f)
{
float l_weight = Mathf.Clamp(Mathf.InverseLerp(0.25f, 1f, -m_mouthShapes.y), 0f, 1f) * 100f;
p_component.BlendShapeValues[(int)ViveSR.anipal.Lip.LipShape_v2.Mouth_Pout] = 0f;
p_component.BlendShapeValues[(int)ViveSR.anipal.Lip.LipShape_v2.Mouth_Smile_Left] = l_weight;
p_component.BlendShapeValues[(int)ViveSR.anipal.Lip.LipShape_v2.Mouth_Smile_Right] = l_weight;
}
if(m_mouthShapes.y > 0f)
{
float l_weight = Mathf.Clamp(Mathf.InverseLerp(0.25f, 1f, m_mouthShapes.y), 0f, 1f) * 100f;
p_component.BlendShapeValues[(int)ViveSR.anipal.Lip.LipShape_v2.Mouth_Pout] = l_weight;
p_component.BlendShapeValues[(int)ViveSR.anipal.Lip.LipShape_v2.Mouth_Smile_Left] = 0f;
p_component.BlendShapeValues[(int)ViveSR.anipal.Lip.LipShape_v2.Mouth_Smile_Right] = 0f;
}
p_component.LipSyncWasUpdated = true;
p_component.UpdateLipShapes();
}
}
public void OnSetupAvatarGeneral()
{
m_avatarDescriptior = PlayerSetup.Instance._avatar.GetComponent<CVRAvatar>();
m_headBone = PlayerSetup.Instance._animator.GetBoneTransform(HumanBodyBones.Head);
m_lookIK = PlayerSetup.Instance._avatar.GetComponent<RootMotion.FinalIK.LookAtIK>();
@ -71,9 +104,11 @@ namespace ml_dht
if(m_lookIK != null)
m_lookIK.solver.OnPostUpdate += this.OnLookIKPostUpdate;
}
public void OnAvatarClear()
{
m_avatarDescriptior = null;
m_lookIK = null;
m_headBone = null;
m_lastHeadRotation = Quaternion.identity;
@ -97,5 +132,9 @@ namespace ml_dht
{
m_mirrored = p_state;
}
public void SetFaceOverride(bool p_state)
{
m_faceOverride = p_state;
}
}
}

26
ml_dht/Main.cs
View file

@ -1,3 +1,4 @@
using ABI.CCK.Components;
using ABI_RC.Core.Player;
namespace ml_dht
@ -21,6 +22,7 @@ namespace ml_dht
Settings.EnabledChange += this.OnEnabledChanged;
Settings.MirroredChange += this.OnMirroredChanged;
Settings.SmoothingChange += this.OnSmoothingChanged;
Settings.FaceOverrideChange += this.OnFaceOverrideChange;
m_mapReader = new MemoryMapReader();
m_buffer = new byte[1024];
@ -43,6 +45,11 @@ namespace ml_dht
null,
new HarmonyLib.HarmonyMethod(typeof(DesktopHeadTracking).GetMethod(nameof(OnEyeControllerUpdate_Postfix), System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic))
);
HarmonyInstance.Patch(
typeof(CVRFaceTracking).GetMethod("Update", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic),
null,
new HarmonyLib.HarmonyMethod(typeof(DesktopHeadTracking).GetMethod(nameof(OnFaceTrackingUpdate_Postfix), System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic))
);
MelonLoader.MelonCoroutines.Start(WaitForPlayer());
}
@ -83,6 +90,11 @@ namespace ml_dht
if(m_localTracked != null)
m_localTracked.SetSmoothing(p_value);
}
void OnFaceOverrideChange(bool p_state)
{
if(m_localTracked != null)
m_localTracked.SetFaceOverride(p_state);
}
static void OnSetupAvatarGeneral_Postfix() => ms_instance?.OnSetupAvatarGeneral();
void OnSetupAvatarGeneral()
@ -129,5 +141,19 @@ namespace ml_dht
if(m_localTracked != null)
m_localTracked.OnEyeControllerUpdate();
}
static void OnFaceTrackingUpdate_Postfix(ref CVRFaceTracking __instance) => ms_instance?.OnFaceTrackingUpdate(__instance);
void OnFaceTrackingUpdate(CVRFaceTracking p_component)
{
try
{
if(p_component.isLocal && (m_localTracked != null))
m_localTracked.OnFaceTrackingUpdate(p_component);
}
catch(System.Exception e)
{
MelonLoader.MelonLogger.Error(e);
}
}
}
}

6
ml_dht/Properties/AssemblyInfo.cs
View file

@ -1,10 +1,10 @@
using System.Reflection;
[assembly: AssemblyTitle("DesktopHeadTracking")]
[assembly: AssemblyVersion("1.0.0")]
[assembly: AssemblyFileVersion("1.0.0")]
[assembly: AssemblyVersion("1.0.1")]
[assembly: AssemblyFileVersion("1.0.1")]
[assembly: MelonLoader.MelonInfo(typeof(ml_dht.DesktopHeadTracking), "DesktopHeadTracking", "1.0.0", "SDraw", "https://github.com/SDraw/ml_mods_cvr")]
[assembly: MelonLoader.MelonInfo(typeof(ml_dht.DesktopHeadTracking), "DesktopHeadTracking", "1.0.1", "SDraw", "https://github.com/SDraw/ml_mods_cvr")]
[assembly: MelonLoader.MelonGame(null, "ChilloutVR")]
[assembly: MelonLoader.MelonPlatform(MelonLoader.MelonPlatformAttribute.CompatiblePlatforms.WINDOWS_X64)]
[assembly: MelonLoader.MelonPlatformDomain(MelonLoader.MelonPlatformDomainAttribute.CompatibleDomains.MONO)]

4
ml_dht/README.md
View file

@ -8,6 +8,7 @@ Refer to `TrackingData.cs` for reference in case of implementing own software.
* Head rotation
* Eyes gaze direction
* Blinking
* Basic mouth shapes
# Installation
* Install [latest MelonLoader](https://github.com/LavaGang/MelonLoader)
@ -18,7 +19,8 @@ Refer to `TrackingData.cs` for reference in case of implementing own software.
Available mod's settings in `Settings - Implementation - Desktop Head Tracking`:
* **Enabled:** enabled head tracking; default value - `false`.
* **Mirrored movement:** mirrors movement and gaze along 0YZ plane; default value - `false`.
* **Movement smoothing:** smoothing factor between new and old movement data; default value - `50`;
* **Movement smoothing:** smoothing factor between new and old movement data; default value - `50`.
* **Override face tracking:** Overrides and activates avatar's `VRC Face Tracking` components. List of used shapes: `Jaw_Open`, `Mouth_Pout`, `Mouth_Smile_Left`, `Mouth_Smile_Right`.
# Known compatible tracking software
* [VSeeFace](https://www.vseeface.icu) with [Tracking Data Parser mod](https://github.com/SDraw/ml_mods_vsf)

17
ml_dht/Settings.cs
View file

@ -11,12 +11,14 @@ namespace ml_dht
{
Enabled = 0,
Mirrored,
Smoothing
Smoothing,
FaceOverride
}
static bool ms_enabled = false;
static bool ms_mirrored = false;
static float ms_smoothing = 0.5f;
static bool ms_faceOverride = true;
static MelonLoader.MelonPreferences_Category ms_category = null;
static List<MelonLoader.MelonPreferences_Entry> ms_entries = null;
@ -24,6 +26,7 @@ namespace ml_dht
static public event Action<bool> EnabledChange;
static public event Action<bool> MirroredChange;
static public event Action<float> SmoothingChange;
static public event Action<bool> FaceOverrideChange;
public static void Init()
{
@ -33,6 +36,7 @@ namespace ml_dht
ms_entries.Add(ms_category.CreateEntry(ModSetting.Enabled.ToString(), false));
ms_entries.Add(ms_category.CreateEntry(ModSetting.Mirrored.ToString(), false));
ms_entries.Add(ms_category.CreateEntry(ModSetting.Smoothing.ToString(), 50));
ms_entries.Add(ms_category.CreateEntry(ModSetting.FaceOverride.ToString(), true));
Load();
@ -66,6 +70,7 @@ namespace ml_dht
ms_enabled = (bool)ms_entries[(int)ModSetting.Enabled].BoxedValue;
ms_mirrored = (bool)ms_entries[(int)ModSetting.Mirrored].BoxedValue;
ms_smoothing = ((int)ms_entries[(int)ModSetting.Smoothing].BoxedValue) * 0.01f;
ms_faceOverride = (bool)ms_entries[(int)ModSetting.FaceOverride].BoxedValue;
}
static void OnSliderUpdate(string p_name, string p_value)
@ -105,6 +110,12 @@ namespace ml_dht
MirroredChange?.Invoke(ms_mirrored);
}
break;
case ModSetting.FaceOverride:
{
ms_faceOverride = bool.Parse(p_value);
FaceOverrideChange?.Invoke(ms_faceOverride);
} break;
}
ms_entries[(int)l_setting].BoxedValue = bool.Parse(p_value);
@ -123,5 +134,9 @@ namespace ml_dht
{
get => ms_smoothing;
}
public static bool FaceOverride
{
get => ms_faceOverride;
}
}
}

6
ml_dht/TrackingData.cs
View file

@ -13,9 +13,9 @@ struct TrackingData
public float m_gazeX; // Range - [0;1], 0.5 - center
public float m_gazeY; // Range - [0;1], 0.5 - center
public float m_blink; // Range - [0;1], 1.0 - closed
public float m_mouthOpen; // Range - [0;1], not used yet
public float m_mouthShape; // Range - [0;1], not used yet
public float m_brows; // Range - [0;1], not used yet
public float m_mouthOpen; // Range - [0;1]
public float m_mouthShape; // Range - [-1;1], -1 - wide, 1 - narrow
public float m_brows; // Range - [-1;1], -1 - up, 1 - down; not used yet
static public byte[] ToBytes(TrackingData p_faceData)
{

7
ml_dht/resources/menu.js
View file

@ -200,6 +200,13 @@ function inp_toggle_mod_dht(_obj, _callbackName) {
<div id="Smoothing" class ="inp_slider no-scroll" data-min="0" data-max="99" data-current="50"></div>
</div>
</div>
<div class ="row-wrapper">
<div class ="option-caption">Override face tracking: </div>
<div class ="option-input">
<div id="FaceOverride" class ="inp_toggle no-scroll" data-current="true"></div>
</div>
</div>
`;
document.getElementById('settings-implementation').appendChild(l_block);