Ryujinx/Ryujinx.Memory/Tracking/MultiRegionHandle.cs
riperiperi 3d98e1361b
GPU: Use a bitmap to track buffer modified flags. (#3775)
* Initial implementation

* Some improvements.

* Fix incorrect cast

* Performance improvement and improved correctness

* Add very fast path when all handles are checked.

* Slightly faster

* Add comment

* De-virtualize region handle

All region handles are now bitmap backed.

* Remove non-bitmap tracking

* Remove unused methods

* Add docs, remove unused methods

* Address Feedback

* Rename file
2022-10-29 22:07:37 +00:00

400 lines
14 KiB
C#

using System;
using System.Collections.Generic;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Threading;
namespace Ryujinx.Memory.Tracking
{
/// <summary>
/// A region handle that tracks a large region using many smaller handles, to provide
/// granular tracking that can be used to track partial updates. Backed by a bitmap
/// to improve performance when scanning large regions.
/// </summary>
public class MultiRegionHandle : IMultiRegionHandle
{
/// <summary>
/// A list of region handles for each granularity sized chunk of the whole region.
/// </summary>
private readonly RegionHandle[] _handles;
private readonly ulong Address;
private readonly ulong Granularity;
private readonly ulong Size;
private ConcurrentBitmap _dirtyBitmap;
private int _sequenceNumber;
private BitMap _sequenceNumberBitmap;
private int _uncheckedHandles;
public bool Dirty { get; private set; } = true;
internal MultiRegionHandle(MemoryTracking tracking, ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity)
{
_handles = new RegionHandle[size / granularity];
Granularity = granularity;
_dirtyBitmap = new ConcurrentBitmap(_handles.Length, true);
_sequenceNumberBitmap = new BitMap(_handles.Length);
int i = 0;
if (handles != null)
{
// Inherit from the handles we were given. Any gaps must be filled with new handles,
// and old handles larger than our granularity must copy their state onto new granular handles and dispose.
// It is assumed that the provided handles do not overlap, in order, are on page boundaries,
// and don't extend past the requested range.
foreach (RegionHandle handle in handles)
{
int startIndex = (int)((handle.Address - address) / granularity);
// Fill any gap left before this handle.
while (i < startIndex)
{
RegionHandle fillHandle = tracking.BeginTrackingBitmap(address + (ulong)i * granularity, granularity, _dirtyBitmap, i);
fillHandle.Parent = this;
_handles[i++] = fillHandle;
}
lock (tracking.TrackingLock)
{
if (handle is RegionHandle bitHandle && handle.Size == granularity)
{
handle.Parent = this;
bitHandle.ReplaceBitmap(_dirtyBitmap, i);
_handles[i++] = bitHandle;
}
else
{
int endIndex = (int)((handle.EndAddress - address) / granularity);
while (i < endIndex)
{
RegionHandle splitHandle = tracking.BeginTrackingBitmap(address + (ulong)i * granularity, granularity, _dirtyBitmap, i);
splitHandle.Parent = this;
splitHandle.Reprotect(handle.Dirty);
RegionSignal signal = handle.PreAction;
if (signal != null)
{
splitHandle.RegisterAction(signal);
}
_handles[i++] = splitHandle;
}
handle.Dispose();
}
}
}
}
// Fill any remaining space with new handles.
while (i < _handles.Length)
{
RegionHandle handle = tracking.BeginTrackingBitmap(address + (ulong)i * granularity, granularity, _dirtyBitmap, i);
handle.Parent = this;
_handles[i++] = handle;
}
_uncheckedHandles = _handles.Length;
Address = address;
Size = size;
}
public void SignalWrite()
{
Dirty = true;
}
public IEnumerable<RegionHandle> GetHandles()
{
return _handles;
}
public void ForceDirty(ulong address, ulong size)
{
Dirty = true;
int startHandle = (int)((address - Address) / Granularity);
int lastHandle = (int)((address + (size - 1) - Address) / Granularity);
for (int i = startHandle; i <= lastHandle; i++)
{
if (_sequenceNumberBitmap.Clear(i))
{
_uncheckedHandles++;
}
_handles[i].ForceDirty();
}
}
public void QueryModified(Action<ulong, ulong> modifiedAction)
{
if (!Dirty)
{
return;
}
Dirty = false;
QueryModified(Address, Size, modifiedAction);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ParseDirtyBits(long dirtyBits, ref int baseBit, ref int prevHandle, ref ulong rgStart, ref ulong rgSize, Action<ulong, ulong> modifiedAction)
{
while (dirtyBits != 0)
{
int bit = BitOperations.TrailingZeroCount(dirtyBits);
dirtyBits &= ~(1L << bit);
int handleIndex = baseBit + bit;
RegionHandle handle = _handles[handleIndex];
if (handleIndex != prevHandle + 1)
{
// Submit handles scanned until the gap as dirty
if (rgSize != 0)
{
modifiedAction(rgStart, rgSize);
rgSize = 0;
}
rgStart = handle.Address;
}
if (handle.Dirty)
{
rgSize += handle.Size;
handle.Reprotect();
}
prevHandle = handleIndex;
}
baseBit += ConcurrentBitmap.IntSize;
}
public void QueryModified(ulong address, ulong size, Action<ulong, ulong> modifiedAction)
{
int startHandle = (int)((address - Address) / Granularity);
int lastHandle = (int)((address + (size - 1) - Address) / Granularity);
ulong rgStart = _handles[startHandle].Address;
if (startHandle == lastHandle)
{
RegionHandle handle = _handles[startHandle];
if (handle.Dirty)
{
handle.Reprotect();
modifiedAction(rgStart, handle.Size);
}
return;
}
ulong rgSize = 0;
long[] masks = _dirtyBitmap.Masks;
int startIndex = startHandle >> ConcurrentBitmap.IntShift;
int startBit = startHandle & ConcurrentBitmap.IntMask;
long startMask = -1L << startBit;
int endIndex = lastHandle >> ConcurrentBitmap.IntShift;
int endBit = lastHandle & ConcurrentBitmap.IntMask;
long endMask = (long)(ulong.MaxValue >> (ConcurrentBitmap.IntMask - endBit));
long startValue = Volatile.Read(ref masks[startIndex]);
int baseBit = startIndex << ConcurrentBitmap.IntShift;
int prevHandle = startHandle - 1;
if (startIndex == endIndex)
{
ParseDirtyBits(startValue & startMask & endMask, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
}
else
{
ParseDirtyBits(startValue & startMask, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
for (int i = startIndex + 1; i < endIndex; i++)
{
ParseDirtyBits(Volatile.Read(ref masks[i]), ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
}
long endValue = Volatile.Read(ref masks[endIndex]);
ParseDirtyBits(endValue & endMask, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
}
if (rgSize != 0)
{
modifiedAction(rgStart, rgSize);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ParseDirtyBits(long dirtyBits, long mask, int index, long[] seqMasks, ref int baseBit, ref int prevHandle, ref ulong rgStart, ref ulong rgSize, Action<ulong, ulong> modifiedAction)
{
long seqMask = mask & ~seqMasks[index];
dirtyBits &= seqMask;
while (dirtyBits != 0)
{
int bit = BitOperations.TrailingZeroCount(dirtyBits);
dirtyBits &= ~(1L << bit);
int handleIndex = baseBit + bit;
RegionHandle handle = _handles[handleIndex];
if (handleIndex != prevHandle + 1)
{
// Submit handles scanned until the gap as dirty
if (rgSize != 0)
{
modifiedAction(rgStart, rgSize);
rgSize = 0;
}
rgStart = handle.Address;
}
rgSize += handle.Size;
handle.Reprotect();
prevHandle = handleIndex;
}
seqMasks[index] |= mask;
_uncheckedHandles -= BitOperations.PopCount((ulong)seqMask);
baseBit += ConcurrentBitmap.IntSize;
}
public void QueryModified(ulong address, ulong size, Action<ulong, ulong> modifiedAction, int sequenceNumber)
{
int startHandle = (int)((address - Address) / Granularity);
int lastHandle = (int)((address + (size - 1) - Address) / Granularity);
ulong rgStart = Address + (ulong)startHandle * Granularity;
if (sequenceNumber != _sequenceNumber)
{
if (_uncheckedHandles != _handles.Length)
{
_sequenceNumberBitmap.Clear();
_uncheckedHandles = _handles.Length;
}
_sequenceNumber = sequenceNumber;
}
if (startHandle == lastHandle)
{
var handle = _handles[startHandle];
if (_sequenceNumberBitmap.Set(startHandle))
{
_uncheckedHandles--;
if (handle.DirtyOrVolatile())
{
handle.Reprotect();
modifiedAction(rgStart, handle.Size);
}
}
return;
}
if (_uncheckedHandles == 0)
{
return;
}
ulong rgSize = 0;
long[] seqMasks = _sequenceNumberBitmap.Masks;
long[] masks = _dirtyBitmap.Masks;
int startIndex = startHandle >> ConcurrentBitmap.IntShift;
int startBit = startHandle & ConcurrentBitmap.IntMask;
long startMask = -1L << startBit;
int endIndex = lastHandle >> ConcurrentBitmap.IntShift;
int endBit = lastHandle & ConcurrentBitmap.IntMask;
long endMask = (long)(ulong.MaxValue >> (ConcurrentBitmap.IntMask - endBit));
long startValue = Volatile.Read(ref masks[startIndex]);
int baseBit = startIndex << ConcurrentBitmap.IntShift;
int prevHandle = startHandle - 1;
if (startIndex == endIndex)
{
ParseDirtyBits(startValue, startMask & endMask, startIndex, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
}
else
{
ParseDirtyBits(startValue, startMask, startIndex, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
for (int i = startIndex + 1; i < endIndex; i++)
{
ParseDirtyBits(Volatile.Read(ref masks[i]), -1L, i, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
}
long endValue = Volatile.Read(ref masks[endIndex]);
ParseDirtyBits(endValue, endMask, endIndex, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
}
if (rgSize != 0)
{
modifiedAction(rgStart, rgSize);
}
}
public void RegisterAction(ulong address, ulong size, RegionSignal action)
{
int startHandle = (int)((address - Address) / Granularity);
int lastHandle = (int)((address + (size - 1) - Address) / Granularity);
for (int i = startHandle; i <= lastHandle; i++)
{
_handles[i].RegisterAction(action);
}
}
public void RegisterPreciseAction(ulong address, ulong size, PreciseRegionSignal action)
{
int startHandle = (int)((address - Address) / Granularity);
int lastHandle = (int)((address + (size - 1) - Address) / Granularity);
for (int i = startHandle; i <= lastHandle; i++)
{
_handles[i].RegisterPreciseAction(action);
}
}
public void Dispose()
{
foreach (var handle in _handles)
{
handle.Dispose();
}
}
}
}