4 May
2021
4 May
'21
10:02 a.m.
https://git.reactos.org/?p=reactos.git;a=commitdiff;h=f421bccbcc02eb15e43c9…
commit f421bccbcc02eb15e43c9eb102266a70b5af7fbd
Author: Jérôme Gardou <jerome.gardou(a)reactos.org>
AuthorDate: Mon Apr 12 11:36:19 2021 +0200
Commit: Jérôme Gardou <zefklop(a)users.noreply.github.com>
CommitDate: Tue May 4 12:02:41 2021 +0200
[NTOS:MM] First shot for Working Set list support
- Initialize
- Add private page (no shared page support yet)
- Remove pages
- Trim
Yes, this is C++ in the kernel.
---
ntoskrnl/include/internal/mm.h | 6 +
ntoskrnl/mm/ARM3/mminit.c | 3 +
ntoskrnl/mm/ARM3/procsup.c | 74 +++----
ntoskrnl/mm/ARM3/session.c | 1 -
ntoskrnl/mm/ARM3/wslist.cpp | 484 +++++++++++++++++++++++++++++++++++++++++
ntoskrnl/ntos.cmake | 1 +
sdk/include/ndk/mmtypes.h | 16 ++
7 files changed, 541 insertions(+), 44 deletions(-)
diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h
index 5eab082b3f2..2f101c97956 100644
--- a/ntoskrnl/include/internal/mm.h
+++ b/ntoskrnl/include/internal/mm.h
@@ -1645,6 +1645,12 @@ MmCopyVirtualMemory(IN PEPROCESS SourceProcess,
IN KPROCESSOR_MODE PreviousMode,
OUT PSIZE_T ReturnSize);
+/* wslist.cpp ****************************************************************/
+_Requires_exclusive_lock_held_(WorkingSet->WorkingSetMutex)
+VOID
+NTAPI
+MiInitializeWorkingSetList(_Inout_ PMMSUPPORT WorkingSet);
+
#ifdef __cplusplus
} // extern "C"
#endif
diff --git a/ntoskrnl/mm/ARM3/mminit.c b/ntoskrnl/mm/ARM3/mminit.c
index 9b18537f3ac..400f8965dd5 100644
--- a/ntoskrnl/mm/ARM3/mminit.c
+++ b/ntoskrnl/mm/ARM3/mminit.c
@@ -2139,6 +2139,9 @@ MmArmInitSystem(IN ULONG Phase,
/* Initialize the user mode image list */
InitializeListHead(&MmLoadedUserImageList);
+ /* Initalize the Working set list */
+ InitializeListHead(&MmWorkingSetExpansionHead);
+
/* Initialize critical section timeout value (relative time is negative) */
MmCriticalSectionTimeout.QuadPart = MmCritsectTimeoutSeconds * (-10000000LL);
diff --git a/ntoskrnl/mm/ARM3/procsup.c b/ntoskrnl/mm/ARM3/procsup.c
index 23981bcadf1..aa9133b1929 100644
--- a/ntoskrnl/mm/ARM3/procsup.c
+++ b/ntoskrnl/mm/ARM3/procsup.c
@@ -896,40 +896,6 @@ MiInsertSharedUserPageVad(VOID)
}
#endif
-VOID
-NTAPI
-MiInitializeWorkingSetList(IN PEPROCESS CurrentProcess)
-{
- PMMPFN Pfn1;
- PMMPTE sysPte;
- MMPTE tempPte;
-
- /* Setup some bogus list data */
- MmWorkingSetList->LastEntry = CurrentProcess->Vm.MinimumWorkingSetSize;
- MmWorkingSetList->HashTable = NULL;
- MmWorkingSetList->HashTableSize = 0;
- MmWorkingSetList->NumberOfImageWaiters = 0;
- MmWorkingSetList->Wsle = (PVOID)(ULONG_PTR)0xDEADBABEDEADBABEULL;
- MmWorkingSetList->VadBitMapHint = 1;
- MmWorkingSetList->HashTableStart = (PVOID)(ULONG_PTR)0xBADAB00BBADAB00BULL;
- MmWorkingSetList->HighestPermittedHashAddress =
(PVOID)(ULONG_PTR)0xCAFEBABECAFEBABEULL;
- MmWorkingSetList->FirstFree = 1;
- MmWorkingSetList->FirstDynamic = 2;
- MmWorkingSetList->NextSlot = 3;
- MmWorkingSetList->LastInitializedWsle = 4;
-
- /* The rule is that the owner process is always in the FLINK of the PDE's PFN
entry */
- Pfn1 = MiGetPfnEntry(CurrentProcess->Pcb.DirectoryTableBase[0] >>
PAGE_SHIFT);
- ASSERT(Pfn1->u4.PteFrame == MiGetPfnEntryIndex(Pfn1));
- Pfn1->u1.Event = (PKEVENT)CurrentProcess;
-
- /* Map the process working set in kernel space */
- sysPte = MiReserveSystemPtes(1, SystemPteSpace);
- MI_MAKE_HARDWARE_PTE_KERNEL(&tempPte, sysPte, MM_READWRITE,
CurrentProcess->WorkingSetPage);
- MI_WRITE_VALID_PTE(sysPte, tempPte);
- CurrentProcess->Vm.VmWorkingSetList = MiPteToAddress(sysPte);
-}
-
NTSTATUS
NTAPI
MmInitializeProcessAddressSpace(IN PEPROCESS Process,
@@ -944,6 +910,7 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process,
PMMPTE PointerPte;
KIRQL OldIrql;
PMMPDE PointerPde;
+ PMMPFN Pfn;
PFN_NUMBER PageFrameNumber;
UNICODE_STRING FileName;
PWCHAR Source;
@@ -980,6 +947,8 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process,
/* On x64 the PFNs for the initial process are already set up */
if (Process != &KiInitialProcess) {
#endif
+ /* Lock our working set */
+ MiLockProcessWorkingSet(Process, PsGetCurrentThread());
/* Lock PFN database */
OldIrql = MiAcquirePfnLock();
@@ -997,7 +966,7 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process,
MiInitializePfn(PageFrameNumber, PointerPte, TRUE);
/* Do the same for hyperspace */
- PointerPde = MiAddressToPde((PVOID)HYPER_SPACE);
+ PointerPde = MiAddressToPde(HYPER_SPACE);
PageFrameNumber = PFN_FROM_PTE(PointerPde);
MiInitializePfn(PageFrameNumber, (PMMPTE)PointerPde, TRUE);
#if (_MI_PAGING_LEVELS == 2)
@@ -1019,18 +988,30 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process,
ASSERT(Process->Pcb.DirectoryTableBase[1] == PageFrameNumber * PAGE_SIZE);
#endif
- /* Setup the PFN for the PTE for the working set */
- PointerPte = MiAddressToPte(MI_WORKING_SET_LIST);
- MI_MAKE_HARDWARE_PTE(&TempPte, PointerPte, MM_READWRITE, 0);
- ASSERT(PointerPte->u.Long != 0);
+ /* Do the same for the Working set list */
+ PointerPte = MiAddressToPte(MmWorkingSetList);
PageFrameNumber = PFN_FROM_PTE(PointerPte);
- MI_WRITE_INVALID_PTE(PointerPte, DemandZeroPte);
MiInitializePfn(PageFrameNumber, PointerPte, TRUE);
- TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
- MI_WRITE_VALID_PTE(PointerPte, TempPte);
+
+ /* This should be in hyper space, but not in the mapping range */
+ Process->Vm.VmWorkingSetList = MmWorkingSetList;
+ ASSERT(((ULONG_PTR)MmWorkingSetList >= MI_MAPPING_RANGE_END) &&
((ULONG_PTR)MmWorkingSetList <= HYPER_SPACE_END));
/* Now initialize the working set list */
- MiInitializeWorkingSetList(Process);
+ MiInitializeWorkingSetList(&Process->Vm);
+
+ /* Map the process working set in kernel space */
+ /* FIXME: there should be no need */
+ PointerPte = MiReserveSystemPtes(1, SystemPteSpace);
+ MI_MAKE_HARDWARE_PTE_KERNEL(&TempPte, PointerPte, MM_READWRITE,
Process->WorkingSetPage);
+ MI_WRITE_VALID_PTE(PointerPte, TempPte);
+ Process->Vm.VmWorkingSetList = MiPteToAddress(PointerPte);
+
+ /* The rule is that the owner process is always in the FLINK of the PDE's PFN
entry */
+ Pfn = MiGetPfnEntry(Process->Pcb.DirectoryTableBase[0] >> PAGE_SHIFT);
+ ASSERT(Pfn->u4.PteFrame == MiGetPfnEntryIndex(Pfn));
+ ASSERT(Pfn->u1.WsIndex == 0);
+ Pfn->u1.Event = (PKEVENT)Process;
/* Sanity check */
ASSERT(Process->PhysicalVadRoot == NULL);
@@ -1038,6 +1019,8 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process,
/* Release PFN lock */
MiReleasePfnLock(OldIrql);
+ /* Release the process working set */
+ MiUnlockProcessWorkingSet(Process, PsGetCurrentThread());
#ifdef _M_AMD64
} /* On x64 the PFNs for the initial process are already set up */
#endif
@@ -1358,6 +1341,11 @@ MmDeleteProcessAddressSpace2(IN PEPROCESS Process)
//ASSERT(Process->CommitCharge == 0);
+ /* Remove us from the list */
+ OldIrql = MiAcquireExpansionLock();
+ RemoveEntryList(&Process->Vm.WorkingSetExpansionLinks);
+ MiReleaseExpansionLock(OldIrql);
+
/* Acquire the PFN lock */
OldIrql = MiAcquirePfnLock();
diff --git a/ntoskrnl/mm/ARM3/session.c b/ntoskrnl/mm/ARM3/session.c
index 898dbbbaebc..b897a491bb0 100644
--- a/ntoskrnl/mm/ARM3/session.c
+++ b/ntoskrnl/mm/ARM3/session.c
@@ -41,7 +41,6 @@ MiInitializeSessionWsSupport(VOID)
{
/* Initialize the list heads */
InitializeListHead(&MiSessionWsList);
- InitializeListHead(&MmWorkingSetExpansionHead);
}
BOOLEAN
diff --git a/ntoskrnl/mm/ARM3/wslist.cpp b/ntoskrnl/mm/ARM3/wslist.cpp
new file mode 100644
index 00000000000..430e2c1a564
--- /dev/null
+++ b/ntoskrnl/mm/ARM3/wslist.cpp
@@ -0,0 +1,484 @@
+/*
+ * PROJECT: ReactOS Kernel
+ * LICENSE: BSD-3-Clause (https://spdx.org/licenses/BSD-3-Clause.html)
+ * FILE: ntoskrnl/mm/ARM3/wslist.cpp
+ * PURPOSE: Working set list management
+ * PROGRAMMERS: Jérôme Gardou
+ */
+
+/* INCLUDES *******************************************************************/
+#include <ntoskrnl.h>
+
+#define NDEBUG
+#include <debug.h>
+
+#define MODULE_INVOLVED_IN_ARM3
+#include "miarm.h"
+
+/* GLOBALS ********************************************************************/
+PMMWSL MmWorkingSetList;
+KEVENT MmWorkingSetManagerEvent;
+
+/* LOCAL FUNCTIONS ************************************************************/
+namespace ntoskrnl
+{
+
+static MMPTE GetPteTemplateForWsList(PMMWSL WsList)
+{
+ return (WsList == MmSystemCacheWorkingSetList) ? ValidKernelPte :
ValidKernelPteLocal;
+}
+
+static ULONG GetNextPageColorForWsList(PMMWSL WsList)
+{
+ return (WsList == MmSystemCacheWorkingSetList) ? MI_GET_NEXT_COLOR() :
MI_GET_NEXT_PROCESS_COLOR(PsGetCurrentProcess());
+}
+
+static void FreeWsleIndex(PMMWSL WsList, ULONG Index)
+{
+ PMMWSLE Wsle = WsList->Wsle;
+ ULONG& LastEntry = WsList->LastEntry;
+ ULONG& FirstFree = WsList->FirstFree;
+ ULONG& LastInitializedWsle = WsList->LastInitializedWsle;
+
+ /* Erase it now */
+ Wsle[Index].u1.Long = 0;
+
+ if (Index == (LastEntry - 1))
+ {
+ /* We're freeing the last index of our list. */
+ while (Wsle[Index].u1.e1.Valid == 0)
+ Index--;
+
+ /* Should we bother about the Free entries */
+ if (FirstFree < Index)
+ {
+ /* Try getting the index of the last free entry */
+ ASSERT(Wsle[Index + 1].u1.Free.MustBeZero == 0);
+ ULONG PreviousFree = Wsle[Index + 1].u1.Free.PreviousFree;
+ ASSERT(PreviousFree < LastEntry);
+ ULONG LastFree = Index + 1 - PreviousFree;
+#ifdef MMWSLE_PREVIOUS_FREE_JUMP
+ while (Wsle[LastFree].u1.e1.Valid)
+ {
+ ASSERT(LastFree > MMWSLE_PREVIOUS_FREE_JUMP);
+ LastFree -= MMWSLE_PREVIOUS_FREE_JUMP;
+ }
+#endif
+ /* Update */
+ ASSERT(LastFree >= FirstFree);
+ Wsle[FirstFree].u1.Free.PreviousFree = (Index + 1 - LastFree) &
MMWSLE_PREVIOUS_FREE_MASK;
+ Wsle[LastFree].u1.Free.NextFree = 0;
+ }
+ else
+ {
+ /* No more free entries in our array */
+ FirstFree = ULONG_MAX;
+ }
+ /* This is the new size of our array */
+ LastEntry = Index + 1;
+ /* Should we shrink the alloc? */
+ while ((LastInitializedWsle - LastEntry) > (PAGE_SIZE / sizeof(MMWSLE)))
+ {
+ PMMPTE PointerPte = MiAddressToPte(Wsle + LastInitializedWsle - 1);
+ /* We must not free ourself! */
+ ASSERT(MiPteToAddress(PointerPte) != WsList);
+
+ PFN_NUMBER Page = PFN_FROM_PTE(PointerPte);
+ PMMPFN Pfn = MiGetPfnEntry(Page);
+
+ MI_SET_PFN_DELETED(Pfn);
+ MiDecrementShareCount(MiGetPfnEntry(Pfn->u4.PteFrame),
Pfn->u4.PteFrame);
+ MiDecrementShareCount(Pfn, Page);
+
+ PointerPte->u.Long = 0;
+
+ KeInvalidateTlbEntry(Wsle + LastInitializedWsle - 1);
+ LastInitializedWsle -= PAGE_SIZE / sizeof(MMWSLE);
+ }
+ return;
+ }
+
+ if (FirstFree == ULONG_MAX)
+ {
+ /* We're the first one. */
+ FirstFree = Index;
+ Wsle[FirstFree].u1.Free.PreviousFree = (LastEntry - FirstFree) &
MMWSLE_PREVIOUS_FREE_MASK;
+ return;
+ }
+
+ /* We must find where to place ourself */
+ ULONG NextFree = FirstFree;
+ ULONG PreviousFree = 0;
+ while (NextFree < Index)
+ {
+ ASSERT(Wsle[NextFree].u1.Free.MustBeZero == 0);
+ if (Wsle[NextFree].u1.Free.NextFree == 0)
+ break;
+ PreviousFree = NextFree;
+ NextFree += Wsle[NextFree].u1.Free.NextFree;
+ }
+
+ if (NextFree < Index)
+ {
+ /* This is actually the last free entry */
+ Wsle[NextFree].u1.Free.NextFree = Index - NextFree;
+ Wsle[Index].u1.Free.PreviousFree = (Index - NextFree) &
MMWSLE_PREVIOUS_FREE_MASK;
+ Wsle[FirstFree].u1.Free.PreviousFree = (LastEntry - Index) &
MMWSLE_PREVIOUS_FREE_MASK;
+ return;
+ }
+
+ if (PreviousFree == 0)
+ {
+ /* This is the first free */
+ Wsle[Index].u1.Free.NextFree = FirstFree - Index;
+ Wsle[Index].u1.Free.PreviousFree = Wsle[FirstFree].u1.Free.PreviousFree;
+ Wsle[FirstFree].u1.Free.PreviousFree = (FirstFree - Index) &
MMWSLE_PREVIOUS_FREE_MASK;
+ FirstFree = Index;
+ return;
+ }
+
+ /* Insert */
+ Wsle[PreviousFree].u1.Free.NextFree = (Index - PreviousFree);
+ Wsle[Index].u1.Free.PreviousFree = (Index - PreviousFree) &
MMWSLE_PREVIOUS_FREE_MASK;
+ Wsle[Index].u1.Free.NextFree = NextFree - Index;
+ Wsle[NextFree].u1.Free.PreviousFree = (NextFree - Index) &
MMWSLE_PREVIOUS_FREE_MASK;
+}
+
+static ULONG GetFreeWsleIndex(PMMWSL WsList)
+{
+ ULONG Index;
+ if (WsList->FirstFree != ULONG_MAX)
+ {
+ Index = WsList->FirstFree;
+ ASSERT(Index < WsList->LastInitializedWsle);
+ MMWSLE_FREE_ENTRY& FreeWsle = WsList->Wsle[Index].u1.Free;
+ ASSERT(FreeWsle.MustBeZero == 0);
+ if (FreeWsle.NextFree != 0)
+ {
+ WsList->FirstFree += FreeWsle.NextFree;
+ WsList->Wsle[WsList->FirstFree].u1.Free.PreviousFree =
FreeWsle.PreviousFree;
+ }
+ else
+ {
+ WsList->FirstFree = ULONG_MAX;
+ }
+ }
+ else
+ {
+ Index = WsList->LastEntry++;
+ if (Index >= WsList->LastInitializedWsle)
+ {
+ /* Grow our array */
+ PMMPTE PointerPte =
MiAddressToPte(&WsList->Wsle[WsList->LastInitializedWsle]);
+ ASSERT(PointerPte->u.Hard.Valid == 0);
+ MMPTE TempPte = GetPteTemplateForWsList(WsList);
+ TempPte.u.Hard.PageFrameNumber =
MiRemoveAnyPage(GetNextPageColorForWsList(WsList));
+ MiInitializePfnAndMakePteValid(TempPte.u.Hard.PageFrameNumber, PointerPte,
TempPte);
+ WsList->LastInitializedWsle += PAGE_SIZE / sizeof(MMWSLE);
+ }
+ }
+
+ WsList->Wsle[Index].u1.Long = 0;
+ return Index;
+}
+
+static
+VOID
+RemoveFromWsList(PMMWSL WsList, PVOID Address)
+{
+ /* Make sure that we are holding the right locks. */
+ ASSERT(MM_ANY_WS_LOCK_HELD(PsGetCurrentThread()));
+ MI_ASSERT_PFN_LOCK_HELD();
+
+ PMMPTE PointerPte = MiAddressToPte(Address);
+
+ /* Make sure we are removing a paged-in address */
+ ASSERT(PointerPte->u.Hard.Valid == 1);
+ PMMPFN Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte));
+ ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
+
+ /* Shared pages not supported yet */
+ ASSERT(Pfn1->u3.e1.PrototypePte == 0);
+
+ /* Nor are "ROS PFN" */
+ ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE);
+
+ /* And we should have a valid index here */
+ ASSERT(Pfn1->u1.WsIndex != 0);
+
+ FreeWsleIndex(WsList, Pfn1->u1.WsIndex);
+}
+
+static
+ULONG
+TrimWsList(PMMWSL WsList)
+{
+ /* This should be done under WS lock */
+ ASSERT(MM_ANY_WS_LOCK_HELD(PsGetCurrentThread()));
+
+ ULONG Ret = 0;
+
+ /* Walk the array */
+ for (ULONG i = WsList->FirstDynamic; i < WsList->LastEntry; i++)
+ {
+ MMWSLE& Entry = WsList->Wsle[i];
+ if (!Entry.u1.e1.Valid)
+ continue;
+
+ /* Only direct entries for now */
+ ASSERT(Entry.u1.e1.Direct == 1);
+
+ /* Check the PTE */
+ PMMPTE PointerPte = MiAddressToPte(Entry.u1.VirtualAddress);
+
+ /* This must be valid */
+ ASSERT(PointerPte->u.Hard.Valid);
+
+ /* If the PTE was accessed, simply reset and that's the end of it */
+ if (PointerPte->u.Hard.Accessed)
+ {
+ Entry.u1.e1.Age = 0;
+ PointerPte->u.Hard.Accessed = 0;
+ KeInvalidateTlbEntry(Entry.u1.VirtualAddress);
+ continue;
+ }
+
+ /* If the entry is not so old, just age it */
+ if (Entry.u1.e1.Age < 3)
+ {
+ Entry.u1.e1.Age++;
+ continue;
+ }
+
+ if ((Entry.u1.e1.LockedInMemory) || (Entry.u1.e1.LockedInWs))
+ {
+ /* This one is locked. Next time, maybe... */
+ continue;
+ }
+
+ /* FIXME: Invalidating PDEs breaks legacy MMs */
+ if (MI_IS_PAGE_TABLE_ADDRESS(Entry.u1.VirtualAddress))
+ continue;
+
+ /* Please put yourself aside and make place for the younger ones */
+ PFN_NUMBER Page = PFN_FROM_PTE(PointerPte);
+ KIRQL OldIrql = MiAcquirePfnLock();
+
+ PMMPFN Pfn = MiGetPfnEntry(Page);
+
+ /* Not supported yet */
+ ASSERT(Pfn->u3.e1.PrototypePte == 0);
+ ASSERT(!MI_IS_ROS_PFN(Pfn));
+
+ /* FIXME: Remove this hack when possible */
+ if (Pfn->Wsle.u1.e1.LockedInMemory || (Pfn->Wsle.u1.e1.LockedInWs))
+ {
+ MiReleasePfnLock(OldIrql);
+ continue;
+ }
+
+ /* We can remove it from the list. Save Protection first */
+ ULONG Protection = Entry.u1.e1.Protection;
+ RemoveFromWsList(WsList, Entry.u1.VirtualAddress);
+
+ /* Dirtify the page, if needed */
+ if (PointerPte->u.Hard.Dirty)
+ Pfn->u3.e1.Modified = 1;
+
+ /* Make this a transition PTE */
+ MI_MAKE_TRANSITION_PTE(PointerPte, Page, Protection);
+ KeInvalidateTlbEntry(MiAddressToPte(PointerPte));
+
+ /* Drop the share count. This will take care of putting it in the standby or
modified list. */
+ MiDecrementShareCount(Pfn, Page);
+
+ MiReleasePfnLock(OldIrql);
+ Ret++;
+ }
+ return Ret;
+}
+
+/* GLOBAL FUNCTIONS ***********************************************************/
+extern "C"
+{
+
+VOID
+NTAPI
+MiInsertInWorkingSetList(
+ _Inout_ PMMSUPPORT Vm,
+ _In_ PVOID Address,
+ _In_ ULONG Protection)
+{
+ PMMWSL WsList = Vm->VmWorkingSetList;
+
+ /* Make sure that we are holding the right locks. */
+ ASSERT(MM_ANY_WS_LOCK_HELD(PsGetCurrentThread()));
+ MI_ASSERT_PFN_LOCK_HELD();
+
+ PMMPTE PointerPte = MiAddressToPte(Address);
+
+ /* Make sure we are adding a paged-in address */
+ ASSERT(PointerPte->u.Hard.Valid == 1);
+ PMMPFN Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte));
+ ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
+
+ /* Shared pages not supported yet */
+ ASSERT(Pfn1->u1.WsIndex == 0);
+ ASSERT(Pfn1->u3.e1.PrototypePte == 0);
+
+ /* Nor are "ROS PFN" */
+ ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE);
+
+ Pfn1->u1.WsIndex = GetFreeWsleIndex(WsList);
+ MMWSLENTRY& NewWsle = WsList->Wsle[Pfn1->u1.WsIndex].u1.e1;
+ NewWsle.VirtualPageNumber = reinterpret_cast<ULONG_PTR>(Address) >>
PAGE_SHIFT;
+ NewWsle.Protection = Protection;
+ NewWsle.Direct = 1;
+ NewWsle.Hashed = 0;
+ NewWsle.LockedInMemory = 0;
+ NewWsle.LockedInWs = 0;
+ NewWsle.Age = 0;
+ NewWsle.Valid = 1;
+
+ Vm->WorkingSetSize += PAGE_SIZE;
+ if (Vm->WorkingSetSize > Vm->PeakWorkingSetSize)
+ Vm->PeakWorkingSetSize = Vm->WorkingSetSize;
+}
+
+VOID
+NTAPI
+MiRemoveFromWorkingSetList(
+ _Inout_ PMMSUPPORT Vm,
+ _In_ PVOID Address)
+{
+ RemoveFromWsList(Vm->VmWorkingSetList, Address);
+
+ Vm->WorkingSetSize -= PAGE_SIZE;
+}
+
+_Requires_exclusive_lock_held_(WorkingSet->WorkingSetMutex)
+VOID
+NTAPI
+MiInitializeWorkingSetList(_Inout_ PMMSUPPORT WorkingSet)
+{
+ PMMWSL WsList = WorkingSet->VmWorkingSetList;
+
+ /* Initialize some fields */
+ WsList->FirstFree = ULONG_MAX;
+ WsList->Wsle = reinterpret_cast<PMMWSLE>(WsList + 1);
+ WsList->LastEntry = 0;
+ /* The first page is already allocated */
+ WsList->LastInitializedWsle = (PAGE_SIZE - sizeof(*WsList)) / sizeof(MMWSLE);
+
+ /* Insert the address we already know: our PDE base and the Working Set List */
+ if (MI_IS_PROCESS_WORKING_SET(WorkingSet))
+ {
+ ASSERT(WorkingSet->VmWorkingSetList == MmWorkingSetList);
+#if _MI_PAGING_LEVELS == 4
+ MiInsertInWorkingSetList(WorkingSet, (PVOID)PXE_BASE, 0U);
+#elif _MI_PAGING_LEVELS == 3
+ MiInsertInWorkingSetList(WorkingSet, (PVOID)PPE_BASE, 0U);
+#elif _MI_PAGING_LEVELS == 2
+ MiInsertInWorkingSetList(WorkingSet, (PVOID)PDE_BASE, 0U);
+#endif
+ }
+
+#if _MI_PAGING_LEVELS == 4
+ MiInsertInWorkingSetList(WorkingSet, MiAddressToPpe(WorkingSet->VmWorkingSetList),
0UL);
+#endif
+#if _MI_PAGING_LEVELS >= 3
+ MiInsertInWorkingSetList(WorkingSet, MiAddressToPde(WorkingSet->VmWorkingSetList),
0UL);
+#endif
+ MiInsertInWorkingSetList(WorkingSet,
(PVOID)MiAddressToPte(WorkingSet->VmWorkingSetList), 0UL);
+ MiInsertInWorkingSetList(WorkingSet, (PVOID)WorkingSet->VmWorkingSetList, 0UL);
+
+ /* From now on, every added page can be trimmed at any time */
+ WsList->FirstDynamic = WsList->LastEntry;
+
+ /* We can add this to our list */
+ ExInterlockedInsertTailList(&MmWorkingSetExpansionHead,
&WorkingSet->WorkingSetExpansionLinks, &MmExpansionLock);
+}
+
+VOID
+NTAPI
+MmWorkingSetManager(VOID)
+{
+ PLIST_ENTRY VmListEntry;
+ PMMSUPPORT Vm = NULL;
+ KIRQL OldIrql;
+
+ OldIrql = MiAcquireExpansionLock();
+
+ for (VmListEntry = MmWorkingSetExpansionHead.Flink;
+ VmListEntry != &MmWorkingSetExpansionHead;
+ VmListEntry = VmListEntry->Flink)
+ {
+ BOOLEAN TrimHard = MmAvailablePages < MmMinimumFreePages;
+ PEPROCESS Process = NULL;
+
+ /* Don't do anything if we have plenty of free pages. */
+ if ((MmAvailablePages + MmModifiedPageListHead.Total) >= MmPlentyFreePages)
+ break;
+
+ Vm = CONTAINING_RECORD(VmListEntry, MMSUPPORT, WorkingSetExpansionLinks);
+
+ /* Let the legacy Mm System space alone */
+ if (Vm == MmGetKernelAddressSpace())
+ continue;
+
+ if (MI_IS_PROCESS_WORKING_SET(Vm))
+ {
+ Process = CONTAINING_RECORD(Vm, EPROCESS, Vm);
+
+ /* Make sure the process is not terminating abd attach to it */
+ if (!ExAcquireRundownProtection(&Process->RundownProtect))
+ continue;
+ ASSERT(!KeIsAttachedProcess());
+ KeAttachProcess(&Process->Pcb);
+ }
+ else
+ {
+ /* FIXME: Session & system space unsupported */
+ continue;
+ }
+
+ MiReleaseExpansionLock(OldIrql);
+
+ /* Share-lock for now, we're only reading */
+ MiLockWorkingSetShared(PsGetCurrentThread(), Vm);
+
+ if (((Vm->WorkingSetSize > Vm->MaximumWorkingSetSize) ||
+ (TrimHard && (Vm->WorkingSetSize >
Vm->MinimumWorkingSetSize))) &&
+ MiConvertSharedWorkingSetLockToExclusive(PsGetCurrentThread(), Vm))
+ {
+ /* We're done */
+ Vm->Flags.BeingTrimmed = 1;
+
+ ULONG Trimmed = TrimWsList(Vm->VmWorkingSetList);
+
+ /* We're done */
+ Vm->WorkingSetSize -= Trimmed * PAGE_SIZE;
+ Vm->Flags.BeingTrimmed = 0;
+ MiUnlockWorkingSet(PsGetCurrentThread(), Vm);
+ }
+ else
+ {
+ MiUnlockWorkingSetShared(PsGetCurrentThread(), Vm);
+ }
+
+ /* Lock again */
+ OldIrql = MiAcquireExpansionLock();
+
+ if (Process)
+ {
+ KeDetachProcess();
+ ExReleaseRundownProtection(&Process->RundownProtect);
+ }
+ }
+
+ MiReleaseExpansionLock(OldIrql);
+}
+
+} // extern "C"
+
+} // namespace ntoskrnl
diff --git a/ntoskrnl/ntos.cmake b/ntoskrnl/ntos.cmake
index dbb9f911016..6e1edf2d944 100644
--- a/ntoskrnl/ntos.cmake
+++ b/ntoskrnl/ntos.cmake
@@ -229,6 +229,7 @@ list(APPEND SOURCE
${REACTOS_SOURCE_DIR}/ntoskrnl/mm/ARM3/syspte.c
${REACTOS_SOURCE_DIR}/ntoskrnl/mm/ARM3/vadnode.c
${REACTOS_SOURCE_DIR}/ntoskrnl/mm/ARM3/virtual.c
+ ${REACTOS_SOURCE_DIR}/ntoskrnl/mm/ARM3/wslist.cpp
${REACTOS_SOURCE_DIR}/ntoskrnl/mm/ARM3/zeropage.c
${REACTOS_SOURCE_DIR}/ntoskrnl/mm/balance.c
${REACTOS_SOURCE_DIR}/ntoskrnl/mm/freelist.c
diff --git a/sdk/include/ndk/mmtypes.h b/sdk/include/ndk/mmtypes.h
index 3e9fabf1c02..28d6cb00006 100644
--- a/sdk/include/ndk/mmtypes.h
+++ b/sdk/include/ndk/mmtypes.h
@@ -831,6 +831,21 @@ typedef struct _MMWSLENTRY
ULONG_PTR VirtualPageNumber: MM_PAGE_FRAME_NUMBER_SIZE;
} MMWSLENTRY, *PMMWSLENTRY;
+typedef struct _MMWSLE_FREE_ENTRY
+{
+ ULONG MustBeZero:1;
+#ifdef _WIN64
+ ULONG PreviousFree: 31;
+ LONG NextFree;
+#define MMWSLE_PREVIOUS_FREE_MASK 0x7FFFFFFF
+#else
+ ULONG PreviousFree: 11;
+#define MMWSLE_PREVIOUS_FREE_MASK 0x7FF
+#define MMWSLE_PREVIOUS_FREE_JUMP 0x800
+ LONG NextFree: 20;
+#endif
+} MMWSLE_FREE_ENTRY, *PMMWSLE_FREE_ENTRY;
+
typedef struct _MMWSLE
{
union
@@ -838,6 +853,7 @@ typedef struct _MMWSLE
PVOID VirtualAddress;
ULONG_PTR Long;
MMWSLENTRY e1;
+ MMWSLE_FREE_ENTRY Free;
} u1;
} MMWSLE, *PMMWSLE;