[reactos] 02/03: [NTOS/MM] - Fix PFNs tracing - Add private pages to the process working set
https://git.reactos.org/?p=reactos.git;a=commitdiff;h=4c5351bf55527a35bce094...
commit 4c5351bf55527a35bce09446420dd7296eedbe99 Author: Jérôme Gardou jerome.gardou@reactos.org AuthorDate: Fri Oct 16 15:27:07 2020 +0200 Commit: Jérôme Gardou jerome.gardou@reactos.org CommitDate: Tue Oct 20 15:20:59 2020 +0200
[NTOS/MM] - Fix PFNs tracing - Add private pages to the process working set --- ntoskrnl/cc/view.c | 20 ---- ntoskrnl/include/internal/mm.h | 44 ++++++++- ntoskrnl/mm/ARM3/miarm.h | 4 + ntoskrnl/mm/ARM3/mminit.c | 4 +- ntoskrnl/mm/ARM3/pagfault.c | 69 +++++++++++--- ntoskrnl/mm/ARM3/pfnlist.c | 13 ++- ntoskrnl/mm/ARM3/procsup.c | 212 ++++++++++++++++++++++++++++++++++++++++- ntoskrnl/mm/ARM3/sysldr.c | 9 +- ntoskrnl/mm/ARM3/virtual.c | 40 ++++---- ntoskrnl/mm/freelist.c | 15 +++ ntoskrnl/mm/i386/page.c | 6 ++ ntoskrnl/mm/marea.c | 5 +- ntoskrnl/mm/section.c | 11 --- 13 files changed, 364 insertions(+), 88 deletions(-)
diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index fde2bbc926a..c576997f943 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -665,7 +665,6 @@ CcRosMapVacbInKernelSpace( { PFN_NUMBER PageFrameNumber;
- MI_SET_USAGE(MI_USAGE_CACHE); Status = MmRequestPageMemoryConsumer(MC_CACHE, TRUE, &PageFrameNumber); if (PageFrameNumber == 0) { @@ -907,25 +906,6 @@ Retry: InsertTailList(&VacbLruListHead, ¤t->VacbLruListEntry); KeReleaseQueuedSpinLock(LockQueueMasterLock, oldIrql);
- MI_SET_USAGE(MI_USAGE_CACHE); -#if MI_TRACE_PFNS - if ((SharedCacheMap->FileObject) && (SharedCacheMap->FileObject->FileName.Buffer)) - { - PWCHAR pos; - ULONG len = 0; - pos = wcsrchr(SharedCacheMap->FileObject->FileName.Buffer, '\'); - if (pos) - { - len = wcslen(pos) * sizeof(WCHAR); - snprintf(MI_PFN_CURRENT_PROCESS_NAME, min(16, len), "%S", pos); - } - else - { - snprintf(MI_PFN_CURRENT_PROCESS_NAME, min(16, len), "%wZ", &SharedCacheMap->FileObject->FileName); - } - } -#endif - /* Reference it to allow release */ CcRosVacbIncRefCount(current);
diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 32fd7b1146d..d2ead92df1f 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -2,6 +2,8 @@
#include <internal/arch/mm.h>
+#define MI_TRACE_PFNS 1 + /* TYPES *********************************************************************/
struct _EPROCESS; @@ -248,9 +250,45 @@ MM_RMAP_ENTRY, *PMM_RMAP_ENTRY; extern ULONG MI_PFN_CURRENT_USAGE; extern CHAR MI_PFN_CURRENT_PROCESS_NAME[16]; #define MI_SET_USAGE(x) MI_PFN_CURRENT_USAGE = x -#define MI_SET_PROCESS2(x) memcpy(MI_PFN_CURRENT_PROCESS_NAME, x, 16) +#define MI_SET_PROCESS2(x) memcpy(MI_PFN_CURRENT_PROCESS_NAME, x, min(sizeof(x), sizeof(MI_PFN_CURRENT_PROCESS_NAME))) +FORCEINLINE +void +MI_SET_PROCESS(PEPROCESS Process) +{ + if (!Process) + MI_SET_PROCESS2("Kernel"); + else if (Process == (PEPROCESS)1) + MI_SET_PROCESS2("Hydra"); + else + MI_SET_PROCESS2(Process->ImageFileName); +} + +FORCEINLINE +void +MI_SET_PROCESS_USTR(PUNICODE_STRING ustr) +{ + PWSTR pos, strEnd; + int i; + + if (!ustr->Buffer || ustr->Length == 0) + { + MI_PFN_CURRENT_PROCESS_NAME[0] = 0; + return; + } + + pos = strEnd = &ustr->Buffer[ustr->Length / sizeof(WCHAR)]; + while ((*pos != L'\') && (pos > ustr->Buffer)) + pos--; + + if (*pos == L'\') + pos++; + + for (i = 0; i < sizeof(MI_PFN_CURRENT_PROCESS_NAME) && pos <= strEnd; i++, pos++) + MI_PFN_CURRENT_PROCESS_NAME[i] = (CHAR)*pos; +} #else #define MI_SET_USAGE(x) +#define MI_SET_PROCESS(x) #define MI_SET_PROCESS2(x) #endif
@@ -278,6 +316,9 @@ typedef enum _MI_PFN_USAGES MI_USAGE_PFN_DATABASE, MI_USAGE_BOOT_DRIVER, MI_USAGE_INIT_MEMORY, + MI_USAGE_PAGE_FILE, + MI_USAGE_COW, + MI_USAGE_WSLE, MI_USAGE_FREE_PAGE } MI_PFN_USAGES;
@@ -358,6 +399,7 @@ typedef struct _MMPFN #if MI_TRACE_PFNS MI_PFN_USAGES PfnUsage; CHAR ProcessName[16]; +#define MI_SET_PFN_PROCESS_NAME(pfn, x) memcpy(pfn->ProcessName, x, min(sizeof(x), sizeof(pfn->ProcessName))) #endif
// HACK until WS lists are supported diff --git a/ntoskrnl/mm/ARM3/miarm.h b/ntoskrnl/mm/ARM3/miarm.h index 6df6b7e5385..a30e9957874 100644 --- a/ntoskrnl/mm/ARM3/miarm.h +++ b/ntoskrnl/mm/ARM3/miarm.h @@ -1058,6 +1058,10 @@ VOID NTAPI MiInsertInWorkingSetList(_Inout_ PMMSUPPORT Vm, _In_ PVOID Address, _In_ ULONG Protection);
+VOID +NTAPI +MiRemoveFromWorkingSetList(_Inout_ PMMSUPPORT Vm, _In_ PVOID Address); + // // New ARM3<->RosMM PAGE Architecture // diff --git a/ntoskrnl/mm/ARM3/mminit.c b/ntoskrnl/mm/ARM3/mminit.c index 0bae38e3e02..3c724d8ea3e 100644 --- a/ntoskrnl/mm/ARM3/mminit.c +++ b/ntoskrnl/mm/ARM3/mminit.c @@ -801,7 +801,7 @@ MiBuildPfnDatabaseFromPages(IN PLOADER_PARAMETER_BLOCK LoaderBlock) Pfn1->u3.e1.CacheAttribute = MiNonCached; #if MI_TRACE_PFNS Pfn1->PfnUsage = MI_USAGE_INIT_MEMORY; - memcpy(Pfn1->ProcessName, "Initial PDE", 16); + MI_SET_PFN_PROCESS_NAME(Pfn1, "Initial PDE"); #endif } else @@ -848,7 +848,7 @@ MiBuildPfnDatabaseFromPages(IN PLOADER_PARAMETER_BLOCK LoaderBlock) Pfn2->u3.e1.CacheAttribute = MiNonCached; #if MI_TRACE_PFNS Pfn2->PfnUsage = MI_USAGE_INIT_MEMORY; - memcpy(Pfn1->ProcessName, "Initial PTE", 16); + MI_SET_PFN_PROCESS_NAME(Pfn2, "Initial PTE"); #endif } } diff --git a/ntoskrnl/mm/ARM3/pagfault.c b/ntoskrnl/mm/ARM3/pagfault.c index 1ab109e9f65..a487d3a1ba2 100644 --- a/ntoskrnl/mm/ARM3/pagfault.c +++ b/ntoskrnl/mm/ARM3/pagfault.c @@ -697,16 +697,6 @@ MiResolveDemandZeroFault(IN PVOID Address, /* Increment demand zero faults */ KeGetCurrentPrcb()->MmDemandZeroCount++;
- /* Do we have the lock? */ - if (HaveLock) - { - /* Release it */ - MiReleasePfnLock(OldIrql); - - /* Update performance counters */ - if (Process > HYDRA_PROCESS) Process->NumberOfPrivatePages++; - } - /* Zero the page if need be */ if (NeedZero) MiZeroPfn(PageFrameNumber);
@@ -745,6 +735,23 @@ MiResolveDemandZeroFault(IN PVOID Address, ASSERT(Pfn1->u3.e1.PrototypePte == 0); }
+ /* Add the page to our working set, if it's not a proto PTE */ + if ((Process > HYDRA_PROCESS) && (PointerPte == MiAddressToPte(Address))) + { + /* FIXME: Also support session VM scenario */ + MiInsertInWorkingSetList(&Process->Vm, Address, Protection); + } + + /* Do we have the lock? */ + if (HaveLock) + { + /* Release it */ + MiReleasePfnLock(OldIrql); + + /* Update performance counters */ + if (Process > HYDRA_PROCESS) Process->NumberOfPrivatePages++; + } + // // It's all good now // @@ -899,6 +906,9 @@ MiResolvePageFileFault(_In_ BOOLEAN StoreInstruction, ASSERT(CurrentProcess > HYDRA_PROCESS); ASSERT(*OldIrql != MM_NOIRQL);
+ MI_SET_USAGE(MI_USAGE_PAGE_FILE); + MI_SET_PROCESS(CurrentProcess); + /* We must hold the PFN lock */ MI_ASSERT_PFN_LOCK_HELD();
@@ -959,6 +969,9 @@ MiResolvePageFileFault(_In_ BOOLEAN StoreInstruction, KeSetEvent(Pfn1->u1.Event, IO_NO_INCREMENT, FALSE); }
+ /* And we can insert this into the working set */ + MiInsertInWorkingSetList(&CurrentProcess->Vm, FaultingAddress, Protection); + return Status; }
@@ -976,6 +989,8 @@ MiResolveTransitionFault(IN BOOLEAN StoreInstruction, PMMPFN Pfn1; MMPTE TempPte; PMMPTE PointerToPteForProtoPage; + ULONG Protection; + DPRINT("Transition fault on 0x%p with PTE 0x%p in process %s\n", FaultingAddress, PointerPte, CurrentProcess->ImageFileName);
@@ -1069,8 +1084,9 @@ MiResolveTransitionFault(IN BOOLEAN StoreInstruction, ASSERT(PointerPte->u.Hard.Valid == 0); ASSERT(PointerPte->u.Trans.Prototype == 0); ASSERT(PointerPte->u.Trans.Transition == 1); + Protection = TempPte.u.Trans.Protection; TempPte.u.Long = (PointerPte->u.Long & ~0xFFF) | - (MmProtectToPteMask[PointerPte->u.Trans.Protection]) | + (MmProtectToPteMask[Protection]) | MiDetermineUserGlobalPteMask(PointerPte);
/* Is the PTE writeable? */ @@ -1090,6 +1106,10 @@ MiResolveTransitionFault(IN BOOLEAN StoreInstruction, /* Write the valid PTE */ MI_WRITE_VALID_PTE(PointerPte, TempPte);
+ /* If this was a user fault, add it to the working set */ + if (CurrentProcess > HYDRA_PROCESS) + MiInsertInWorkingSetList(&CurrentProcess->Vm, FaultingAddress, Protection); + /* Return success */ return STATUS_PAGE_FAULT_TRANSITION; } @@ -1210,6 +1230,9 @@ MiResolveProtoPteFault(IN BOOLEAN StoreInstruction, ASSERT(TempPte.u.Hard.Valid == 1); ProtoPageFrameIndex = PFN_FROM_PTE(&TempPte);
+ MI_SET_USAGE(MI_USAGE_COW); + MI_SET_PROCESS(Process); + /* Get a new page for the private copy */ if (Process > HYDRA_PROCESS) Color = MI_GET_NEXT_PROCESS_COLOR(Process); @@ -1245,6 +1268,13 @@ MiResolveProtoPteFault(IN BOOLEAN StoreInstruction, /* And finally, write the valid PTE */ MI_WRITE_VALID_PTE(PointerPte, PteContents);
+ /* Add the page to our working set */ + if (Process > HYDRA_PROCESS) + { + /* FIXME: Also support session VM scenario */ + MiInsertInWorkingSetList(&Process->Vm, Address, Protection); + } + /* The caller expects us to release the PFN lock */ MiReleasePfnLock(OldIrql); return Status; @@ -2205,11 +2235,15 @@ UserFault: { PFN_NUMBER PageFrameIndex, OldPageFrameIndex; PMMPFN Pfn1; + ProtectionCode = TempPte.u.Soft.Protection;
LockIrql = MiAcquirePfnLock();
ASSERT(MmAvailablePages > 0);
+ MI_SET_USAGE(MI_USAGE_COW); + MI_SET_PROCESS(CurrentProcess); + /* Allocate a new page and copy it */ PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(CurrentProcess)); OldPageFrameIndex = PFN_FROM_PTE(&TempPte); @@ -2231,6 +2265,9 @@ UserFault:
MI_WRITE_VALID_PTE(PointerPte, TempPte);
+ /* We can now add it to our working set */ + MiInsertInWorkingSetList(&CurrentProcess->Vm, Address, ProtectionCode); + MiReleasePfnLock(LockIrql);
/* Return the status */ @@ -2347,6 +2384,7 @@ UserFault: TempPte.u.Soft.Protection = ProtectionCode; MI_WRITE_INVALID_PTE(PointerPte, TempPte); } + ProtectionCode = PointerPte->u.Soft.Protection;
/* Lock the PFN database since we're going to grab a page */ OldIrql = MiAcquirePfnLock(); @@ -2384,9 +2422,6 @@ UserFault: /* One more demand-zero fault */ KeGetCurrentPrcb()->MmDemandZeroCount++;
- /* And we're done with the lock */ - MiReleasePfnLock(OldIrql); - /* Fault on user PDE, or fault on user PTE? */ if (PointerPte <= MiHighestUserPte) { @@ -2413,6 +2448,12 @@ UserFault: Pfn1 = MI_PFN_ELEMENT(PageFrameIndex); ASSERT(Pfn1->u1.Event == NULL);
+ /* We can now insert it into the working set */ + MiInsertInWorkingSetList(&CurrentProcess->Vm, Address, ProtectionCode); + + /* And we're done with the lock */ + MiReleasePfnLock(OldIrql); + /* Demand zero */ ASSERT(KeGetCurrentIrql() <= APC_LEVEL); MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread); diff --git a/ntoskrnl/mm/ARM3/pfnlist.c b/ntoskrnl/mm/ARM3/pfnlist.c index b039a5181a2..866a29cd15f 100644 --- a/ntoskrnl/mm/ARM3/pfnlist.c +++ b/ntoskrnl/mm/ARM3/pfnlist.c @@ -254,8 +254,8 @@ MiUnlinkFreeOrZeroedPage(IN PMMPFN Entry) ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET); Entry->PfnUsage = MI_PFN_CURRENT_USAGE; memcpy(Entry->ProcessName, MI_PFN_CURRENT_PROCESS_NAME, 16); -// MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET; -// memcpy(MI_PFN_CURRENT_PROCESS_NAME, "Not Set", 16); + MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET; + MI_SET_PROCESS2("Not Set"); #endif }
@@ -459,11 +459,11 @@ MiRemovePageByColor(IN PFN_NUMBER PageIndex, MiDecrementAvailablePages();
#if MI_TRACE_PFNS - //ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET); + ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET); Pfn1->PfnUsage = MI_PFN_CURRENT_USAGE; memcpy(Pfn1->ProcessName, MI_PFN_CURRENT_PROCESS_NAME, 16); - //MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET; - //memcpy(MI_PFN_CURRENT_PROCESS_NAME, "Not Set", 16); + MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET; + MI_SET_PROCESS2("Not Set"); #endif
/* Return the page */ @@ -937,9 +937,8 @@ MiInsertPageInList(IN PMMPFNLIST ListHead, ColorHead->Count++;
#if MI_TRACE_PFNS - //ASSERT(MI_PFN_CURRENT_USAGE == MI_USAGE_NOT_SET); + ASSERT(MI_PFN_CURRENT_USAGE == MI_USAGE_NOT_SET); Pfn1->PfnUsage = MI_USAGE_FREE_PAGE; - MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET; RtlZeroMemory(Pfn1->ProcessName, 16); #endif } diff --git a/ntoskrnl/mm/ARM3/procsup.c b/ntoskrnl/mm/ARM3/procsup.c index ab01d902686..2e4d9029d36 100644 --- a/ntoskrnl/mm/ARM3/procsup.c +++ b/ntoskrnl/mm/ARM3/procsup.c @@ -6,6 +6,8 @@ * PROGRAMMERS: ReactOS Portable Systems Group */
+#define GROW_WSLE 1 + /* INCLUDES *******************************************************************/
#include <ntoskrnl.h> @@ -862,18 +864,34 @@ MiGetFirstFreeWsleIndex(_Inout_ PMMSUPPORT Vm) if (WsList->LastEntry == WsList->LastInitializedWsle) { /* We must grow our array. Allocate a new page */ - PMMPTE PointerPte = MiAddressToPte(&WsList->Wsle[WsList->LastInitializedWsle + 1]); + PVOID Address = &WsList->Wsle[WsList->LastInitializedWsle + 1]; + PMMPTE PointerPte = MiAddressToPte(Address); MMPTE TempPte; + + MI_SET_USAGE(MI_USAGE_WSLE); + MI_SET_PROCESS(PsGetCurrentProcess()); + + /* We must be at page boundary */ + ASSERT(Address == ALIGN_DOWN_POINTER_BY(Address, PAGE_SIZE)); + PFN_NUMBER PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
MiInitializePfn(PageFrameIndex, PointerPte, TRUE); - MI_MAKE_HARDWARE_PTE_KERNEL(&TempPte, PointerPte, MM_READWRITE, PageFrameIndex); + + TempPte = ValidKernelPteLocal; + TempPte.u.Hard.PageFrameNumber = PageFrameIndex; MI_WRITE_VALID_PTE(PointerPte, TempPte);
WsList->LastInitializedWsle += PAGE_SIZE / sizeof(MMWSLE);
+ /* Make sure we are staying on the same page */ + ASSERT(Address == ALIGN_DOWN_POINTER_BY(&WsList->Wsle[WsList->LastInitializedWsle], PAGE_SIZE)); + /* We must insert this page in our working set ! */ MiInsertInWorkingSetList(Vm, &WsList->Wsle[WsList->LastInitializedWsle], MM_READWRITE); + + /* Now the last entry is the tail of our WSLE array */ + ASSERT(WsList->Wsle[WsList->LastEntry].u1.e1.VirtualPageNumber == ((ULONG_PTR)&WsList->Wsle[WsList->LastInitializedWsle]) >> PAGE_SHIFT); }
/* At this point we must be good to go */ @@ -909,18 +927,26 @@ MiGetFirstFreeWsleIndex(_Inout_ PMMSUPPORT Vm)
VOID NTAPI -MiInsertInWorkingSetList(_Inout_ PMMSUPPORT Vm, _In_ PVOID Address, _In_ ULONG Protection) +MiInsertInWorkingSetList( + _Inout_ PMMSUPPORT Vm, + _In_ PVOID Address, + _In_ ULONG Protection) { ULONG WsIndex = MiGetFirstFreeWsleIndex(Vm); PMMWSLE WsleEntry = &Vm->VmWorkingSetList->Wsle[WsIndex]; PMMPTE PointerPte = MiAddressToPte(Address); - PMMPFN Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte)); + PMMPFN Pfn1;
/* Make sure we got a rounded address */ Address = ALIGN_DOWN_POINTER_BY(Address, PAGE_SIZE);
- /* Make sure we are locking the right thing */ + /* Make sure we are locking the right things */ ASSERT(MM_ANY_WS_LOCK_HELD(PsGetCurrentThread())); + MI_ASSERT_PFN_LOCK_HELD(); + + /* Make sure we are adding a paged-in address */ + ASSERT(PointerPte->u.Hard.Valid == 1); + Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte));
/* The Pfn must be an active one */ ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid); @@ -931,6 +957,11 @@ MiInsertInWorkingSetList(_Inout_ PMMSUPPORT Vm, _In_ PVOID Address, _In_ ULONG P
/* Shared pages not supported yet */ ASSERT(Pfn1->u1.WsIndex == 0); + ASSERT(Pfn1->u3.e1.PrototypePte == 0); + + /* Nor are "ROS PFN" */ + ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE); + WsleEntry->u1.e1.Direct = 1;
Pfn1->u1.WsIndex = WsIndex; @@ -941,6 +972,177 @@ MiInsertInWorkingSetList(_Inout_ PMMSUPPORT Vm, _In_ PVOID Address, _In_ ULONG P Vm->PeakWorkingSetSize = Vm->WorkingSetSize; }
+static +void +MiShrinkWorkingSet(_Inout_ PMMSUPPORT Vm) +{ + PMMWSL WsList = Vm->VmWorkingSetList; + ULONG LastValid = WsList->LastEntry; + + while(WsList->Wsle[LastValid].u1.e1.Valid == 0) + { + LastValid--; + } + + if (LastValid != WsList->LastEntry) + { + /* There was a hole behind us. Handle this */ + PMMWSLE NextFree = &WsList->Wsle[LastValid + 1]; + if (NextFree->u1.Free.PreviousFree == MMWSLE_PREVIOUS_FREE_INVALID) + { + /* This was actually our first free entry. */ + ASSERT(WsList->FirstFree == LastValid + 1); + WsList->FirstFree = MMWSLE_NEXT_FREE_INVALID; + } + else + { + /* The previous one is now the last in the queue */ + PMMWSLE PreviousFree = &WsList->Wsle[NextFree->u1.Free.PreviousFree]; + + ASSERT(PreviousFree->u1.Free.MustBeZero == 0); + PreviousFree->u1.Free.NextFree = MMWSLE_NEXT_FREE_INVALID; + } + + /* Nuke everyone */ + RtlZeroMemory(&WsList->Wsle[LastValid + 1], (WsList->LastEntry - LastValid) * sizeof(MMWSLE)); + WsList->LastEntry = LastValid; + } + + if (LastValid < WsList->FirstDynamic) + { + /* Do not mess around with the protected ones */ + return; + } + + /* See if we should shrink our array */ + if (LastValid == (WsList->LastInitializedWsle - (PAGE_SIZE / sizeof(MMWSLE)) + 1)) + { + PVOID WsleArrayQueue = ALIGN_DOWN_POINTER_BY(&WsList->Wsle[LastValid], PAGE_SIZE); + PEPROCESS Process = MmGetAddressSpaceOwner(Vm); + + ASSERT(WsList->Wsle[WsList->LastEntry].u1.e1.VirtualPageNumber == ((ULONG_PTR)WsleArrayQueue) >> PAGE_SHIFT); + + /* Kernel address space not supported yet */ + ASSERT(Process != NULL); + + /* Nuke the PTE. This will remove the virtual address from the working set */ + MiDeletePte(MiAddressToPte(WsleArrayQueue), WsleArrayQueue, Process, NULL); + } +} + +VOID +NTAPI +MiRemoveFromWorkingSetList( + _Inout_ PMMSUPPORT Vm, + _In_ PVOID Address) +{ + PMMWSL WsList = Vm->VmWorkingSetList; + ULONG WsIndex; + PMMWSLE WsleEntry; + PMMPTE PointerPte = MiAddressToPte(Address); + PMMPFN Pfn1; + + /* Make sure we got a rounded address */ + Address = ALIGN_DOWN_POINTER_BY(Address, PAGE_SIZE); + + /* Make sure we are locking the right things */ + ASSERT(MM_ANY_WS_LOCK_HELD(PsGetCurrentThread())); + MI_ASSERT_PFN_LOCK_HELD(); + + /* Make sure we are removing a paged-in address */ + ASSERT(PointerPte->u.Hard.Valid == 1); + Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte)); + + /* The Pfn must be an active one */ + ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid); + + WsIndex = Pfn1->u1.WsIndex; + WsleEntry = &Vm->VmWorkingSetList->Wsle[WsIndex]; + + /* Shared page not handled yet */ + ASSERT(Pfn1->u3.e1.PrototypePte == 0); + ASSERT(WsleEntry->u1.e1.Direct == 1); + /* Nor are "ROS PFN" */ + ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE); + + /* Some sanity checks */ + ASSERT(WsIndex >= WsList->FirstDynamic); + ASSERT(WsIndex <= WsList->LastEntry); + ASSERT(WsIndex <= WsList->LastInitializedWsle); + ASSERT(WsleEntry->u1.e1.Valid == 1); + ASSERT(WsleEntry->u1.e1.VirtualPageNumber == ((ULONG_PTR)Address) >> PAGE_SHIFT); + + /* Let this go */ + Pfn1->u1.WsIndex = 0; + + /* Nuke it */ + WsleEntry->u1.Long = 0; + + /* Insert our entry into the free list */ + if (WsIndex == WsList->LastEntry) + { + /* Let's shrink the active list */ + WsList->LastEntry--; + MiShrinkWorkingSet(Vm); + } + else if (WsList->FirstFree > WsList->LastEntry) + { + /* We are the first free entry to be inserted */ + WsList->FirstFree = WsIndex; + WsleEntry->u1.Free.PreviousFree = MMWSLE_PREVIOUS_FREE_INVALID; + WsleEntry->u1.Free.NextFree = MMWSLE_NEXT_FREE_INVALID; + } + else + { + /* Keep this sorted */ + PMMWSLE NextFree = &WsList->Wsle[WsList->FirstFree]; + PMMWSLE PreviousFree = NULL; + + ASSERT(NextFree->u1.Free.MustBeZero == 0); + + while (NextFree < WsleEntry) + { + PreviousFree = NextFree; + if (NextFree->u1.Free.NextFree != MMWSLE_NEXT_FREE_INVALID) + { + NextFree = &WsList->Wsle[NextFree->u1.Free.NextFree]; + ASSERT(NextFree->u1.Free.MustBeZero == 0); + } + else + { + NextFree = NULL; + break; + } + } + + ASSERT(PreviousFree || NextFree); + + if (PreviousFree) + { + ASSERT((NextFree != NULL) || (PreviousFree->u1.Free.NextFree == MMWSLE_NEXT_FREE_INVALID)); + PreviousFree->u1.Free.NextFree = WsIndex; + WsleEntry->u1.Free.PreviousFree = PreviousFree - WsList->Wsle; + } + else + { + WsleEntry->u1.Free.PreviousFree = MMWSLE_PREVIOUS_FREE_INVALID; + ASSERT(NextFree->u1.Free.PreviousFree == MMWSLE_PREVIOUS_FREE_INVALID); + WsList->FirstFree = WsIndex; + } + + if (NextFree) + { + NextFree->u1.Free.PreviousFree = WsIndex; + WsleEntry->u1.Free.NextFree = NextFree - WsList->Wsle; + } + else + { + WsleEntry->u1.Free.NextFree = MMWSLE_NEXT_FREE_INVALID; + } + } + + Vm->WorkingSetSize -= PAGE_SIZE; +}
VOID NTAPI diff --git a/ntoskrnl/mm/ARM3/sysldr.c b/ntoskrnl/mm/ARM3/sysldr.c index d414b1f6b43..58e4c3057b2 100644 --- a/ntoskrnl/mm/ARM3/sysldr.c +++ b/ntoskrnl/mm/ARM3/sysldr.c @@ -188,14 +188,7 @@ MiLoadImageSection(IN OUT PVOID *SectionPtr, /* Some debug stuff */ MI_SET_USAGE(MI_USAGE_DRIVER_PAGE); #if MI_TRACE_PFNS - if (FileName->Buffer) - { - PWCHAR pos = NULL; - ULONG len = 0; - pos = wcsrchr(FileName->Buffer, '\'); - len = wcslen(pos) * sizeof(WCHAR); - if (pos) snprintf(MI_PFN_CURRENT_PROCESS_NAME, min(16, len), "%S", pos); - } + MI_SET_PROCESS_USTR(FileName); #endif
/* Grab a page */ diff --git a/ntoskrnl/mm/ARM3/virtual.c b/ntoskrnl/mm/ARM3/virtual.c index f0ab5d08f7b..630afc5c990 100644 --- a/ntoskrnl/mm/ARM3/virtual.c +++ b/ntoskrnl/mm/ARM3/virtual.c @@ -503,6 +503,9 @@ MiDeletePte(IN PMMPTE PointerPte, } else { + /* Remove this address from the WS list */ + MiRemoveFromWorkingSetList(&CurrentProcess->Vm, VirtualAddress); + /* Make sure the saved PTE address is valid */ if ((PMMPTE)((ULONG_PTR)Pfn1->PteAddress & ~0x1) != PointerPte) { @@ -2303,13 +2306,15 @@ MiProtectVirtualMemory(IN PEPROCESS Process, { KIRQL OldIrql = MiAcquirePfnLock();
+ /* Remove this from the working set */ + MiRemoveFromWorkingSetList(AddressSpace, MiPteToAddress(PointerPte)); + /* Mark the PTE as transition and change its protection */ PteContents.u.Hard.Valid = 0; PteContents.u.Soft.Transition = 1; PteContents.u.Trans.Protection = ProtectionMask; /* Decrease PFN share count and write the PTE */ MiDecrementShareCount(Pfn1, PFN_FROM_PTE(&PteContents)); - // FIXME: remove the page from the WS MI_WRITE_INVALID_PTE(PointerPte, PteContents); #ifdef CONFIG_SMP // FIXME: Should invalidate entry in every CPU TLB @@ -2447,8 +2452,10 @@ MiMakePdeExistAndMakeValid(IN PMMPDE PointerPde,
VOID NTAPI -MiProcessValidPteList(IN PMMPTE *ValidPteList, - IN ULONG Count) +MiProcessValidPteList( + _Inout_ PMMSUPPORT Vm, + _Inout_ PMMPTE *ValidPteList, + _In_ ULONG Count) { KIRQL OldIrql; ULONG i; @@ -2468,6 +2475,11 @@ MiProcessValidPteList(IN PMMPTE *ValidPteList, TempPte = *ValidPteList[i]; ASSERT(TempPte.u.Hard.Valid == 1);
+ // + // We can now remove this addres from the working set + // + MiRemoveFromWorkingSetList(Vm, MiPteToAddress(ValidPteList[i])); + // // Get the PFN entry for the page itself, and then for its page table // @@ -2509,7 +2521,6 @@ MiDecommitPages(IN PVOID StartingAddress, ULONG CommitReduction = 0; PMMPTE ValidPteList[256]; ULONG PteCount = 0; - PMMPFN Pfn1; MMPTE PteContents; PETHREAD CurrentThread = PsGetCurrentThread();
@@ -2541,10 +2552,10 @@ MiDecommitPages(IN PVOID StartingAddress, // such, and does not flush the entire TLB all the time, but right // now we have bigger problems to worry about than TLB flushing. // - PointerPde = MiAddressToPde(StartingAddress); + PointerPde = MiPteToPde(PointerPte); if (PteCount) { - MiProcessValidPteList(ValidPteList, PteCount); + MiProcessValidPteList(&Process->Vm, ValidPteList, PteCount); PteCount = 0; }
@@ -2579,21 +2590,13 @@ MiDecommitPages(IN PVOID StartingAddress, //Process->NumberOfPrivatePages--; if (PteContents.u.Hard.Valid) { - // - // It's valid. At this point make sure that it is not a ROS - // PFN. Also, we don't support ProtoPTEs in this code path. - // - Pfn1 = MiGetPfnEntry(PteContents.u.Hard.PageFrameNumber); - ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE); - ASSERT(Pfn1->u3.e1.PrototypePte == FALSE); - // // Flush any pending PTEs that we had not yet flushed, if our // list has gotten too big, then add this PTE to the flush list. // if (PteCount == 256) { - MiProcessValidPteList(ValidPteList, PteCount); + MiProcessValidPteList(&Process->Vm, ValidPteList, PteCount); PteCount = 0; } ValidPteList[PteCount++] = PointerPte; @@ -2623,7 +2626,7 @@ MiDecommitPages(IN PVOID StartingAddress, // This used to be a zero PTE and it no longer is, so we must add a // reference to the pagetable. // - MiIncrementPageTableReferences(StartingAddress); + MiIncrementPageTableReferences(MiPteToAddress(PointerPte));
// // Next, we account for decommitted PTEs and make the PTE as such @@ -2633,17 +2636,16 @@ MiDecommitPages(IN PVOID StartingAddress, }
// - // Move to the next PTE and the next address + // Move to the next PTE // PointerPte++; - StartingAddress = (PVOID)((ULONG_PTR)StartingAddress + PAGE_SIZE); }
// // Flush any dangling PTEs from the loop in the last page table, and then // release the working set and return the commit reduction accounting. // - if (PteCount) MiProcessValidPteList(ValidPteList, PteCount); + if (PteCount) MiProcessValidPteList(&Process->Vm, ValidPteList, PteCount); MiUnlockProcessWorkingSetUnsafe(Process, CurrentThread); return CommitReduction; } diff --git a/ntoskrnl/mm/freelist.c b/ntoskrnl/mm/freelist.c index 93d9bbd9f59..80447593ecd 100644 --- a/ntoskrnl/mm/freelist.c +++ b/ntoskrnl/mm/freelist.c @@ -574,6 +574,21 @@ MmAllocPage(ULONG Type)
OldIrql = MiAcquirePfnLock();
+#if MI_TRACE_PFNS + switch(Type) + { + case MC_CACHE: + case MC_SYSTEM: + MI_SET_USAGE(MI_USAGE_CACHE); + break; + case MC_USER: + MI_SET_USAGE(MI_USAGE_SECTION); + break; + default: + ASSERT(FALSE); + } +#endif + PfnOffset = MiRemoveZeroPage(MI_GET_NEXT_COLOR()); if (!PfnOffset) { diff --git a/ntoskrnl/mm/i386/page.c b/ntoskrnl/mm/i386/page.c index 6b5ac1e8353..f8337dadc26 100644 --- a/ntoskrnl/mm/i386/page.c +++ b/ntoskrnl/mm/i386/page.c @@ -264,6 +264,10 @@ MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create)
MI_WRITE_INVALID_PTE(PointerPde, DemandZeroPde); // Tiny HACK: Parameter 1 is the architecture specific FaultCode for an access violation (i.e. page is present) + + /* Lock the working set, as this will add this address to it */ + MiLockProcessWorkingSetUnsafe(Process, PsGetCurrentThread()); + Status = MiDispatchFault(0x1, Pt, PointerPde, @@ -275,6 +279,8 @@ MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create) DBG_UNREFERENCED_LOCAL_VARIABLE(Status); ASSERT(KeAreAllApcsDisabled() == TRUE); ASSERT(PointerPde->u.Hard.Valid == 1); + + MiUnlockProcessWorkingSetUnsafe(Process, PsGetCurrentThread()); } return (PULONG)MiAddressToPte(Address); } diff --git a/ntoskrnl/mm/marea.c b/ntoskrnl/mm/marea.c index d953f422f96..a68adecd9be 100644 --- a/ntoskrnl/mm/marea.c +++ b/ntoskrnl/mm/marea.c @@ -331,13 +331,16 @@ MmFreeMemoryArea( ASSERT(AddressSpace != MmGetKernelAddressSpace()); if (MiQueryPageTableReferences((PVOID)Address) == 0) { + KIRQL OldIrql; /* No PTE relies on this PDE. Release it */ - KIRQL OldIrql = MiAcquirePfnLock(); + MiLockProcessWorkingSet(Process, PsGetCurrentThread()); + OldIrql = MiAcquirePfnLock(); PMMPDE PointerPde = MiAddressToPde(Address); ASSERT(PointerPde->u.Hard.Valid == 1); MiDeletePte(PointerPde, MiPdeToPte(PointerPde), Process, NULL); ASSERT(PointerPde->u.Hard.Valid == 0); MiReleasePfnLock(OldIrql); + MiUnlockProcessWorkingSet(Process, PsGetCurrentThread()); } } #endif diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index ce63e796a3d..0825d4ecb49 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -1163,8 +1163,6 @@ MiReadPage(PMEMORY_AREA MemoryArea, * Allocate a page, this is rather complicated by the possibility * we might have to move other things out of memory */ - MI_SET_USAGE(MI_USAGE_SECTION); - MI_SET_PROCESS2(PsGetCurrentProcess()->ImageFileName); Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, Page); if (!NT_SUCCESS(Status)) { @@ -1624,9 +1622,6 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, ((Offset.QuadPart >= (LONGLONG)PAGE_ROUND_UP(Segment->RawLength.QuadPart) && (Section->AllocationAttributes & SEC_IMAGE)))) { - MI_SET_USAGE(MI_USAGE_SECTION); - if (Process) MI_SET_PROCESS2(Process->ImageFileName); - if (!Process) MI_SET_PROCESS2("Kernel Section"); Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &Page); if (!NT_SUCCESS(Status)) { @@ -1707,9 +1702,6 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, MmUnlockSectionSegment(Segment);
MmUnlockAddressSpace(AddressSpace); - MI_SET_USAGE(MI_USAGE_SECTION); - if (Process) MI_SET_PROCESS2(Process->ImageFileName); - if (!Process) MI_SET_PROCESS2("Kernel Section"); Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &Page); if (!NT_SUCCESS(Status)) { @@ -1878,9 +1870,6 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, /* * Allocate a page */ - MI_SET_USAGE(MI_USAGE_SECTION); - if (Process) MI_SET_PROCESS2(Process->ImageFileName); - if (!Process) MI_SET_PROCESS2("Kernel Section"); Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &NewPage); if (!NT_SUCCESS(Status)) {
-
Jérôme Gardou