20 Oct
2020
20 Oct
'20
1:21 p.m.
https://git.reactos.org/?p=reactos.git;a=commitdiff;h=4c5351bf55527a35bce09…
commit 4c5351bf55527a35bce09446420dd7296eedbe99
Author: Jérôme Gardou <jerome.gardou(a)reactos.org>
AuthorDate: Fri Oct 16 15:27:07 2020 +0200
Commit: Jérôme Gardou <jerome.gardou(a)reactos.org>
CommitDate: Tue Oct 20 15:20:59 2020 +0200
[NTOS/MM]
- Fix PFNs tracing
- Add private pages to the process working set
---
ntoskrnl/cc/view.c | 20 ----
ntoskrnl/include/internal/mm.h | 44 ++++++++-
ntoskrnl/mm/ARM3/miarm.h | 4 +
ntoskrnl/mm/ARM3/mminit.c | 4 +-
ntoskrnl/mm/ARM3/pagfault.c | 69 +++++++++++---
ntoskrnl/mm/ARM3/pfnlist.c | 13 ++-
ntoskrnl/mm/ARM3/procsup.c | 212 ++++++++++++++++++++++++++++++++++++++++-
ntoskrnl/mm/ARM3/sysldr.c | 9 +-
ntoskrnl/mm/ARM3/virtual.c | 40 ++++----
ntoskrnl/mm/freelist.c | 15 +++
ntoskrnl/mm/i386/page.c | 6 ++
ntoskrnl/mm/marea.c | 5 +-
ntoskrnl/mm/section.c | 11 ---
13 files changed, 364 insertions(+), 88 deletions(-)
diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c
index fde2bbc926a..c576997f943 100644
--- a/ntoskrnl/cc/view.c
+++ b/ntoskrnl/cc/view.c
@@ -665,7 +665,6 @@ CcRosMapVacbInKernelSpace(
{
PFN_NUMBER PageFrameNumber;
- MI_SET_USAGE(MI_USAGE_CACHE);
Status = MmRequestPageMemoryConsumer(MC_CACHE, TRUE, &PageFrameNumber);
if (PageFrameNumber == 0)
{
@@ -907,25 +906,6 @@ Retry:
InsertTailList(&VacbLruListHead, ¤t->VacbLruListEntry);
KeReleaseQueuedSpinLock(LockQueueMasterLock, oldIrql);
- MI_SET_USAGE(MI_USAGE_CACHE);
-#if MI_TRACE_PFNS
- if ((SharedCacheMap->FileObject) &&
(SharedCacheMap->FileObject->FileName.Buffer))
- {
- PWCHAR pos;
- ULONG len = 0;
- pos = wcsrchr(SharedCacheMap->FileObject->FileName.Buffer, '\\');
- if (pos)
- {
- len = wcslen(pos) * sizeof(WCHAR);
- snprintf(MI_PFN_CURRENT_PROCESS_NAME, min(16, len), "%S", pos);
- }
- else
- {
- snprintf(MI_PFN_CURRENT_PROCESS_NAME, min(16, len), "%wZ",
&SharedCacheMap->FileObject->FileName);
- }
- }
-#endif
-
/* Reference it to allow release */
CcRosVacbIncRefCount(current);
diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h
index 32fd7b1146d..d2ead92df1f 100644
--- a/ntoskrnl/include/internal/mm.h
+++ b/ntoskrnl/include/internal/mm.h
@@ -2,6 +2,8 @@
#include <internal/arch/mm.h>
+#define MI_TRACE_PFNS 1
+
/* TYPES *********************************************************************/
struct _EPROCESS;
@@ -248,9 +250,45 @@ MM_RMAP_ENTRY, *PMM_RMAP_ENTRY;
extern ULONG MI_PFN_CURRENT_USAGE;
extern CHAR MI_PFN_CURRENT_PROCESS_NAME[16];
#define MI_SET_USAGE(x) MI_PFN_CURRENT_USAGE = x
-#define MI_SET_PROCESS2(x) memcpy(MI_PFN_CURRENT_PROCESS_NAME, x, 16)
+#define MI_SET_PROCESS2(x) memcpy(MI_PFN_CURRENT_PROCESS_NAME, x, min(sizeof(x),
sizeof(MI_PFN_CURRENT_PROCESS_NAME)))
+FORCEINLINE
+void
+MI_SET_PROCESS(PEPROCESS Process)
+{
+ if (!Process)
+ MI_SET_PROCESS2("Kernel");
+ else if (Process == (PEPROCESS)1)
+ MI_SET_PROCESS2("Hydra");
+ else
+ MI_SET_PROCESS2(Process->ImageFileName);
+}
+
+FORCEINLINE
+void
+MI_SET_PROCESS_USTR(PUNICODE_STRING ustr)
+{
+ PWSTR pos, strEnd;
+ int i;
+
+ if (!ustr->Buffer || ustr->Length == 0)
+ {
+ MI_PFN_CURRENT_PROCESS_NAME[0] = 0;
+ return;
+ }
+
+ pos = strEnd = &ustr->Buffer[ustr->Length / sizeof(WCHAR)];
+ while ((*pos != L'\\') && (pos > ustr->Buffer))
+ pos--;
+
+ if (*pos == L'\\')
+ pos++;
+
+ for (i = 0; i < sizeof(MI_PFN_CURRENT_PROCESS_NAME) && pos <= strEnd;
i++, pos++)
+ MI_PFN_CURRENT_PROCESS_NAME[i] = (CHAR)*pos;
+}
#else
#define MI_SET_USAGE(x)
+#define MI_SET_PROCESS(x)
#define MI_SET_PROCESS2(x)
#endif
@@ -278,6 +316,9 @@ typedef enum _MI_PFN_USAGES
MI_USAGE_PFN_DATABASE,
MI_USAGE_BOOT_DRIVER,
MI_USAGE_INIT_MEMORY,
+ MI_USAGE_PAGE_FILE,
+ MI_USAGE_COW,
+ MI_USAGE_WSLE,
MI_USAGE_FREE_PAGE
} MI_PFN_USAGES;
@@ -358,6 +399,7 @@ typedef struct _MMPFN
#if MI_TRACE_PFNS
MI_PFN_USAGES PfnUsage;
CHAR ProcessName[16];
+#define MI_SET_PFN_PROCESS_NAME(pfn, x) memcpy(pfn->ProcessName, x, min(sizeof(x),
sizeof(pfn->ProcessName)))
#endif
// HACK until WS lists are supported
diff --git a/ntoskrnl/mm/ARM3/miarm.h b/ntoskrnl/mm/ARM3/miarm.h
index 6df6b7e5385..a30e9957874 100644
--- a/ntoskrnl/mm/ARM3/miarm.h
+++ b/ntoskrnl/mm/ARM3/miarm.h
@@ -1058,6 +1058,10 @@ VOID
NTAPI
MiInsertInWorkingSetList(_Inout_ PMMSUPPORT Vm, _In_ PVOID Address, _In_ ULONG
Protection);
+VOID
+NTAPI
+MiRemoveFromWorkingSetList(_Inout_ PMMSUPPORT Vm, _In_ PVOID Address);
+
//
// New ARM3<->RosMM PAGE Architecture
//
diff --git a/ntoskrnl/mm/ARM3/mminit.c b/ntoskrnl/mm/ARM3/mminit.c
index 0bae38e3e02..3c724d8ea3e 100644
--- a/ntoskrnl/mm/ARM3/mminit.c
+++ b/ntoskrnl/mm/ARM3/mminit.c
@@ -801,7 +801,7 @@ MiBuildPfnDatabaseFromPages(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
Pfn1->u3.e1.CacheAttribute = MiNonCached;
#if MI_TRACE_PFNS
Pfn1->PfnUsage = MI_USAGE_INIT_MEMORY;
- memcpy(Pfn1->ProcessName, "Initial PDE", 16);
+ MI_SET_PFN_PROCESS_NAME(Pfn1, "Initial PDE");
#endif
}
else
@@ -848,7 +848,7 @@ MiBuildPfnDatabaseFromPages(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
Pfn2->u3.e1.CacheAttribute = MiNonCached;
#if MI_TRACE_PFNS
Pfn2->PfnUsage = MI_USAGE_INIT_MEMORY;
- memcpy(Pfn1->ProcessName, "Initial PTE",
16);
+ MI_SET_PFN_PROCESS_NAME(Pfn2, "Initial PTE");
#endif
}
}
diff --git a/ntoskrnl/mm/ARM3/pagfault.c b/ntoskrnl/mm/ARM3/pagfault.c
index 1ab109e9f65..a487d3a1ba2 100644
--- a/ntoskrnl/mm/ARM3/pagfault.c
+++ b/ntoskrnl/mm/ARM3/pagfault.c
@@ -697,16 +697,6 @@ MiResolveDemandZeroFault(IN PVOID Address,
/* Increment demand zero faults */
KeGetCurrentPrcb()->MmDemandZeroCount++;
- /* Do we have the lock? */
- if (HaveLock)
- {
- /* Release it */
- MiReleasePfnLock(OldIrql);
-
- /* Update performance counters */
- if (Process > HYDRA_PROCESS) Process->NumberOfPrivatePages++;
- }
-
/* Zero the page if need be */
if (NeedZero) MiZeroPfn(PageFrameNumber);
@@ -745,6 +735,23 @@ MiResolveDemandZeroFault(IN PVOID Address,
ASSERT(Pfn1->u3.e1.PrototypePte == 0);
}
+ /* Add the page to our working set, if it's not a proto PTE */
+ if ((Process > HYDRA_PROCESS) && (PointerPte == MiAddressToPte(Address)))
+ {
+ /* FIXME: Also support session VM scenario */
+ MiInsertInWorkingSetList(&Process->Vm, Address, Protection);
+ }
+
+ /* Do we have the lock? */
+ if (HaveLock)
+ {
+ /* Release it */
+ MiReleasePfnLock(OldIrql);
+
+ /* Update performance counters */
+ if (Process > HYDRA_PROCESS) Process->NumberOfPrivatePages++;
+ }
+
//
// It's all good now
//
@@ -899,6 +906,9 @@ MiResolvePageFileFault(_In_ BOOLEAN StoreInstruction,
ASSERT(CurrentProcess > HYDRA_PROCESS);
ASSERT(*OldIrql != MM_NOIRQL);
+ MI_SET_USAGE(MI_USAGE_PAGE_FILE);
+ MI_SET_PROCESS(CurrentProcess);
+
/* We must hold the PFN lock */
MI_ASSERT_PFN_LOCK_HELD();
@@ -959,6 +969,9 @@ MiResolvePageFileFault(_In_ BOOLEAN StoreInstruction,
KeSetEvent(Pfn1->u1.Event, IO_NO_INCREMENT, FALSE);
}
+ /* And we can insert this into the working set */
+ MiInsertInWorkingSetList(&CurrentProcess->Vm, FaultingAddress, Protection);
+
return Status;
}
@@ -976,6 +989,8 @@ MiResolveTransitionFault(IN BOOLEAN StoreInstruction,
PMMPFN Pfn1;
MMPTE TempPte;
PMMPTE PointerToPteForProtoPage;
+ ULONG Protection;
+
DPRINT("Transition fault on 0x%p with PTE 0x%p in process %s\n",
FaultingAddress, PointerPte, CurrentProcess->ImageFileName);
@@ -1069,8 +1084,9 @@ MiResolveTransitionFault(IN BOOLEAN StoreInstruction,
ASSERT(PointerPte->u.Hard.Valid == 0);
ASSERT(PointerPte->u.Trans.Prototype == 0);
ASSERT(PointerPte->u.Trans.Transition == 1);
+ Protection = TempPte.u.Trans.Protection;
TempPte.u.Long = (PointerPte->u.Long & ~0xFFF) |
- (MmProtectToPteMask[PointerPte->u.Trans.Protection]) |
+ (MmProtectToPteMask[Protection]) |
MiDetermineUserGlobalPteMask(PointerPte);
/* Is the PTE writeable? */
@@ -1090,6 +1106,10 @@ MiResolveTransitionFault(IN BOOLEAN StoreInstruction,
/* Write the valid PTE */
MI_WRITE_VALID_PTE(PointerPte, TempPte);
+ /* If this was a user fault, add it to the working set */
+ if (CurrentProcess > HYDRA_PROCESS)
+ MiInsertInWorkingSetList(&CurrentProcess->Vm, FaultingAddress,
Protection);
+
/* Return success */
return STATUS_PAGE_FAULT_TRANSITION;
}
@@ -1210,6 +1230,9 @@ MiResolveProtoPteFault(IN BOOLEAN StoreInstruction,
ASSERT(TempPte.u.Hard.Valid == 1);
ProtoPageFrameIndex = PFN_FROM_PTE(&TempPte);
+ MI_SET_USAGE(MI_USAGE_COW);
+ MI_SET_PROCESS(Process);
+
/* Get a new page for the private copy */
if (Process > HYDRA_PROCESS)
Color = MI_GET_NEXT_PROCESS_COLOR(Process);
@@ -1245,6 +1268,13 @@ MiResolveProtoPteFault(IN BOOLEAN StoreInstruction,
/* And finally, write the valid PTE */
MI_WRITE_VALID_PTE(PointerPte, PteContents);
+ /* Add the page to our working set */
+ if (Process > HYDRA_PROCESS)
+ {
+ /* FIXME: Also support session VM scenario */
+ MiInsertInWorkingSetList(&Process->Vm, Address, Protection);
+ }
+
/* The caller expects us to release the PFN lock */
MiReleasePfnLock(OldIrql);
return Status;
@@ -2205,11 +2235,15 @@ UserFault:
{
PFN_NUMBER PageFrameIndex, OldPageFrameIndex;
PMMPFN Pfn1;
+ ProtectionCode = TempPte.u.Soft.Protection;
LockIrql = MiAcquirePfnLock();
ASSERT(MmAvailablePages > 0);
+ MI_SET_USAGE(MI_USAGE_COW);
+ MI_SET_PROCESS(CurrentProcess);
+
/* Allocate a new page and copy it */
PageFrameIndex =
MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(CurrentProcess));
OldPageFrameIndex = PFN_FROM_PTE(&TempPte);
@@ -2231,6 +2265,9 @@ UserFault:
MI_WRITE_VALID_PTE(PointerPte, TempPte);
+ /* We can now add it to our working set */
+ MiInsertInWorkingSetList(&CurrentProcess->Vm, Address,
ProtectionCode);
+
MiReleasePfnLock(LockIrql);
/* Return the status */
@@ -2347,6 +2384,7 @@ UserFault:
TempPte.u.Soft.Protection = ProtectionCode;
MI_WRITE_INVALID_PTE(PointerPte, TempPte);
}
+ ProtectionCode = PointerPte->u.Soft.Protection;
/* Lock the PFN database since we're going to grab a page */
OldIrql = MiAcquirePfnLock();
@@ -2384,9 +2422,6 @@ UserFault:
/* One more demand-zero fault */
KeGetCurrentPrcb()->MmDemandZeroCount++;
- /* And we're done with the lock */
- MiReleasePfnLock(OldIrql);
-
/* Fault on user PDE, or fault on user PTE? */
if (PointerPte <= MiHighestUserPte)
{
@@ -2413,6 +2448,12 @@ UserFault:
Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
ASSERT(Pfn1->u1.Event == NULL);
+ /* We can now insert it into the working set */
+ MiInsertInWorkingSetList(&CurrentProcess->Vm, Address,
ProtectionCode);
+
+ /* And we're done with the lock */
+ MiReleasePfnLock(OldIrql);
+
/* Demand zero */
ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
diff --git a/ntoskrnl/mm/ARM3/pfnlist.c b/ntoskrnl/mm/ARM3/pfnlist.c
index b039a5181a2..866a29cd15f 100644
--- a/ntoskrnl/mm/ARM3/pfnlist.c
+++ b/ntoskrnl/mm/ARM3/pfnlist.c
@@ -254,8 +254,8 @@ MiUnlinkFreeOrZeroedPage(IN PMMPFN Entry)
ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET);
Entry->PfnUsage = MI_PFN_CURRENT_USAGE;
memcpy(Entry->ProcessName, MI_PFN_CURRENT_PROCESS_NAME, 16);
-// MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
-// memcpy(MI_PFN_CURRENT_PROCESS_NAME, "Not Set", 16);
+ MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
+ MI_SET_PROCESS2("Not Set");
#endif
}
@@ -459,11 +459,11 @@ MiRemovePageByColor(IN PFN_NUMBER PageIndex,
MiDecrementAvailablePages();
#if MI_TRACE_PFNS
- //ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET);
+ ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET);
Pfn1->PfnUsage = MI_PFN_CURRENT_USAGE;
memcpy(Pfn1->ProcessName, MI_PFN_CURRENT_PROCESS_NAME, 16);
- //MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
- //memcpy(MI_PFN_CURRENT_PROCESS_NAME, "Not Set", 16);
+ MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
+ MI_SET_PROCESS2("Not Set");
#endif
/* Return the page */
@@ -937,9 +937,8 @@ MiInsertPageInList(IN PMMPFNLIST ListHead,
ColorHead->Count++;
#if MI_TRACE_PFNS
- //ASSERT(MI_PFN_CURRENT_USAGE == MI_USAGE_NOT_SET);
+ ASSERT(MI_PFN_CURRENT_USAGE == MI_USAGE_NOT_SET);
Pfn1->PfnUsage = MI_USAGE_FREE_PAGE;
- MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
RtlZeroMemory(Pfn1->ProcessName, 16);
#endif
}
diff --git a/ntoskrnl/mm/ARM3/procsup.c b/ntoskrnl/mm/ARM3/procsup.c
index ab01d902686..2e4d9029d36 100644
--- a/ntoskrnl/mm/ARM3/procsup.c
+++ b/ntoskrnl/mm/ARM3/procsup.c
@@ -6,6 +6,8 @@
* PROGRAMMERS: ReactOS Portable Systems Group
*/
+#define GROW_WSLE 1
+
/* INCLUDES *******************************************************************/
#include <ntoskrnl.h>
@@ -862,18 +864,34 @@ MiGetFirstFreeWsleIndex(_Inout_ PMMSUPPORT Vm)
if (WsList->LastEntry == WsList->LastInitializedWsle)
{
/* We must grow our array. Allocate a new page */
- PMMPTE PointerPte =
MiAddressToPte(&WsList->Wsle[WsList->LastInitializedWsle + 1]);
+ PVOID Address = &WsList->Wsle[WsList->LastInitializedWsle + 1];
+ PMMPTE PointerPte = MiAddressToPte(Address);
MMPTE TempPte;
+
+ MI_SET_USAGE(MI_USAGE_WSLE);
+ MI_SET_PROCESS(PsGetCurrentProcess());
+
+ /* We must be at page boundary */
+ ASSERT(Address == ALIGN_DOWN_POINTER_BY(Address, PAGE_SIZE));
+
PFN_NUMBER PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
MiInitializePfn(PageFrameIndex, PointerPte, TRUE);
- MI_MAKE_HARDWARE_PTE_KERNEL(&TempPte, PointerPte, MM_READWRITE,
PageFrameIndex);
+
+ TempPte = ValidKernelPteLocal;
+ TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
MI_WRITE_VALID_PTE(PointerPte, TempPte);
WsList->LastInitializedWsle += PAGE_SIZE / sizeof(MMWSLE);
+ /* Make sure we are staying on the same page */
+ ASSERT(Address ==
ALIGN_DOWN_POINTER_BY(&WsList->Wsle[WsList->LastInitializedWsle], PAGE_SIZE));
+
/* We must insert this page in our working set ! */
MiInsertInWorkingSetList(Vm,
&WsList->Wsle[WsList->LastInitializedWsle], MM_READWRITE);
+
+ /* Now the last entry is the tail of our WSLE array */
+ ASSERT(WsList->Wsle[WsList->LastEntry].u1.e1.VirtualPageNumber ==
((ULONG_PTR)&WsList->Wsle[WsList->LastInitializedWsle]) >> PAGE_SHIFT);
}
/* At this point we must be good to go */
@@ -909,18 +927,26 @@ MiGetFirstFreeWsleIndex(_Inout_ PMMSUPPORT Vm)
VOID
NTAPI
-MiInsertInWorkingSetList(_Inout_ PMMSUPPORT Vm, _In_ PVOID Address, _In_ ULONG
Protection)
+MiInsertInWorkingSetList(
+ _Inout_ PMMSUPPORT Vm,
+ _In_ PVOID Address,
+ _In_ ULONG Protection)
{
ULONG WsIndex = MiGetFirstFreeWsleIndex(Vm);
PMMWSLE WsleEntry = &Vm->VmWorkingSetList->Wsle[WsIndex];
PMMPTE PointerPte = MiAddressToPte(Address);
- PMMPFN Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte));
+ PMMPFN Pfn1;
/* Make sure we got a rounded address */
Address = ALIGN_DOWN_POINTER_BY(Address, PAGE_SIZE);
- /* Make sure we are locking the right thing */
+ /* Make sure we are locking the right things */
ASSERT(MM_ANY_WS_LOCK_HELD(PsGetCurrentThread()));
+ MI_ASSERT_PFN_LOCK_HELD();
+
+ /* Make sure we are adding a paged-in address */
+ ASSERT(PointerPte->u.Hard.Valid == 1);
+ Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte));
/* The Pfn must be an active one */
ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
@@ -931,6 +957,11 @@ MiInsertInWorkingSetList(_Inout_ PMMSUPPORT Vm, _In_ PVOID Address,
_In_ ULONG P
/* Shared pages not supported yet */
ASSERT(Pfn1->u1.WsIndex == 0);
+ ASSERT(Pfn1->u3.e1.PrototypePte == 0);
+
+ /* Nor are "ROS PFN" */
+ ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE);
+
WsleEntry->u1.e1.Direct = 1;
Pfn1->u1.WsIndex = WsIndex;
@@ -941,6 +972,177 @@ MiInsertInWorkingSetList(_Inout_ PMMSUPPORT Vm, _In_ PVOID Address,
_In_ ULONG P
Vm->PeakWorkingSetSize = Vm->WorkingSetSize;
}
+static
+void
+MiShrinkWorkingSet(_Inout_ PMMSUPPORT Vm)
+{
+ PMMWSL WsList = Vm->VmWorkingSetList;
+ ULONG LastValid = WsList->LastEntry;
+
+ while(WsList->Wsle[LastValid].u1.e1.Valid == 0)
+ {
+ LastValid--;
+ }
+
+ if (LastValid != WsList->LastEntry)
+ {
+ /* There was a hole behind us. Handle this */
+ PMMWSLE NextFree = &WsList->Wsle[LastValid + 1];
+ if (NextFree->u1.Free.PreviousFree == MMWSLE_PREVIOUS_FREE_INVALID)
+ {
+ /* This was actually our first free entry. */
+ ASSERT(WsList->FirstFree == LastValid + 1);
+ WsList->FirstFree = MMWSLE_NEXT_FREE_INVALID;
+ }
+ else
+ {
+ /* The previous one is now the last in the queue */
+ PMMWSLE PreviousFree =
&WsList->Wsle[NextFree->u1.Free.PreviousFree];
+
+ ASSERT(PreviousFree->u1.Free.MustBeZero == 0);
+ PreviousFree->u1.Free.NextFree = MMWSLE_NEXT_FREE_INVALID;
+ }
+
+ /* Nuke everyone */
+ RtlZeroMemory(&WsList->Wsle[LastValid + 1], (WsList->LastEntry -
LastValid) * sizeof(MMWSLE));
+ WsList->LastEntry = LastValid;
+ }
+
+ if (LastValid < WsList->FirstDynamic)
+ {
+ /* Do not mess around with the protected ones */
+ return;
+ }
+
+ /* See if we should shrink our array */
+ if (LastValid == (WsList->LastInitializedWsle - (PAGE_SIZE / sizeof(MMWSLE)) +
1))
+ {
+ PVOID WsleArrayQueue = ALIGN_DOWN_POINTER_BY(&WsList->Wsle[LastValid],
PAGE_SIZE);
+ PEPROCESS Process = MmGetAddressSpaceOwner(Vm);
+
+ ASSERT(WsList->Wsle[WsList->LastEntry].u1.e1.VirtualPageNumber ==
((ULONG_PTR)WsleArrayQueue) >> PAGE_SHIFT);
+
+ /* Kernel address space not supported yet */
+ ASSERT(Process != NULL);
+
+ /* Nuke the PTE. This will remove the virtual address from the working set */
+ MiDeletePte(MiAddressToPte(WsleArrayQueue), WsleArrayQueue, Process, NULL);
+ }
+}
+
+VOID
+NTAPI
+MiRemoveFromWorkingSetList(
+ _Inout_ PMMSUPPORT Vm,
+ _In_ PVOID Address)
+{
+ PMMWSL WsList = Vm->VmWorkingSetList;
+ ULONG WsIndex;
+ PMMWSLE WsleEntry;
+ PMMPTE PointerPte = MiAddressToPte(Address);
+ PMMPFN Pfn1;
+
+ /* Make sure we got a rounded address */
+ Address = ALIGN_DOWN_POINTER_BY(Address, PAGE_SIZE);
+
+ /* Make sure we are locking the right things */
+ ASSERT(MM_ANY_WS_LOCK_HELD(PsGetCurrentThread()));
+ MI_ASSERT_PFN_LOCK_HELD();
+
+ /* Make sure we are removing a paged-in address */
+ ASSERT(PointerPte->u.Hard.Valid == 1);
+ Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte));
+
+ /* The Pfn must be an active one */
+ ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
+
+ WsIndex = Pfn1->u1.WsIndex;
+ WsleEntry = &Vm->VmWorkingSetList->Wsle[WsIndex];
+
+ /* Shared page not handled yet */
+ ASSERT(Pfn1->u3.e1.PrototypePte == 0);
+ ASSERT(WsleEntry->u1.e1.Direct == 1);
+ /* Nor are "ROS PFN" */
+ ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE);
+
+ /* Some sanity checks */
+ ASSERT(WsIndex >= WsList->FirstDynamic);
+ ASSERT(WsIndex <= WsList->LastEntry);
+ ASSERT(WsIndex <= WsList->LastInitializedWsle);
+ ASSERT(WsleEntry->u1.e1.Valid == 1);
+ ASSERT(WsleEntry->u1.e1.VirtualPageNumber == ((ULONG_PTR)Address) >>
PAGE_SHIFT);
+
+ /* Let this go */
+ Pfn1->u1.WsIndex = 0;
+
+ /* Nuke it */
+ WsleEntry->u1.Long = 0;
+
+ /* Insert our entry into the free list */
+ if (WsIndex == WsList->LastEntry)
+ {
+ /* Let's shrink the active list */
+ WsList->LastEntry--;
+ MiShrinkWorkingSet(Vm);
+ }
+ else if (WsList->FirstFree > WsList->LastEntry)
+ {
+ /* We are the first free entry to be inserted */
+ WsList->FirstFree = WsIndex;
+ WsleEntry->u1.Free.PreviousFree = MMWSLE_PREVIOUS_FREE_INVALID;
+ WsleEntry->u1.Free.NextFree = MMWSLE_NEXT_FREE_INVALID;
+ }
+ else
+ {
+ /* Keep this sorted */
+ PMMWSLE NextFree = &WsList->Wsle[WsList->FirstFree];
+ PMMWSLE PreviousFree = NULL;
+
+ ASSERT(NextFree->u1.Free.MustBeZero == 0);
+
+ while (NextFree < WsleEntry)
+ {
+ PreviousFree = NextFree;
+ if (NextFree->u1.Free.NextFree != MMWSLE_NEXT_FREE_INVALID)
+ {
+ NextFree = &WsList->Wsle[NextFree->u1.Free.NextFree];
+ ASSERT(NextFree->u1.Free.MustBeZero == 0);
+ }
+ else
+ {
+ NextFree = NULL;
+ break;
+ }
+ }
+
+ ASSERT(PreviousFree || NextFree);
+
+ if (PreviousFree)
+ {
+ ASSERT((NextFree != NULL) || (PreviousFree->u1.Free.NextFree ==
MMWSLE_NEXT_FREE_INVALID));
+ PreviousFree->u1.Free.NextFree = WsIndex;
+ WsleEntry->u1.Free.PreviousFree = PreviousFree - WsList->Wsle;
+ }
+ else
+ {
+ WsleEntry->u1.Free.PreviousFree = MMWSLE_PREVIOUS_FREE_INVALID;
+ ASSERT(NextFree->u1.Free.PreviousFree == MMWSLE_PREVIOUS_FREE_INVALID);
+ WsList->FirstFree = WsIndex;
+ }
+
+ if (NextFree)
+ {
+ NextFree->u1.Free.PreviousFree = WsIndex;
+ WsleEntry->u1.Free.NextFree = NextFree - WsList->Wsle;
+ }
+ else
+ {
+ WsleEntry->u1.Free.NextFree = MMWSLE_NEXT_FREE_INVALID;
+ }
+ }
+
+ Vm->WorkingSetSize -= PAGE_SIZE;
+}
VOID
NTAPI
diff --git a/ntoskrnl/mm/ARM3/sysldr.c b/ntoskrnl/mm/ARM3/sysldr.c
index d414b1f6b43..58e4c3057b2 100644
--- a/ntoskrnl/mm/ARM3/sysldr.c
+++ b/ntoskrnl/mm/ARM3/sysldr.c
@@ -188,14 +188,7 @@ MiLoadImageSection(IN OUT PVOID *SectionPtr,
/* Some debug stuff */
MI_SET_USAGE(MI_USAGE_DRIVER_PAGE);
#if MI_TRACE_PFNS
- if (FileName->Buffer)
- {
- PWCHAR pos = NULL;
- ULONG len = 0;
- pos = wcsrchr(FileName->Buffer, '\\');
- len = wcslen(pos) * sizeof(WCHAR);
- if (pos) snprintf(MI_PFN_CURRENT_PROCESS_NAME, min(16, len), "%S",
pos);
- }
+ MI_SET_PROCESS_USTR(FileName);
#endif
/* Grab a page */
diff --git a/ntoskrnl/mm/ARM3/virtual.c b/ntoskrnl/mm/ARM3/virtual.c
index f0ab5d08f7b..630afc5c990 100644
--- a/ntoskrnl/mm/ARM3/virtual.c
+++ b/ntoskrnl/mm/ARM3/virtual.c
@@ -503,6 +503,9 @@ MiDeletePte(IN PMMPTE PointerPte,
}
else
{
+ /* Remove this address from the WS list */
+ MiRemoveFromWorkingSetList(&CurrentProcess->Vm, VirtualAddress);
+
/* Make sure the saved PTE address is valid */
if ((PMMPTE)((ULONG_PTR)Pfn1->PteAddress & ~0x1) != PointerPte)
{
@@ -2303,13 +2306,15 @@ MiProtectVirtualMemory(IN PEPROCESS Process,
{
KIRQL OldIrql = MiAcquirePfnLock();
+ /* Remove this from the working set */
+ MiRemoveFromWorkingSetList(AddressSpace,
MiPteToAddress(PointerPte));
+
/* Mark the PTE as transition and change its protection */
PteContents.u.Hard.Valid = 0;
PteContents.u.Soft.Transition = 1;
PteContents.u.Trans.Protection = ProtectionMask;
/* Decrease PFN share count and write the PTE */
MiDecrementShareCount(Pfn1, PFN_FROM_PTE(&PteContents));
- // FIXME: remove the page from the WS
MI_WRITE_INVALID_PTE(PointerPte, PteContents);
#ifdef CONFIG_SMP
// FIXME: Should invalidate entry in every CPU TLB
@@ -2447,8 +2452,10 @@ MiMakePdeExistAndMakeValid(IN PMMPDE PointerPde,
VOID
NTAPI
-MiProcessValidPteList(IN PMMPTE *ValidPteList,
- IN ULONG Count)
+MiProcessValidPteList(
+ _Inout_ PMMSUPPORT Vm,
+ _Inout_ PMMPTE *ValidPteList,
+ _In_ ULONG Count)
{
KIRQL OldIrql;
ULONG i;
@@ -2468,6 +2475,11 @@ MiProcessValidPteList(IN PMMPTE *ValidPteList,
TempPte = *ValidPteList[i];
ASSERT(TempPte.u.Hard.Valid == 1);
+ //
+ // We can now remove this addres from the working set
+ //
+ MiRemoveFromWorkingSetList(Vm, MiPteToAddress(ValidPteList[i]));
+
//
// Get the PFN entry for the page itself, and then for its page table
//
@@ -2509,7 +2521,6 @@ MiDecommitPages(IN PVOID StartingAddress,
ULONG CommitReduction = 0;
PMMPTE ValidPteList[256];
ULONG PteCount = 0;
- PMMPFN Pfn1;
MMPTE PteContents;
PETHREAD CurrentThread = PsGetCurrentThread();
@@ -2541,10 +2552,10 @@ MiDecommitPages(IN PVOID StartingAddress,
// such, and does not flush the entire TLB all the time, but right
// now we have bigger problems to worry about than TLB flushing.
//
- PointerPde = MiAddressToPde(StartingAddress);
+ PointerPde = MiPteToPde(PointerPte);
if (PteCount)
{
- MiProcessValidPteList(ValidPteList, PteCount);
+ MiProcessValidPteList(&Process->Vm, ValidPteList, PteCount);
PteCount = 0;
}
@@ -2579,21 +2590,13 @@ MiDecommitPages(IN PVOID StartingAddress,
//Process->NumberOfPrivatePages--;
if (PteContents.u.Hard.Valid)
{
- //
- // It's valid. At this point make sure that it is not a ROS
- // PFN. Also, we don't support ProtoPTEs in this code path.
- //
- Pfn1 = MiGetPfnEntry(PteContents.u.Hard.PageFrameNumber);
- ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE);
- ASSERT(Pfn1->u3.e1.PrototypePte == FALSE);
-
//
// Flush any pending PTEs that we had not yet flushed, if our
// list has gotten too big, then add this PTE to the flush list.
//
if (PteCount == 256)
{
- MiProcessValidPteList(ValidPteList, PteCount);
+ MiProcessValidPteList(&Process->Vm, ValidPteList,
PteCount);
PteCount = 0;
}
ValidPteList[PteCount++] = PointerPte;
@@ -2623,7 +2626,7 @@ MiDecommitPages(IN PVOID StartingAddress,
// This used to be a zero PTE and it no longer is, so we must add a
// reference to the pagetable.
//
- MiIncrementPageTableReferences(StartingAddress);
+ MiIncrementPageTableReferences(MiPteToAddress(PointerPte));
//
// Next, we account for decommitted PTEs and make the PTE as such
@@ -2633,17 +2636,16 @@ MiDecommitPages(IN PVOID StartingAddress,
}
//
- // Move to the next PTE and the next address
+ // Move to the next PTE
//
PointerPte++;
- StartingAddress = (PVOID)((ULONG_PTR)StartingAddress + PAGE_SIZE);
}
//
// Flush any dangling PTEs from the loop in the last page table, and then
// release the working set and return the commit reduction accounting.
//
- if (PteCount) MiProcessValidPteList(ValidPteList, PteCount);
+ if (PteCount) MiProcessValidPteList(&Process->Vm, ValidPteList, PteCount);
MiUnlockProcessWorkingSetUnsafe(Process, CurrentThread);
return CommitReduction;
}
diff --git a/ntoskrnl/mm/freelist.c b/ntoskrnl/mm/freelist.c
index 93d9bbd9f59..80447593ecd 100644
--- a/ntoskrnl/mm/freelist.c
+++ b/ntoskrnl/mm/freelist.c
@@ -574,6 +574,21 @@ MmAllocPage(ULONG Type)
OldIrql = MiAcquirePfnLock();
+#if MI_TRACE_PFNS
+ switch(Type)
+ {
+ case MC_CACHE:
+ case MC_SYSTEM:
+ MI_SET_USAGE(MI_USAGE_CACHE);
+ break;
+ case MC_USER:
+ MI_SET_USAGE(MI_USAGE_SECTION);
+ break;
+ default:
+ ASSERT(FALSE);
+ }
+#endif
+
PfnOffset = MiRemoveZeroPage(MI_GET_NEXT_COLOR());
if (!PfnOffset)
{
diff --git a/ntoskrnl/mm/i386/page.c b/ntoskrnl/mm/i386/page.c
index 6b5ac1e8353..f8337dadc26 100644
--- a/ntoskrnl/mm/i386/page.c
+++ b/ntoskrnl/mm/i386/page.c
@@ -264,6 +264,10 @@ MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN
Create)
MI_WRITE_INVALID_PTE(PointerPde, DemandZeroPde);
// Tiny HACK: Parameter 1 is the architecture specific FaultCode for an
access violation (i.e. page is present)
+
+ /* Lock the working set, as this will add this address to it */
+ MiLockProcessWorkingSetUnsafe(Process, PsGetCurrentThread());
+
Status = MiDispatchFault(0x1,
Pt,
PointerPde,
@@ -275,6 +279,8 @@ MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN
Create)
DBG_UNREFERENCED_LOCAL_VARIABLE(Status);
ASSERT(KeAreAllApcsDisabled() == TRUE);
ASSERT(PointerPde->u.Hard.Valid == 1);
+
+ MiUnlockProcessWorkingSetUnsafe(Process, PsGetCurrentThread());
}
return (PULONG)MiAddressToPte(Address);
}
diff --git a/ntoskrnl/mm/marea.c b/ntoskrnl/mm/marea.c
index d953f422f96..a68adecd9be 100644
--- a/ntoskrnl/mm/marea.c
+++ b/ntoskrnl/mm/marea.c
@@ -331,13 +331,16 @@ MmFreeMemoryArea(
ASSERT(AddressSpace != MmGetKernelAddressSpace());
if (MiQueryPageTableReferences((PVOID)Address) == 0)
{
+ KIRQL OldIrql;
/* No PTE relies on this PDE. Release it */
- KIRQL OldIrql = MiAcquirePfnLock();
+ MiLockProcessWorkingSet(Process, PsGetCurrentThread());
+ OldIrql = MiAcquirePfnLock();
PMMPDE PointerPde = MiAddressToPde(Address);
ASSERT(PointerPde->u.Hard.Valid == 1);
MiDeletePte(PointerPde, MiPdeToPte(PointerPde), Process, NULL);
ASSERT(PointerPde->u.Hard.Valid == 0);
MiReleasePfnLock(OldIrql);
+ MiUnlockProcessWorkingSet(Process, PsGetCurrentThread());
}
}
#endif
diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c
index ce63e796a3d..0825d4ecb49 100644
--- a/ntoskrnl/mm/section.c
+++ b/ntoskrnl/mm/section.c
@@ -1163,8 +1163,6 @@ MiReadPage(PMEMORY_AREA MemoryArea,
* Allocate a page, this is rather complicated by the possibility
* we might have to move other things out of memory
*/
- MI_SET_USAGE(MI_USAGE_SECTION);
- MI_SET_PROCESS2(PsGetCurrentProcess()->ImageFileName);
Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, Page);
if (!NT_SUCCESS(Status))
{
@@ -1624,9 +1622,6 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace,
((Offset.QuadPart >=
(LONGLONG)PAGE_ROUND_UP(Segment->RawLength.QuadPart) &&
(Section->AllocationAttributes & SEC_IMAGE))))
{
- MI_SET_USAGE(MI_USAGE_SECTION);
- if (Process) MI_SET_PROCESS2(Process->ImageFileName);
- if (!Process) MI_SET_PROCESS2("Kernel Section");
Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &Page);
if (!NT_SUCCESS(Status))
{
@@ -1707,9 +1702,6 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace,
MmUnlockSectionSegment(Segment);
MmUnlockAddressSpace(AddressSpace);
- MI_SET_USAGE(MI_USAGE_SECTION);
- if (Process) MI_SET_PROCESS2(Process->ImageFileName);
- if (!Process) MI_SET_PROCESS2("Kernel Section");
Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &Page);
if (!NT_SUCCESS(Status))
{
@@ -1878,9 +1870,6 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace,
/*
* Allocate a page
*/
- MI_SET_USAGE(MI_USAGE_SECTION);
- if (Process) MI_SET_PROCESS2(Process->ImageFileName);
- if (!Process) MI_SET_PROCESS2("Kernel Section");
Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &NewPage);
if (!NT_SUCCESS(Status))
{