29 Sep
2010
29 Sep
'10
1:10 a.m.
Author: sir_richard
Date: Wed Sep 29 01:10:28 2010
New Revision: 48927
URL: http://svn.reactos.org/svn/reactos?rev=48927&view=rev
Log:
[NTOS]: Add MiRemoveZeroPageSafe helper function, when a zero page is required, but the
inline zeroing of MiRemoveZeroPage is not. This function will only try grabbing a zero
page if one exists, otherwise a free page will be grabbed and zeroed with custom code of
the caller's choosing.
[NTOS]: Add concept of process color and system color. Compute correct color to use
whenever requesting a page.
[NTOS]: Uncondtionally enable the color code when inserting/removing pages.
For now, when requesting a page, colors are still ignored, and the global PFN lists are
scanned instead. If there are no regressions, we are one patch away from that.
Modified:
trunk/reactos/ntoskrnl/mm/ARM3/miarm.h
trunk/reactos/ntoskrnl/mm/ARM3/pagfault.c
trunk/reactos/ntoskrnl/mm/ARM3/pfnlist.c
trunk/reactos/ntoskrnl/mm/ARM3/pool.c
trunk/reactos/ntoskrnl/mm/ARM3/procsup.c
trunk/reactos/ntoskrnl/mm/ARM3/zeropage.c
trunk/reactos/ntoskrnl/mm/freelist.c
Modified: trunk/reactos/ntoskrnl/mm/ARM3/miarm.h
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/ARM3/miarm.h?r…
==============================================================================
--- trunk/reactos/ntoskrnl/mm/ARM3/miarm.h [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/mm/ARM3/miarm.h [iso-8859-1] Wed Sep 29 01:10:28 2010
@@ -217,6 +217,13 @@
// Special IRQL value (found in assertions)
//
#define MM_NOIRQL (KIRQL)0xFFFFFFFF
+
+//
+// Returns the color of a page
+//
+#define MI_GET_PAGE_COLOR(x) ((x) & MmSecondaryColorMask)
+#define MI_GET_NEXT_COLOR(x) (MI_GET_PAGE_COLOR(++MmSystemPageColor))
+#define MI_GET_NEXT_PROCESS_COLOR(x) (MI_GET_PAGE_COLOR(++(x)->NextPageColor))
//
// FIXFIX: These should go in ex.h after the pool merge
@@ -455,9 +462,8 @@
extern LIST_ENTRY MmProcessList;
extern BOOLEAN MmZeroingPageThreadActive;
extern KEVENT MmZeroingPageEvent;
-
-#define MI_PFN_TO_PFNENTRY(x) (&MmPfnDatabase[1][x])
-#define MI_PFNENTRY_TO_PFN(x) (x - MmPfnDatabase[1])
+extern ULONG MmSystemPageColor;
+extern ULONG MmProcessColorSeed;
//
// Figures out the hardware bits for a PTE
@@ -1093,4 +1099,19 @@
IN PMMADDRESS_NODE Node
);
+//
+// MiRemoveZeroPage will use inline code to zero out the page manually if only
+// free pages are available. In some scenarios, we don't/can't run that piece of
+// code and would rather only have a real zero page. If we can't have a zero page,
+// then we'd like to have our own code to grab a free page and zero it out, by
+// using MiRemoveAnyPage. This macro implements this.
+//
+PFN_NUMBER
+FORCEINLINE
+MiRemoveZeroPageSafe(IN ULONG Color)
+{
+ if (MmFreePagesByColor[ZeroedPageList][Color].Flink != LIST_HEAD) return
MiRemoveZeroPage(Color);
+ return 0;
+}
+
/* EOF */
Modified: trunk/reactos/ntoskrnl/mm/ARM3/pagfault.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/ARM3/pagfault.…
==============================================================================
--- trunk/reactos/ntoskrnl/mm/ARM3/pagfault.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/mm/ARM3/pagfault.c [iso-8859-1] Wed Sep 29 01:10:28 2010
@@ -179,9 +179,10 @@
IN PEPROCESS Process,
IN KIRQL OldIrql)
{
- PFN_NUMBER PageFrameNumber;
+ PFN_NUMBER PageFrameNumber = 0;
MMPTE TempPte;
BOOLEAN NeedZero = FALSE;
+ ULONG Color;
DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process:
%p\n",
Address,
Process);
@@ -196,9 +197,17 @@
/* No forking yet */
ASSERT(Process->ForkInProgress == NULL);
+ /* Get process color */
+ Color = MI_GET_NEXT_PROCESS_COLOR(Process);
+
/* We'll need a zero page */
NeedZero = TRUE;
}
+ else
+ {
+ /* Get the next system page color */
+ Color = MI_GET_NEXT_COLOR();
+ }
//
// Lock the PFN database
@@ -206,9 +215,21 @@
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
ASSERT(PointerPte->u.Hard.Valid == 0);
- /* Get a page */
- PageFrameNumber = MiRemoveAnyPage(0);
- DPRINT("New pool page: %lx\n", PageFrameNumber);
+ /* Do we need a zero page? */
+ if (NeedZero)
+ {
+ /* Try to get one, if we couldn't grab a free page and zero it */
+ PageFrameNumber = MiRemoveZeroPageSafe(Color);
+ if (PageFrameNumber) NeedZero = FALSE;
+ }
+
+ /* Did we get a page? */
+ if (!PageFrameNumber)
+ {
+ /* We either failed to find a zero page, or this is a system request */
+ PageFrameNumber = MiRemoveAnyPage(Color);
+ DPRINT("New pool page: %lx\n", PageFrameNumber);
+ }
/* Initialize it */
MiInitializePfn(PageFrameNumber, PointerPte, TRUE);
@@ -463,6 +484,7 @@
ULONG ProtectionCode;
PMMVAD Vad;
PFN_NUMBER PageFrameIndex;
+ ULONG Color;
DPRINT("ARM3 FAULT AT: %p\n", Address);
//
@@ -756,19 +778,25 @@
/* Lock the PFN database since we're going to grab a page */
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
-
- /* Grab a page out of there. Later we should grab a colored zero page */
- PageFrameIndex = MiRemoveAnyPage(0);
- ASSERT(PageFrameIndex);
-
- /* Release the lock since we need to do some zeroing */
- KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
-
- /* Zero out the page, since it's for user-mode */
- MiZeroPfn(PageFrameIndex);
-
- /* Grab the lock again so we can initialize the PFN entry */
- OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+
+ /* Try to get a zero page */
+ Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess);
+ PageFrameIndex = MiRemoveZeroPageSafe(Color);
+ if (!PageFrameIndex)
+ {
+ /* Grab a page out of there. Later we should grab a colored zero page */
+ PageFrameIndex = MiRemoveAnyPage(Color);
+ ASSERT(PageFrameIndex);
+
+ /* Release the lock since we need to do some zeroing */
+ KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+
+ /* Zero out the page, since it's for user-mode */
+ MiZeroPfn(PageFrameIndex);
+
+ /* Grab the lock again so we can initialize the PFN entry */
+ OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+ }
/* Initialize the PFN entry now */
MiInitializePfn(PageFrameIndex, PointerPte, 1);
Modified: trunk/reactos/ntoskrnl/mm/ARM3/pfnlist.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/ARM3/pfnlist.c…
==============================================================================
--- trunk/reactos/ntoskrnl/mm/ARM3/pfnlist.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/mm/ARM3/pfnlist.c [iso-8859-1] Wed Sep 29 01:10:28 2010
@@ -30,12 +30,11 @@
#define ASSERT_LIST_INVARIANT(x)
#endif
-#define ARM3_COLORS 1
-
/* GLOBALS ********************************************************************/
BOOLEAN MmDynamicPfn;
BOOLEAN MmMirroring;
+ULONG MmSystemPageColor;
MMPFNLIST MmZeroedPageListHead = {0, ZeroedPageList, LIST_HEAD, LIST_HEAD};
MMPFNLIST MmFreePageListHead = {0, FreePageList, LIST_HEAD, LIST_HEAD};
@@ -80,11 +79,9 @@
PFN_NUMBER OldFlink, OldBlink;
PMMPFNLIST ListHead;
MMLISTS ListName;
-#ifdef ARM3_COLORS
ULONG Color;
PMMCOLOR_TABLES ColorTable;
PMMPFN Pfn1;
-#endif
/* Make sure the PFN lock is held */
ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
@@ -131,7 +128,7 @@
/* Set the list head's backlink instead */
ListHead->Flink = OldFlink;
}
-#ifdef ARM3_COLORS
+
/* Get the page color */
OldBlink = MiGetPfnEntryIndex(Entry);
Color = OldBlink & MmSecondaryColorMask;
@@ -185,7 +182,7 @@
/* ReactOS Hack */
Entry->OriginalPte.u.Long = 0;
-#endif
+
/* We are not on a list anymore */
Entry->u1.Flink = Entry->u2.Blink = 0;
ASSERT_LIST_INVARIANT(ListHead);
@@ -219,9 +216,8 @@
MMLISTS ListName;
PFN_NUMBER OldFlink, OldBlink;
ULONG OldColor, OldCache;
-#ifdef ARM3_COLORS
PMMCOLOR_TABLES ColorTable;
-#endif
+
/* Make sure PFN lock is held */
ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
ASSERT(Color < MmSecondaryColors);
@@ -280,7 +276,7 @@
Pfn1->u3.e2.ShortFlags = 0;
Pfn1->u3.e1.PageColor = OldColor;
Pfn1->u3.e1.CacheAttribute = OldCache;
-#ifdef ARM3_COLORS
+
/* Get the first page on the color list */
ASSERT(Color < MmSecondaryColors);
ColorTable = &MmFreePagesByColor[ListName][Color];
@@ -306,7 +302,7 @@
/* ReactOS Hack */
Pfn1->OriginalPte.u.Long = 0;
-#endif
+
/* See if we hit any thresholds */
if (MmAvailablePages == MmHighMemoryThreshold)
{
@@ -340,9 +336,8 @@
ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
ASSERT(MmAvailablePages != 0);
ASSERT(Color < MmSecondaryColors);
-
+#if 0
/* Check the colored free list */
-#if 0 // Enable when using ARM3 database */
PageIndex = MmFreePagesByColor[FreePageList][Color].Flink;
if (PageIndex == LIST_HEAD)
{
@@ -368,11 +363,10 @@
ASSERT(MmZeroedPageListHead.Total == 0);
}
}
-#if 0 // Enable when using ARM3 database */
+#if 0
}
}
#endif
-
/* Remove the page from its list */
PageIndex = MiRemovePageByColor(PageIndex, Color);
@@ -403,7 +397,7 @@
ASSERT(Color < MmSecondaryColors);
/* Check the colored zero list */
-#if 0 // Enable when using ARM3 database */
+#if 0
PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
if (PageIndex == LIST_HEAD)
{
@@ -414,9 +408,10 @@
Color = PageIndex & MmSecondaryColorMask;
if (PageIndex == LIST_HEAD)
{
+ /* This means there's no zero pages, we have to look for free ones */
ASSERT(MmZeroedPageListHead.Total == 0);
Zero = TRUE;
-#if 0 // Enable when using ARM3 database */
+#if 0
/* Check the colored free list */
PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
if (PageIndex == LIST_HEAD)
@@ -432,13 +427,14 @@
/* FIXME: Should check the standby list */
ASSERT(MmZeroedPageListHead.Total == 0);
}
-#if 0 // Enable when using ARM3 database */
+#if 0
}
#endif
}
-#if 0 // Enable when using ARM3 database */
+#if 0
}
#endif
+
/* Sanity checks */
Pfn1 = MiGetPfnEntry(PageIndex);
ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
@@ -468,11 +464,10 @@
PMMPFNLIST ListHead;
PFN_NUMBER LastPage;
PMMPFN Pfn1;
-#ifdef ARM3_COLORS
ULONG Color;
PMMPFN Blink;
PMMCOLOR_TABLES ColorTable;
-#endif
+
/* Make sure the page index is valid */
ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
ASSERT((PageFrameIndex != 0) &&
@@ -537,7 +532,7 @@
/* Otherwise check if we reached the high threshold and signal the event */
KeSetEvent(MiHighMemoryEvent, 0, FALSE);
}
-#ifdef ARM3_COLORS
+
/* Get the page color */
Color = PageFrameIndex & MmSecondaryColorMask;
@@ -571,7 +566,7 @@
/* And increase the count in the colored list */
ColorTable->Count++;
-#endif
+
/* Notify zero page thread if enough pages are on the free list now */
if ((ListHead->Total >= 8) && !(MmZeroingPageThreadActive))
{
@@ -590,10 +585,9 @@
PFN_NUMBER Flink;
PMMPFN Pfn1, Pfn2;
MMLISTS ListName;
-#ifdef ARM3_COLORS
PMMCOLOR_TABLES ColorHead;
ULONG Color;
-#endif
+
/* For free pages, use MiInsertPageInFreeList */
ASSERT(ListHead != &MmFreePageListHead);
@@ -657,7 +651,7 @@
KeSetEvent(MiHighMemoryEvent, 0, FALSE);
}
-#ifdef ARM3_COLORS
+ /* Sanity checks */
ASSERT(ListName == ZeroedPageList);
ASSERT(Pfn1->u4.InPageError == 0);
@@ -695,7 +689,6 @@
/* One more paged on the colored list */
ColorHead->Count++;
-#endif
}
VOID
@@ -783,7 +776,7 @@
/* Grab a page */
ASSERT_LIST_INVARIANT(&MmFreePageListHead);
ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
- PageFrameIndex = MiRemoveAnyPage(0);
+ PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
/* Write the software PTE */
MI_WRITE_INVALID_PTE(PointerPte, TempPte);
Modified: trunk/reactos/ntoskrnl/mm/ARM3/pool.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/ARM3/pool.c?re…
==============================================================================
--- trunk/reactos/ntoskrnl/mm/ARM3/pool.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/mm/ARM3/pool.c [iso-8859-1] Wed Sep 29 01:10:28 2010
@@ -473,7 +473,7 @@
ASSERT(PointerPte->u.Hard.Valid == 0);
/* Request a page */
- PageFrameNumber = MiRemoveAnyPage(0);
+ PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
#if (_MI_PAGING_LEVELS >= 3)
@@ -768,7 +768,7 @@
do
{
/* Allocate a page */
- PageFrameNumber = MiRemoveAnyPage(0);
+ PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
/* Get the PFN entry for it and fill it out */
Pfn1 = MiGetPfnEntry(PageFrameNumber);
Modified: trunk/reactos/ntoskrnl/mm/ARM3/procsup.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/ARM3/procsup.c…
==============================================================================
--- trunk/reactos/ntoskrnl/mm/ARM3/procsup.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/mm/ARM3/procsup.c [iso-8859-1] Wed Sep 29 01:10:28 2010
@@ -16,7 +16,9 @@
#define MODULE_INVOLVED_IN_ARM3
#include "../ARM3/miarm.h"
-extern MM_SYSTEMSIZE MmSystemSize;
+/* GLOBALS ********************************************************************/
+
+ULONG MmProcessColorSeed = 0x12345678;
/* PRIVATE FUNCTIONS **********************************************************/
@@ -357,7 +359,7 @@
PointerPte++;
/* Get a page and write the current invalid PTE */
- PageFrameIndex = MiRemoveAnyPage(0);
+ PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
MI_WRITE_INVALID_PTE(PointerPte, InvalidPte);
/* Initialize the PFN entry for this page */
@@ -444,7 +446,7 @@
while (LimitPte >= NewLimitPte)
{
/* Get a page and write the current invalid PTE */
- PageFrameIndex = MiRemoveAnyPage(0);
+ PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
MI_WRITE_INVALID_PTE(LimitPte, InvalidPte);
/* Initialize the PFN entry for this page */
@@ -1058,9 +1060,10 @@
MMPTE TempPte, PdePte;
ULONG PdeOffset;
PMMPTE SystemTable;
-
- /* No page colors yet */
- Process->NextPageColor = 0;
+ ULONG Color;
+
+ /* Choose a process color */
+ Process->NextPageColor = RtlRandom(&MmProcessColorSeed);
/* Setup the hyperspace lock */
KeInitializeSpinLock(&Process->HyperSpaceLock);
@@ -1068,16 +1071,37 @@
/* Lock PFN database */
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
- /* Get a page for the PDE */
- PdeIndex = MiRemoveAnyPage(0);
- KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
- MiZeroPhysicalPage(PdeIndex);
- OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
-
- /* Get a page for hyperspace */
- HyperIndex = MiRemoveAnyPage(0);
- KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
- MiZeroPhysicalPage(HyperIndex);
+ /* Get a zero page for the PDE, if possible */
+ Color = MI_GET_NEXT_PROCESS_COLOR(Process);
+ PdeIndex = MiRemoveZeroPageSafe(Color);
+ if (!PdeIndex)
+ {
+ /* No zero pages, grab a free one */
+ PdeIndex = MiRemoveAnyPage(Color);
+
+ /* Zero it outside the PFN lock */
+ KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+ MiZeroPhysicalPage(PdeIndex);
+ OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+ }
+
+ /* Get a zero page for hyperspace, if possible */
+ Color = MI_GET_NEXT_PROCESS_COLOR(Process);
+ HyperIndex = MiRemoveZeroPageSafe(Color);
+ if (!HyperIndex)
+ {
+ /* No zero pages, grab a free one */
+ HyperIndex = MiRemoveAnyPage(Color);
+
+ /* Zero it outside the PFN lock */
+ KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+ MiZeroPhysicalPage(HyperIndex);
+ }
+ else
+ {
+ /* Release the PFN lock */
+ KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+ }
/* Switch to phase 1 initialization */
ASSERT(Process->AddressSpaceInitialized == 0);
@@ -1112,7 +1136,6 @@
/* Copy all the kernel mappings */
PdeOffset = MiGetPdeOffset(MmSystemRangeStart);
-
RtlCopyMemory(&SystemTable[PdeOffset],
MiAddressToPde(MmSystemRangeStart),
PAGE_SIZE - PdeOffset * sizeof(MMPTE));
Modified: trunk/reactos/ntoskrnl/mm/ARM3/zeropage.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/ARM3/zeropage.…
==============================================================================
--- trunk/reactos/ntoskrnl/mm/ARM3/zeropage.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/mm/ARM3/zeropage.c [iso-8859-1] Wed Sep 29 01:10:28 2010
@@ -69,8 +69,11 @@
}
PageIndex = MmFreePageListHead.Flink;
+ ASSERT(PageIndex != LIST_HEAD);
Pfn1 = MiGetPfnEntry(PageIndex);
- FreePage = MiRemoveAnyPage(0); // FIXME: Use real color
+ FreePage = MiRemoveAnyPage(MI_GET_PAGE_COLOR(PageIndex));
+
+ /* The first global free page should also be the first on its own list */
if (FreePage != PageIndex)
{
KeBugCheckEx(PFN_LIST_CORRUPT,
Modified: trunk/reactos/ntoskrnl/mm/freelist.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/freelist.c?rev…
==============================================================================
--- trunk/reactos/ntoskrnl/mm/freelist.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/mm/freelist.c [iso-8859-1] Wed Sep 29 01:10:28 2010
@@ -604,11 +604,11 @@
if (Type != MC_SYSTEM)
{
- PfnOffset = MiRemoveZeroPage(0);
+ PfnOffset = MiRemoveZeroPage(MI_GET_NEXT_COLOR());
}
else
{
- PfnOffset = MiRemoveAnyPage(0);
+ PfnOffset = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
}
if (!PfnOffset)