Author: sir_richard Date: Wed Sep 29 01:10:28 2010 New Revision: 48927
URL: http://svn.reactos.org/svn/reactos?rev=48927&view=rev Log: [NTOS]: Add MiRemoveZeroPageSafe helper function, when a zero page is required, but the inline zeroing of MiRemoveZeroPage is not. This function will only try grabbing a zero page if one exists, otherwise a free page will be grabbed and zeroed with custom code of the caller's choosing. [NTOS]: Add concept of process color and system color. Compute correct color to use whenever requesting a page. [NTOS]: Uncondtionally enable the color code when inserting/removing pages. For now, when requesting a page, colors are still ignored, and the global PFN lists are scanned instead. If there are no regressions, we are one patch away from that.
Modified: trunk/reactos/ntoskrnl/mm/ARM3/miarm.h trunk/reactos/ntoskrnl/mm/ARM3/pagfault.c trunk/reactos/ntoskrnl/mm/ARM3/pfnlist.c trunk/reactos/ntoskrnl/mm/ARM3/pool.c trunk/reactos/ntoskrnl/mm/ARM3/procsup.c trunk/reactos/ntoskrnl/mm/ARM3/zeropage.c trunk/reactos/ntoskrnl/mm/freelist.c
Modified: trunk/reactos/ntoskrnl/mm/ARM3/miarm.h URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/ARM3/miarm.h?re... ============================================================================== --- trunk/reactos/ntoskrnl/mm/ARM3/miarm.h [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/mm/ARM3/miarm.h [iso-8859-1] Wed Sep 29 01:10:28 2010 @@ -217,6 +217,13 @@ // Special IRQL value (found in assertions) // #define MM_NOIRQL (KIRQL)0xFFFFFFFF + +// +// Returns the color of a page +// +#define MI_GET_PAGE_COLOR(x) ((x) & MmSecondaryColorMask) +#define MI_GET_NEXT_COLOR(x) (MI_GET_PAGE_COLOR(++MmSystemPageColor)) +#define MI_GET_NEXT_PROCESS_COLOR(x) (MI_GET_PAGE_COLOR(++(x)->NextPageColor))
// // FIXFIX: These should go in ex.h after the pool merge @@ -455,9 +462,8 @@ extern LIST_ENTRY MmProcessList; extern BOOLEAN MmZeroingPageThreadActive; extern KEVENT MmZeroingPageEvent; - -#define MI_PFN_TO_PFNENTRY(x) (&MmPfnDatabase[1][x]) -#define MI_PFNENTRY_TO_PFN(x) (x - MmPfnDatabase[1]) +extern ULONG MmSystemPageColor; +extern ULONG MmProcessColorSeed;
// // Figures out the hardware bits for a PTE @@ -1093,4 +1099,19 @@ IN PMMADDRESS_NODE Node );
+// +// MiRemoveZeroPage will use inline code to zero out the page manually if only +// free pages are available. In some scenarios, we don't/can't run that piece of +// code and would rather only have a real zero page. If we can't have a zero page, +// then we'd like to have our own code to grab a free page and zero it out, by +// using MiRemoveAnyPage. This macro implements this. +// +PFN_NUMBER +FORCEINLINE +MiRemoveZeroPageSafe(IN ULONG Color) +{ + if (MmFreePagesByColor[ZeroedPageList][Color].Flink != LIST_HEAD) return MiRemoveZeroPage(Color); + return 0; +} + /* EOF */
Modified: trunk/reactos/ntoskrnl/mm/ARM3/pagfault.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/ARM3/pagfault.c... ============================================================================== --- trunk/reactos/ntoskrnl/mm/ARM3/pagfault.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/mm/ARM3/pagfault.c [iso-8859-1] Wed Sep 29 01:10:28 2010 @@ -179,9 +179,10 @@ IN PEPROCESS Process, IN KIRQL OldIrql) { - PFN_NUMBER PageFrameNumber; + PFN_NUMBER PageFrameNumber = 0; MMPTE TempPte; BOOLEAN NeedZero = FALSE; + ULONG Color; DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process: %p\n", Address, Process); @@ -196,9 +197,17 @@ /* No forking yet */ ASSERT(Process->ForkInProgress == NULL);
+ /* Get process color */ + Color = MI_GET_NEXT_PROCESS_COLOR(Process); + /* We'll need a zero page */ NeedZero = TRUE; } + else + { + /* Get the next system page color */ + Color = MI_GET_NEXT_COLOR(); + }
// // Lock the PFN database @@ -206,9 +215,21 @@ OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); ASSERT(PointerPte->u.Hard.Valid == 0);
- /* Get a page */ - PageFrameNumber = MiRemoveAnyPage(0); - DPRINT("New pool page: %lx\n", PageFrameNumber); + /* Do we need a zero page? */ + if (NeedZero) + { + /* Try to get one, if we couldn't grab a free page and zero it */ + PageFrameNumber = MiRemoveZeroPageSafe(Color); + if (PageFrameNumber) NeedZero = FALSE; + } + + /* Did we get a page? */ + if (!PageFrameNumber) + { + /* We either failed to find a zero page, or this is a system request */ + PageFrameNumber = MiRemoveAnyPage(Color); + DPRINT("New pool page: %lx\n", PageFrameNumber); + }
/* Initialize it */ MiInitializePfn(PageFrameNumber, PointerPte, TRUE); @@ -463,6 +484,7 @@ ULONG ProtectionCode; PMMVAD Vad; PFN_NUMBER PageFrameIndex; + ULONG Color; DPRINT("ARM3 FAULT AT: %p\n", Address);
// @@ -756,19 +778,25 @@
/* Lock the PFN database since we're going to grab a page */ OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); - - /* Grab a page out of there. Later we should grab a colored zero page */ - PageFrameIndex = MiRemoveAnyPage(0); - ASSERT(PageFrameIndex); - - /* Release the lock since we need to do some zeroing */ - KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); - - /* Zero out the page, since it's for user-mode */ - MiZeroPfn(PageFrameIndex); - - /* Grab the lock again so we can initialize the PFN entry */ - OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + + /* Try to get a zero page */ + Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess); + PageFrameIndex = MiRemoveZeroPageSafe(Color); + if (!PageFrameIndex) + { + /* Grab a page out of there. Later we should grab a colored zero page */ + PageFrameIndex = MiRemoveAnyPage(Color); + ASSERT(PageFrameIndex); + + /* Release the lock since we need to do some zeroing */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + + /* Zero out the page, since it's for user-mode */ + MiZeroPfn(PageFrameIndex); + + /* Grab the lock again so we can initialize the PFN entry */ + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + }
/* Initialize the PFN entry now */ MiInitializePfn(PageFrameIndex, PointerPte, 1);
Modified: trunk/reactos/ntoskrnl/mm/ARM3/pfnlist.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/ARM3/pfnlist.c?... ============================================================================== --- trunk/reactos/ntoskrnl/mm/ARM3/pfnlist.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/mm/ARM3/pfnlist.c [iso-8859-1] Wed Sep 29 01:10:28 2010 @@ -30,12 +30,11 @@ #define ASSERT_LIST_INVARIANT(x) #endif
-#define ARM3_COLORS 1 - /* GLOBALS ********************************************************************/
BOOLEAN MmDynamicPfn; BOOLEAN MmMirroring; +ULONG MmSystemPageColor;
MMPFNLIST MmZeroedPageListHead = {0, ZeroedPageList, LIST_HEAD, LIST_HEAD}; MMPFNLIST MmFreePageListHead = {0, FreePageList, LIST_HEAD, LIST_HEAD}; @@ -80,11 +79,9 @@ PFN_NUMBER OldFlink, OldBlink; PMMPFNLIST ListHead; MMLISTS ListName; -#ifdef ARM3_COLORS ULONG Color; PMMCOLOR_TABLES ColorTable; PMMPFN Pfn1; -#endif
/* Make sure the PFN lock is held */ ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); @@ -131,7 +128,7 @@ /* Set the list head's backlink instead */ ListHead->Flink = OldFlink; } -#ifdef ARM3_COLORS + /* Get the page color */ OldBlink = MiGetPfnEntryIndex(Entry); Color = OldBlink & MmSecondaryColorMask; @@ -185,7 +182,7 @@
/* ReactOS Hack */ Entry->OriginalPte.u.Long = 0; -#endif + /* We are not on a list anymore */ Entry->u1.Flink = Entry->u2.Blink = 0; ASSERT_LIST_INVARIANT(ListHead); @@ -219,9 +216,8 @@ MMLISTS ListName; PFN_NUMBER OldFlink, OldBlink; ULONG OldColor, OldCache; -#ifdef ARM3_COLORS PMMCOLOR_TABLES ColorTable; -#endif + /* Make sure PFN lock is held */ ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); ASSERT(Color < MmSecondaryColors); @@ -280,7 +276,7 @@ Pfn1->u3.e2.ShortFlags = 0; Pfn1->u3.e1.PageColor = OldColor; Pfn1->u3.e1.CacheAttribute = OldCache; -#ifdef ARM3_COLORS + /* Get the first page on the color list */ ASSERT(Color < MmSecondaryColors); ColorTable = &MmFreePagesByColor[ListName][Color]; @@ -306,7 +302,7 @@
/* ReactOS Hack */ Pfn1->OriginalPte.u.Long = 0; -#endif + /* See if we hit any thresholds */ if (MmAvailablePages == MmHighMemoryThreshold) { @@ -340,9 +336,8 @@ ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); ASSERT(MmAvailablePages != 0); ASSERT(Color < MmSecondaryColors); - +#if 0 /* Check the colored free list */ -#if 0 // Enable when using ARM3 database */ PageIndex = MmFreePagesByColor[FreePageList][Color].Flink; if (PageIndex == LIST_HEAD) { @@ -368,11 +363,10 @@ ASSERT(MmZeroedPageListHead.Total == 0); } } -#if 0 // Enable when using ARM3 database */ +#if 0 } } #endif - /* Remove the page from its list */ PageIndex = MiRemovePageByColor(PageIndex, Color);
@@ -403,7 +397,7 @@ ASSERT(Color < MmSecondaryColors);
/* Check the colored zero list */ -#if 0 // Enable when using ARM3 database */ +#if 0 PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink; if (PageIndex == LIST_HEAD) { @@ -414,9 +408,10 @@ Color = PageIndex & MmSecondaryColorMask; if (PageIndex == LIST_HEAD) { + /* This means there's no zero pages, we have to look for free ones */ ASSERT(MmZeroedPageListHead.Total == 0); Zero = TRUE; -#if 0 // Enable when using ARM3 database */ +#if 0 /* Check the colored free list */ PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink; if (PageIndex == LIST_HEAD) @@ -432,13 +427,14 @@ /* FIXME: Should check the standby list */ ASSERT(MmZeroedPageListHead.Total == 0); } -#if 0 // Enable when using ARM3 database */ +#if 0 } #endif } -#if 0 // Enable when using ARM3 database */ +#if 0 } #endif + /* Sanity checks */ Pfn1 = MiGetPfnEntry(PageIndex); ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) || @@ -468,11 +464,10 @@ PMMPFNLIST ListHead; PFN_NUMBER LastPage; PMMPFN Pfn1; -#ifdef ARM3_COLORS ULONG Color; PMMPFN Blink; PMMCOLOR_TABLES ColorTable; -#endif + /* Make sure the page index is valid */ ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); ASSERT((PageFrameIndex != 0) && @@ -537,7 +532,7 @@ /* Otherwise check if we reached the high threshold and signal the event */ KeSetEvent(MiHighMemoryEvent, 0, FALSE); } -#ifdef ARM3_COLORS + /* Get the page color */ Color = PageFrameIndex & MmSecondaryColorMask;
@@ -571,7 +566,7 @@
/* And increase the count in the colored list */ ColorTable->Count++; -#endif + /* Notify zero page thread if enough pages are on the free list now */ if ((ListHead->Total >= 8) && !(MmZeroingPageThreadActive)) { @@ -590,10 +585,9 @@ PFN_NUMBER Flink; PMMPFN Pfn1, Pfn2; MMLISTS ListName; -#ifdef ARM3_COLORS PMMCOLOR_TABLES ColorHead; ULONG Color; -#endif + /* For free pages, use MiInsertPageInFreeList */ ASSERT(ListHead != &MmFreePageListHead);
@@ -657,7 +651,7 @@ KeSetEvent(MiHighMemoryEvent, 0, FALSE); }
-#ifdef ARM3_COLORS + /* Sanity checks */ ASSERT(ListName == ZeroedPageList); ASSERT(Pfn1->u4.InPageError == 0);
@@ -695,7 +689,6 @@
/* One more paged on the colored list */ ColorHead->Count++; -#endif }
VOID @@ -783,7 +776,7 @@ /* Grab a page */ ASSERT_LIST_INVARIANT(&MmFreePageListHead); ASSERT_LIST_INVARIANT(&MmZeroedPageListHead); - PageFrameIndex = MiRemoveAnyPage(0); + PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
/* Write the software PTE */ MI_WRITE_INVALID_PTE(PointerPte, TempPte);
Modified: trunk/reactos/ntoskrnl/mm/ARM3/pool.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/ARM3/pool.c?rev... ============================================================================== --- trunk/reactos/ntoskrnl/mm/ARM3/pool.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/mm/ARM3/pool.c [iso-8859-1] Wed Sep 29 01:10:28 2010 @@ -473,7 +473,7 @@ ASSERT(PointerPte->u.Hard.Valid == 0);
/* Request a page */ - PageFrameNumber = MiRemoveAnyPage(0); + PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
#if (_MI_PAGING_LEVELS >= 3) @@ -768,7 +768,7 @@ do { /* Allocate a page */ - PageFrameNumber = MiRemoveAnyPage(0); + PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
/* Get the PFN entry for it and fill it out */ Pfn1 = MiGetPfnEntry(PageFrameNumber);
Modified: trunk/reactos/ntoskrnl/mm/ARM3/procsup.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/ARM3/procsup.c?... ============================================================================== --- trunk/reactos/ntoskrnl/mm/ARM3/procsup.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/mm/ARM3/procsup.c [iso-8859-1] Wed Sep 29 01:10:28 2010 @@ -16,7 +16,9 @@ #define MODULE_INVOLVED_IN_ARM3 #include "../ARM3/miarm.h"
-extern MM_SYSTEMSIZE MmSystemSize; +/* GLOBALS ********************************************************************/ + +ULONG MmProcessColorSeed = 0x12345678;
/* PRIVATE FUNCTIONS **********************************************************/
@@ -357,7 +359,7 @@ PointerPte++;
/* Get a page and write the current invalid PTE */ - PageFrameIndex = MiRemoveAnyPage(0); + PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); MI_WRITE_INVALID_PTE(PointerPte, InvalidPte);
/* Initialize the PFN entry for this page */ @@ -444,7 +446,7 @@ while (LimitPte >= NewLimitPte) { /* Get a page and write the current invalid PTE */ - PageFrameIndex = MiRemoveAnyPage(0); + PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); MI_WRITE_INVALID_PTE(LimitPte, InvalidPte);
/* Initialize the PFN entry for this page */ @@ -1058,9 +1060,10 @@ MMPTE TempPte, PdePte; ULONG PdeOffset; PMMPTE SystemTable; - - /* No page colors yet */ - Process->NextPageColor = 0; + ULONG Color; + + /* Choose a process color */ + Process->NextPageColor = RtlRandom(&MmProcessColorSeed);
/* Setup the hyperspace lock */ KeInitializeSpinLock(&Process->HyperSpaceLock); @@ -1068,16 +1071,37 @@ /* Lock PFN database */ OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
- /* Get a page for the PDE */ - PdeIndex = MiRemoveAnyPage(0); - KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); - MiZeroPhysicalPage(PdeIndex); - OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); - - /* Get a page for hyperspace */ - HyperIndex = MiRemoveAnyPage(0); - KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); - MiZeroPhysicalPage(HyperIndex); + /* Get a zero page for the PDE, if possible */ + Color = MI_GET_NEXT_PROCESS_COLOR(Process); + PdeIndex = MiRemoveZeroPageSafe(Color); + if (!PdeIndex) + { + /* No zero pages, grab a free one */ + PdeIndex = MiRemoveAnyPage(Color); + + /* Zero it outside the PFN lock */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + MiZeroPhysicalPage(PdeIndex); + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + } + + /* Get a zero page for hyperspace, if possible */ + Color = MI_GET_NEXT_PROCESS_COLOR(Process); + HyperIndex = MiRemoveZeroPageSafe(Color); + if (!HyperIndex) + { + /* No zero pages, grab a free one */ + HyperIndex = MiRemoveAnyPage(Color); + + /* Zero it outside the PFN lock */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + MiZeroPhysicalPage(HyperIndex); + } + else + { + /* Release the PFN lock */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + }
/* Switch to phase 1 initialization */ ASSERT(Process->AddressSpaceInitialized == 0); @@ -1112,7 +1136,6 @@
/* Copy all the kernel mappings */ PdeOffset = MiGetPdeOffset(MmSystemRangeStart); - RtlCopyMemory(&SystemTable[PdeOffset], MiAddressToPde(MmSystemRangeStart), PAGE_SIZE - PdeOffset * sizeof(MMPTE));
Modified: trunk/reactos/ntoskrnl/mm/ARM3/zeropage.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/ARM3/zeropage.c... ============================================================================== --- trunk/reactos/ntoskrnl/mm/ARM3/zeropage.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/mm/ARM3/zeropage.c [iso-8859-1] Wed Sep 29 01:10:28 2010 @@ -69,8 +69,11 @@ }
PageIndex = MmFreePageListHead.Flink; + ASSERT(PageIndex != LIST_HEAD); Pfn1 = MiGetPfnEntry(PageIndex); - FreePage = MiRemoveAnyPage(0); // FIXME: Use real color + FreePage = MiRemoveAnyPage(MI_GET_PAGE_COLOR(PageIndex)); + + /* The first global free page should also be the first on its own list */ if (FreePage != PageIndex) { KeBugCheckEx(PFN_LIST_CORRUPT,
Modified: trunk/reactos/ntoskrnl/mm/freelist.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/freelist.c?rev=... ============================================================================== --- trunk/reactos/ntoskrnl/mm/freelist.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/mm/freelist.c [iso-8859-1] Wed Sep 29 01:10:28 2010 @@ -604,11 +604,11 @@
if (Type != MC_SYSTEM) { - PfnOffset = MiRemoveZeroPage(0); + PfnOffset = MiRemoveZeroPage(MI_GET_NEXT_COLOR()); } else { - PfnOffset = MiRemoveAnyPage(0); + PfnOffset = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); }
if (!PfnOffset)