[jgardou] 64228: Try to repair catastrophic merge attempt from r64223, part 1 of 2 [NTOS/MM] Commit what I got so far, including: - A very (very) basic balancer for the paged pool, simply sweeping ...

Author: jgardou
Date: Mon Sep 22 17:20:38 2014
New Revision: 64228

URL: http://svn.reactos.org/svn/reactos?rev=64228&view=rev
Log:
Try to repair catastrophic merge attempt from r64223, part 1 of 2
[NTOS/MM]
Commit what I got so far, including:
 - A very (very) basic balancer for the paged pool, simply sweeping all the pages of it
and ageing the pages as they're run over
 - A page writer thread writing unreferenced pages to the pagefile.
 - Support for reading from the pagefile in the ARM3 page fault handler
 - Popping a page from the standby list when running out of free pages
 - Various fixes here and there
Enjoy!

Modified:
    branches/TransitionPte/ntoskrnl/include/internal/mm.h
    branches/TransitionPte/ntoskrnl/mm/ARM3/miarm.h
    branches/TransitionPte/ntoskrnl/mm/ARM3/pagfault.c
    branches/TransitionPte/ntoskrnl/mm/ARM3/pfnlist.c
    branches/TransitionPte/ntoskrnl/mm/ARM3/section.c
    branches/TransitionPte/ntoskrnl/mm/ARM3/virtual.c
    branches/TransitionPte/ntoskrnl/mm/balance.c
    branches/TransitionPte/ntoskrnl/mm/mminit.c
    branches/TransitionPte/ntoskrnl/mm/pagefile.c

Modified: branches/TransitionPte/ntoskrnl/include/internal/mm.h
URL:
http://svn.reactos.org/svn/reactos/branches/TransitionPte/ntoskrnl/include/…
==============================================================================
--- branches/TransitionPte/ntoskrnl/include/internal/mm.h	[iso-8859-1] (original)
+++ branches/TransitionPte/ntoskrnl/include/internal/mm.h	[iso-8859-1] Mon Sep 22 17:20:38
2014
@@ -373,6 +373,7 @@
 extern MMPFNLIST MmFreePageListHead;
 extern MMPFNLIST MmStandbyPageListHead;
 extern MMPFNLIST MmModifiedPageListHead;
+extern MMPFNLIST MmModifiedPageListByColor[1];
 extern MMPFNLIST MmModifiedNoWritePageListHead;
 
 typedef struct _MM_MEMORY_CONSUMER
@@ -652,8 +653,27 @@
 NTAPI
 MiReadPageFile(
     _In_ PFN_NUMBER Page,
-    _In_ ULONG PageFileIndex,
-    _In_ ULONG_PTR PageFileOffset);
+    _In_ const MMPTE* PointerPte
+);
+
+NTSTATUS
+NTAPI
+MiWritePageFile(
+    _In_ PFN_NUMBER Page,
+    _In_ const MMPTE* PointerPte
+);
+
+VOID
+NTAPI
+MiFreePageFileEntry(
+    _In_ PMMPTE PointerPte
+);
+
+NTSTATUS
+NTAPI
+MiReservePageFileEntry(
+    _Out_ PMMPTE PointerPte
+);
 
 /* process.c ****************************************************************/
 

Modified: branches/TransitionPte/ntoskrnl/mm/ARM3/miarm.h
URL:
http://svn.reactos.org/svn/reactos/branches/TransitionPte/ntoskrnl/mm/ARM3/…
==============================================================================
--- branches/TransitionPte/ntoskrnl/mm/ARM3/miarm.h	[iso-8859-1] (original)
+++ branches/TransitionPte/ntoskrnl/mm/ARM3/miarm.h	[iso-8859-1] Mon Sep 22 17:20:38 2014
@@ -687,6 +687,7 @@
 extern PKEVENT MiHighPagedPoolEvent;
 extern PKEVENT MiLowNonPagedPoolEvent;
 extern PKEVENT MiHighNonPagedPoolEvent;
+extern KEVENT MpwThreadEvent;
 extern PFN_NUMBER MmLowMemoryThreshold;
 extern PFN_NUMBER MmHighMemoryThreshold;
 extern PFN_NUMBER MiLowPagedPoolThreshold;
@@ -695,6 +696,7 @@
 extern PFN_NUMBER MiHighNonPagedPoolThreshold;
 extern PFN_NUMBER MmMinimumFreePages;
 extern PFN_NUMBER MmPlentyFreePages;
+extern ULONG_PTR MmTotalPagesForPagingFile;
 extern SIZE_T MmMinimumStackCommitInBytes;
 extern PFN_COUNT MiExpansionPoolPagesInitialCharge;
 extern PFN_NUMBER MmResidentAvailablePages;

Modified: branches/TransitionPte/ntoskrnl/mm/ARM3/pagfault.c
URL:
http://svn.reactos.org/svn/reactos/branches/TransitionPte/ntoskrnl/mm/ARM3/…
==============================================================================
--- branches/TransitionPte/ntoskrnl/mm/ARM3/pagfault.c	[iso-8859-1] (original)
+++ branches/TransitionPte/ntoskrnl/mm/ARM3/pagfault.c	[iso-8859-1] Mon Sep 22 17:20:38
2014
@@ -583,7 +583,7 @@
     ASSERT(PointerPte->u.Hard.Valid == 0);
 
     /* Assert we have enough pages */
-    ASSERT(MmAvailablePages >= 32);
+    ASSERT(MmAvailablePages != 0);
 
 #if MI_TRACE_PFNS
     if (UserPdeFault) MI_SET_USAGE(MI_USAGE_PAGE_TABLE);
@@ -688,7 +688,8 @@
                         IN PMMPTE PointerPte,
                         IN PMMPTE PointerProtoPte,
                         IN KIRQL OldIrql,
-                        IN PMMPFN* LockedProtoPfn)
+                        IN PMMPFN* LockedProtoPfn,
+                        PVOID InPageBlock)
 {
     MMPTE TempPte;
     PMMPTE OriginalPte, PageTablePte;
@@ -799,6 +800,10 @@
 
     /* Reset the protection if needed */
     if (OriginalProtection) Protection = MM_ZERO_ACCESS;
+
+    /* Let waiters go ahead */
+    if (InPageBlock)
+        KeSetEvent(InPageBlock, IO_NO_INCREMENT, FALSE);
 
     /* Return success */
     ASSERT(PointerPte == MiAddressToPte(Address));
@@ -816,26 +821,30 @@
     ULONG Color;
     PFN_NUMBER Page;
     NTSTATUS Status;
-    MMPTE TempPte = *PointerPte;
-    KEVENT Event;
+    MMPTE PageFilePte = *PointerPte, TempPte;
     PMMPFN Pfn1;
-    ULONG PageFileIndex = TempPte.u.Soft.PageFileLow;
-    ULONG_PTR PageFileOffset = TempPte.u.Soft.PageFileHigh;
+
+    DPRINT1("Serving page fault for PTE %p (%x).\n", PointerPte,
PageFilePte.u.Long);
 
     /* Things we don't support yet */
-    ASSERT(CurrentProcess > HYDRA_PROCESS);
     ASSERT(*OldIrql != MM_NOIRQL);
 
     /* We must hold the PFN lock */
     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 
     /* Some sanity checks */
-    ASSERT(TempPte.u.Hard.Valid == 0);
-    ASSERT(TempPte.u.Soft.PageFileHigh != 0);
-    ASSERT(TempPte.u.Soft.PageFileHigh != MI_PTE_LOOKUP_NEEDED);
+    ASSERT(PageFilePte.u.Hard.Valid == 0);
+    ASSERT(PageFilePte.u.Soft.Transition == 0);
+    ASSERT(PageFilePte.u.Soft.Prototype == 0);
+    ASSERT(PageFilePte.u.Soft.PageFileHigh != 0);
+    ASSERT(PageFilePte.u.Soft.PageFileHigh != MI_PTE_LOOKUP_NEEDED);
 
     /* Get any page, it will be overwritten */
-    Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess);
+    if (CurrentProcess > HYDRA_PROCESS)
+        Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess);
+    else
+        Color = MI_GET_NEXT_COLOR();
+
     Page = MiRemoveAnyPage(Color);
 
     /* Initialize this PFN */
@@ -847,14 +856,13 @@
     ASSERT(Pfn1->u3.e1.ReadInProgress == 0);
     ASSERT(Pfn1->u3.e1.WriteInProgress == 0);
 
-    KeInitializeEvent(&Event, NotificationEvent, FALSE);
-    Pfn1->u1.Event = &Event;
+    /* Set NULL event */
+    Pfn1->u1.Event = NULL;
     Pfn1->u3.e1.ReadInProgress = 1;
 
     /* We must write the PTE now as the PFN lock will be released while performing the IO
operation */
+    TempPte.u.Long = 0;
     TempPte.u.Soft.Transition = 1;
-    TempPte.u.Soft.PageFileLow = 0;
-    TempPte.u.Soft.Prototype = 0;
     TempPte.u.Trans.PageFrameNumber = Page;
 
     MI_WRITE_INVALID_PTE(PointerPte, TempPte);
@@ -863,15 +871,25 @@
     KeReleaseQueuedSpinLock(LockQueuePfnLock, *OldIrql);
 
     /* Do the paging IO */
-    Status = MiReadPageFile(Page, PageFileIndex, PageFileOffset);
+    Status = MiReadPageFile(Page, &PageFilePte);
 
     /* Lock the PFN database again */
     *OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 
     /* Nobody should have changed that while we were not looking */
-    ASSERT(Pfn1->u1.Event == &Event);
     ASSERT(Pfn1->u3.e1.ReadInProgress == 1);
     ASSERT(Pfn1->u3.e1.WriteInProgress == 0);
+
+    /* But maybe someone began waiting on us */
+    if (Pfn1->u1.Event != NULL)
+    {
+        DPRINT1("The page got a waiter!\n");
+        /* Waiters gonna wait */
+        KeSetEvent(Pfn1->u1.Event, IO_NO_INCREMENT, FALSE);
+        Pfn1->u1.Event = NULL;
+    }
+
+    DPRINT1("Pagefile read finished. Status: 0x%08x.\n", Status);
 
     if (!NT_SUCCESS(Status))
     {
@@ -879,38 +897,60 @@
         ASSERT(FALSE);
         Pfn1->u4.InPageError = 1;
         Pfn1->u1.ReadStatus = Status;
+        Status = STATUS_IN_PAGE_ERROR;
+    }
+    else
+    {
+        /* This is a success status */
+        Status = STATUS_PAGE_FAULT_PAGING_FILE;
     }
 
     /* This is now a nice and normal PFN */
-    Pfn1->u1.Event = NULL;
     Pfn1->u3.e1.ReadInProgress = 0;
 
-    /* And the PTE can finally be valid */
-    MI_MAKE_HARDWARE_PTE(&TempPte, PointerPte, TempPte.u.Trans.Protection, Page);
+    /* Check if this is a kernel or user address */
+    if (PointerPte <= MiHighestUserPte)
+    {
+        /* Build the user PTE */
+        MI_MAKE_HARDWARE_PTE_USER(&TempPte,
+            PointerPte,
+            PageFilePte.u.Soft.Protection,
+            Page);
+    }
+    else
+    {
+        /* Build the kernel PTE */
+        MI_MAKE_HARDWARE_PTE(&TempPte,
+            PointerPte,
+            PageFilePte.u.Soft.Protection,
+            Page);
+    }
+
+    /* Write it */
     MI_WRITE_VALID_PTE(PointerPte, TempPte);
-
-    /* Waiters gonna wait */
-    KeSetEvent(&Event, IO_NO_INCREMENT, FALSE);
 
     return Status;
 }
 
 NTSTATUS
 NTAPI
-MiResolveTransitionFault(IN PVOID FaultingAddress,
-                         IN PMMPTE PointerPte,
-                         IN PEPROCESS CurrentProcess,
-                         IN KIRQL OldIrql,
-                         OUT PVOID *InPageBlock)
+MiResolveTransitionFault(
+    _In_ BOOLEAN StoreInstruction,
+    _In_ PVOID FaultingAddress,
+    _In_ PMMPTE PointerPte,
+    _In_ PEPROCESS CurrentProcess,
+    _In_ KIRQL OldIrql,
+    _In_ PVOID *InPageBlock)
 {
     PFN_NUMBER PageFrameIndex;
     PMMPFN Pfn1;
     MMPTE TempPte;
     PMMPTE PointerToPteForProtoPage;
-    DPRINT1("Transition fault on 0x%p with PTE 0x%p in process %s\n",
-            FaultingAddress, PointerPte, CurrentProcess->ImageFileName);
-
-    /* Windowss does this check */
+
+    DPRINT("Transition fault on 0x%p with PTE 0x%p in process %s\n",
+            FaultingAddress, PointerPte, CurrentProcess ?
CurrentProcess->ImageFileName : "Kernel");
+
+    /* Windows does this check */
     ASSERT(*InPageBlock == NULL);
 
     /* ARM3 doesn't support this path */
@@ -934,25 +974,52 @@
     ASSERT(Pfn1->u4.InPageError == 0);
 
     /* See if we should wait before terminating the fault */
-    if (Pfn1->u3.e1.ReadInProgress == 1)
-    {
-    	DPRINT1("The page is currently being read!\n");
-    	ASSERT(Pfn1->u1.Event != NULL);
-    	*InPageBlock = Pfn1->u1.Event;
-    	if (PointerPte == Pfn1->PteAddress)
-    	{
-    	    DPRINT1("And this if for this particular PTE.\n");
-    	    /* The PTE will be made valid by the thread serving the fault */
-    	    return STATUS_SUCCESS; // FIXME: Maybe something more descriptive
-    	}
+    if ((Pfn1->u3.e1.WriteInProgress == 1) || (Pfn1->u3.e1.ReadInProgress == 1))
+    {
+        KEVENT IoEvent;
+        DPRINT1("The page is currently being written to or read!\n");
+
+        /* Both of them, it's not possible */
+        ASSERT(((Pfn1->u3.e1.ReadInProgress == 0) &&
(Pfn1->u3.e1.WriteInProgress == 1)) ||
+                ((Pfn1->u3.e1.ReadInProgress == 1) &&
(Pfn1->u3.e1.WriteInProgress == 0)));
+
+        KeInitializeEvent(&IoEvent, NotificationEvent, FALSE);
+        *InPageBlock = Pfn1->u1.Event;
+        Pfn1->u1.Event = &IoEvent;
+
+        /*
+         * Mark it as unmodified.
+         * The page-out thread will put it back in the standby list
+         * and we will be able to start anew for this PTE
+         */
+        if ((Pfn1->u3.e1.WriteInProgress) && (!StoreInstruction))
+            Pfn1->u3.e1.Modified = 0;
+
+        /* Release the lock so the writer can finish */
+        KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+
+        /* Wait for it to really finish */
+        KeWaitForSingleObject(&IoEvent, WrPageIn, KernelMode, FALSE, NULL);
+
+        /* Get it again */
+        OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+
+        /* Read the PTE again */
+        TempPte = *PointerPte;
+        if (TempPte.u.Hard.Valid == 1)
+        {
+            /* Someone did it while we were not looking. */
+            return STATUS_PAGE_FAULT_TRANSITION;
+        }
+        /* Otherwise the state of this PTE must still be the same */
+        ASSERT(TempPte.u.Soft.Transition == 1);
+        ASSERT(TempPte.u.Soft.Prototype == 0);
+        ASSERT(TempPte.u.Trans.PageFrameNumber == PageFrameIndex);
     }
 
     /* Windows checks there's some free pages and this isn't an in-page error */
     ASSERT(MmAvailablePages > 0);
     ASSERT(Pfn1->u4.InPageError == 0);
-
-    /* ReactOS checks for this */
-    ASSERT(MmAvailablePages > 32);
 
     /* Was this a transition page in the valid list, or free/zero list? */
     if (Pfn1->u3.e1.PageLocation == ActiveAndValid)
@@ -1066,7 +1133,8 @@
                                        PointerPte,
                                        PointerProtoPte,
                                        OldIrql,
-                                       OutPfn);
+                                       OutPfn,
+                                       NULL);
     }
 
     /* Make sure there's some protection mask */
@@ -1125,18 +1193,25 @@
     {
         /* Resolve the transition fault */
         ASSERT(OldIrql != MM_NOIRQL);
-        Status = MiResolveTransitionFault(Address,
+        Status = MiResolveTransitionFault(StoreInstruction,
+                                          Address,
                                           PointerProtoPte,
                                           Process,
                                           OldIrql,
                                           &InPageBlock);
         ASSERT(NT_SUCCESS(Status));
     }
+    else if (TempPte.u.Soft.PageFileHigh != 0)
+    {
+        Status = MiResolvePageFileFault(StoreInstruction,
+                                        Address,
+                                        PointerProtoPte,
+                                        Process,
+                                        &OldIrql);
+        ASSERT(NT_SUCCESS(Status));
+    }
     else
     {
-        /* We also don't support paged out pages */
-        ASSERT(TempPte.u.Soft.PageFileHigh == 0);
-
         /* Resolve the demand zero fault */
         Status = MiResolveDemandZeroFault(Address,
                                           PointerProtoPte,
@@ -1152,7 +1227,8 @@
                                    PointerPte,
                                    PointerProtoPte,
                                    OldIrql,
-                                   OutPfn);
+                                   OutPfn,
+                                   InPageBlock);
 }
 
 NTSTATUS
@@ -1173,9 +1249,8 @@
     PMMPFN Pfn1, OutPfn = NULL;
     PFN_NUMBER PageFrameIndex;
     PFN_COUNT PteCount, ProcessedPtes;
-    DPRINT("ARM3 Page Fault Dispatcher for address: %p in process: %p\n",
-             Address,
-             Process);
+    DPRINT("ARM3 Page Fault Dispatcher for address: %p (PTE %p) in process:
%p\n",
+        Address, PointerPte, Process);
 
     /* Make sure the addresses are ok */
     ASSERT(PointerPte == MiAddressToPte(Address));
@@ -1283,8 +1358,49 @@
                     ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
 
                     /* Should not yet happen in ReactOS */
-                    ASSERT(Pfn1->u3.e1.ReadInProgress == 0);
                     ASSERT(Pfn1->u4.InPageError == 0);
+
+                    if ((Pfn1->u3.e1.ReadInProgress == 1) ||
(Pfn1->u3.e1.WriteInProgress == 1))
+                    {
+                        KEVENT IoEvent;
+                        PKEVENT PageEvent;
+                        DPRINT1("The page is currently being written to or
read!\n");
+
+                        /* Both of them, it's not possible */
+                        ASSERT(((Pfn1->u3.e1.ReadInProgress == 0) &&
(Pfn1->u3.e1.WriteInProgress == 1)) ||
+                                ((Pfn1->u3.e1.ReadInProgress == 1) &&
(Pfn1->u3.e1.WriteInProgress == 0)));
+
+                        KeInitializeEvent(&IoEvent, NotificationEvent, FALSE);
+                        PageEvent = Pfn1->u1.Event;
+                        Pfn1->u1.Event = &IoEvent;
+
+                        /*
+                         * Mark it as unmodified.
+                         * The page-out thread will put it back in the standby list
+                         * and we will be able to start anew for this PTE
+                         */
+                        if (Pfn1->u3.e1.WriteInProgress)
+                            Pfn1->u3.e1.Modified = 0;
+
+                        /* Release the lock so the writer can finish */
+                        KeReleaseQueuedSpinLock(LockQueuePfnLock, LockIrql);
+
+                        /* Wait for it to really finish */
+                        KeWaitForSingleObject(&IoEvent, WrPageIn, KernelMode, FALSE,
NULL);
+
+                        /* Get it again */
+                        LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+
+                        /* Read the PTE again */
+                        TempPte = *PointerProtoPte;
+
+                        /* Maybe someone else was also waiting */
+                        if (PageEvent)
+                            KeSetEvent(PageEvent, IO_NO_INCREMENT, FALSE);
+
+                        /* And re-loop */
+                        continue;
+                    }
 
                     /* Get the page */
                     MiUnlinkPageFromList(Pfn1);
@@ -1337,7 +1453,8 @@
                                             PointerPte,
                                             PointerProtoPte,
                                             LockIrql,
-                                            &OutPfn);
+                                            &OutPfn,
+                                            NULL);
 
                     /* THIS RELEASES THE PFN LOCK! */
                     break;
@@ -1390,14 +1507,14 @@
             {
                 /* We had a locked PFN, so acquire the PFN lock to dereference it */
                 ASSERT(PointerProtoPte != NULL);
-                OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+                LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 
                 /* Dereference the locked PFN */
                 MiDereferencePfnAndDropLockCount(OutPfn);
                 ASSERT(OutPfn->u3.e2.ReferenceCount >= 1);
 
                 /* And now release the lock */
-                KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+                KeReleaseQueuedSpinLock(LockQueuePfnLock, LockIrql);
             }
 
             /* Complete this as a transition fault */
@@ -1416,7 +1533,12 @@
         LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 
         /* Resolve */
-        Status = MiResolveTransitionFault(Address, PointerPte, Process, LockIrql,
&InPageBlock);
+        Status = MiResolveTransitionFault(StoreInstruction,
+            Address,
+            PointerPte,
+            Process,
+            LockIrql,
+            &InPageBlock);
 
         NT_ASSERT(NT_SUCCESS(Status));
 
@@ -1425,8 +1547,8 @@
 
         if (InPageBlock != NULL)
         {
-        	/* The page is being paged in by another process */
-        	KeWaitForSingleObject(InPageBlock, WrPageIn, KernelMode, FALSE, NULL);
+            /* We must wake the blocked threads */
+            KeSetEvent(InPageBlock, IO_NO_INCREMENT, FALSE);
         }
 
         ASSERT(OldIrql == KeGetCurrentIrql());
@@ -1821,9 +1943,6 @@
         }
         else
         {
-            /* We don't implement transition PTEs */
-            ASSERT(TempPte.u.Soft.Transition == 0);
-
             /* Check for no-access PTE */
             if (TempPte.u.Soft.Protection == MM_NOACCESS)
             {
@@ -2090,7 +2209,7 @@
             OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 
             /* Make sure we have enough pages */
-            ASSERT(MmAvailablePages >= 32);
+            ASSERT(MmAvailablePages != 0);
 
             /* Try to get a zero page */
             MI_SET_USAGE(MI_USAGE_PEB_TEB);

Modified: branches/TransitionPte/ntoskrnl/mm/ARM3/pfnlist.c
URL:
http://svn.reactos.org/svn/reactos/branches/TransitionPte/ntoskrnl/mm/ARM3/…
==============================================================================
--- branches/TransitionPte/ntoskrnl/mm/ARM3/pfnlist.c	[iso-8859-1] (original)
+++ branches/TransitionPte/ntoskrnl/mm/ARM3/pfnlist.c	[iso-8859-1] Mon Sep 22 17:20:38
2014
@@ -36,7 +36,7 @@
 ULONG MmSystemPageColor;
 
 ULONG MmTransitionSharedPages;
-ULONG MmTotalPagesForPagingFile;
+ULONG_PTR MmTotalPagesForPagingFile;
 
 MMPFNLIST MmZeroedPageListHead = {0, ZeroedPageList, LIST_HEAD, LIST_HEAD};
 MMPFNLIST MmFreePageListHead = {0, FreePageList, LIST_HEAD, LIST_HEAD};
@@ -266,7 +266,6 @@
         }
 
         /* Decrease transition page counter */
-        ASSERT(Pfn->u3.e1.PrototypePte == 1); /* Only supported ARM3 case */
         MmTransitionSharedPages--;
 
         /* One less page */
@@ -287,6 +286,9 @@
         /* Decrement the counters */
         ListHead->Total--;
         MmTotalPagesForPagingFile--;
+
+        if (MmTotalPagesForPagingFile == 0)
+            KeClearEvent(&MpwThreadEvent);
 
         /* Pick the correct colored list */
         ListHead = &MmModifiedPageListByColor[0];
@@ -477,12 +479,89 @@
     return PageIndex;
 }
 
+static
+PFN_NUMBER
+NTAPI
+MiPopPageFromStandbyList(void)
+{
+    unsigned i;
+    PFN_NUMBER PageFrameIndex;
+    PMMPFN Pfn1;
+    PMMPTE PointerPte, SysPte, PteTable;
+    MMPTE PdePte;
+
+    /* Make sure PFN lock is held and we have pages */
+    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
+    ASSERT(MmAvailablePages != 0);
+
+    /* This should be called in last resort if there are no more free pages */
+    ASSERT(MmZeroedPageListHead.Total == 0);
+    ASSERT(MmFreePageListHead.Total == 0);
+
+    /* Go by priority */
+    for (i = 0; i < 8; i++)
+    {
+        /* Get the oldest one */
+        PageFrameIndex = MmStandbyPageListByPriority[i].Blink;
+        if (PageFrameIndex != LIST_HEAD)
+            break;
+    }
+
+    /* There are no free pages but still available ones, so this must be here */
+    ASSERT(PageFrameIndex != LIST_HEAD);
+
+    /* Get the PFN */
+    Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
+
+    DPRINT1("Popping page %x from standbylist (PTE %p).\n",
+        PageFrameIndex, Pfn1->PteAddress);
+
+    /* Sanity checks */
+    ASSERT(Pfn1->u3.e1.PageLocation == StandbyPageList);
+    ASSERT(Pfn1->u3.e1.Modified == 0);
+
+    /* Get a PTE to map the page directory */
+    SysPte = MiReserveSystemPtes(1, SystemPteSpace);
+    ASSERT(SysPte != NULL);
+
+    /* Build it */
+    MI_MAKE_HARDWARE_PTE_KERNEL(&PdePte,
+                                SysPte,
+                                MM_READWRITE,
+                                Pfn1->u4.PteFrame);
+    /* And map it */
+    MI_WRITE_VALID_PTE(SysPte, PdePte);
+    PteTable = MiPteToAddress(SysPte);
+
+    /* Finally get a pointer to the PTE this page represents */
+    PointerPte =
&PteTable[MiAddressToPteOffset(MiPteToAddress(Pfn1->PteAddress))];
+
+    /* Other sanity checks */
+    ASSERT(PointerPte->u.Hard.Valid == 0);
+    ASSERT(PointerPte->u.Soft.Transition == 1);
+
+    /* Restore the PTE to its original value */
+    *PointerPte = Pfn1->OriginalPte;
+
+    /* We can get rid of this */
+    MiReleaseSystemPtes(SysPte, 1, SystemPteSpace);
+
+    /* And dereference the page table frame */
+    MiDecrementShareCount(MI_PFN_ELEMENT(Pfn1->u4.PteFrame), Pfn1->u4.PteFrame);
+
+    /* Unlink */
+    MiUnlinkPageFromList(Pfn1);
+
+    return PageFrameIndex;
+}
+
 PFN_NUMBER
 NTAPI
 MiRemoveAnyPage(IN ULONG Color)
 {
     PFN_NUMBER PageIndex;
     PMMPFN Pfn1;
+    BOOLEAN FromStandbyList = FALSE;
 
     /* Make sure PFN lock is held and we have pages */
     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
@@ -507,23 +586,27 @@
                 ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
                 PageIndex = MmZeroedPageListHead.Flink;
                 Color = PageIndex & MmSecondaryColorMask;
-                ASSERT(PageIndex != LIST_HEAD);
                 if (PageIndex == LIST_HEAD)
                 {
-                    /* FIXME: Should check the standby list */
                     ASSERT(MmZeroedPageListHead.Total == 0);
+                    /* Pop one page from the standby list and try again */
+                    PageIndex = MiPopPageFromStandbyList();
+                    FromStandbyList = TRUE;
+                    ASSERT(PageIndex != LIST_HEAD);
                 }
             }
         }
     }
 
     /* Remove the page from its list */
-    PageIndex = MiRemovePageByColor(PageIndex, Color);
+    if (!FromStandbyList)
+        PageIndex = MiRemovePageByColor(PageIndex, Color);
 
     /* Sanity checks */
     Pfn1 = MI_PFN_ELEMENT(PageIndex);
     ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
-           (Pfn1->u3.e1.PageLocation == ZeroedPageList));
+           (Pfn1->u3.e1.PageLocation == ZeroedPageList) ||
+           (Pfn1->u3.e1.PageLocation == StandbyPageList));
     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
     ASSERT(Pfn1->u2.ShareCount == 0);
     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
@@ -540,6 +623,7 @@
     PFN_NUMBER PageIndex;
     PMMPFN Pfn1;
     BOOLEAN Zero = FALSE;
+    BOOLEAN FromStandbyList = FALSE;
 
     /* Make sure PFN lock is held and we have pages */
     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
@@ -567,11 +651,13 @@
                 ASSERT_LIST_INVARIANT(&MmFreePageListHead);
                 PageIndex = MmFreePageListHead.Flink;
                 Color = PageIndex & MmSecondaryColorMask;
-                ASSERT(PageIndex != LIST_HEAD);
                 if (PageIndex == LIST_HEAD)
                 {
-                    /* FIXME: Should check the standby list */
                     ASSERT(MmZeroedPageListHead.Total == 0);
+                    /* Pop one page from the standby list and try again */
+                    PageIndex = MiPopPageFromStandbyList();
+                    FromStandbyList = TRUE;
+                    ASSERT(PageIndex != LIST_HEAD);
                 }
             }
         }
@@ -584,10 +670,12 @@
     /* Sanity checks */
     Pfn1 = MI_PFN_ELEMENT(PageIndex);
     ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
-           (Pfn1->u3.e1.PageLocation == ZeroedPageList));
+           (Pfn1->u3.e1.PageLocation == ZeroedPageList)||
+           (Pfn1->u3.e1.PageLocation == StandbyPageList));
 
     /* Remove the page from its list */
-    PageIndex = MiRemovePageByColor(PageIndex, Color);
+    if (!FromStandbyList)
+        PageIndex = MiRemovePageByColor(PageIndex, Color);
     ASSERT(Pfn1 == MI_PFN_ELEMENT(PageIndex));
 
     /* Zero it, if needed */
@@ -744,7 +832,6 @@
     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
     ASSERT(Pfn1->u4.MustBeCached == 0);
     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
-    ASSERT(Pfn1->u3.e1.PrototypePte == 1);
     ASSERT(Pfn1->u3.e1.Rom != 1);
 
     /* One more transition page on a list */
@@ -979,9 +1066,8 @@
     }
     else if (ListName == ModifiedPageList)
     {
-        /* In ARM3, page must be destined for page file, and not yet written out */
+        /* In ARM3, page must be destined for page file */
         ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0);
-        ASSERT(Pfn1->OriginalPte.u.Soft.PageFileHigh == 0);
 
         /* One more transition page */
         MmTransitionSharedPages++;
@@ -989,7 +1075,8 @@
         /* Increment the number of per-process modified pages */
         PsGetCurrentProcess()->ModifiedPageCount++;
 
-        /* FIXME: Wake up modified page writer if there are not enough free pages */
+        /* Wake the page writer thread */
+        KeSetEvent(&MpwThreadEvent, IO_NO_INCREMENT, FALSE);
     }
     else if (ListName == ModifiedNoWritePageList)
     {
@@ -1059,6 +1146,9 @@
     PageFrameIndex = PFN_FROM_PTE(PointerPtePte);
     ASSERT(PageFrameIndex != 0);
     Pfn1->u4.PteFrame = PageFrameIndex;
+
+    /* HACK until we get working sets rolling */
+    Pfn1->Wsle.u1.e1.Age = 0;
 
     /* Increase its share count so we don't get rid of it */
     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
@@ -1270,32 +1360,8 @@
         /* PFN lock must be held */
         ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 
-        if (Pfn1->u3.e2.ReferenceCount == 1)
-        {
-            /* Is there still a PFN for this page? */
-            if (MI_IS_PFN_DELETED(Pfn1) == TRUE)
-            {
-                /* Clear the last reference */
-                Pfn1->u3.e2.ReferenceCount = 0;
-                ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0);
-
-                /* Mark the page temporarily as valid, we're going to make it free
soon */
-                Pfn1->u3.e1.PageLocation = ActiveAndValid;
-
-                /* Bring it back into the free list */
-                MiInsertPageInFreeList(PageFrameIndex);
-            }
-            else
-            {
-                /* PFN not yet deleted, drop a ref count */
-                MiDecrementReferenceCount(Pfn1, PageFrameIndex);
-            }
-        }
-        else
-        {
-            /* Otherwise, just drop the reference count */
-            InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
-        }
+        /* Drop a ref count */
+        MiDecrementReferenceCount(Pfn1, PageFrameIndex);
     }
 }
 
@@ -1338,6 +1404,13 @@
     /* Did someone set the delete flag? */
     if (MI_IS_PFN_DELETED(Pfn1))
     {
+        /* Did we have a Pagefile entry for it */
+        if ((Pfn1->OriginalPte.u.Soft.Prototype == 0)
+                && (Pfn1->OriginalPte.u.Soft.PageFileHigh != 0))
+        {
+            /* Yes. Set it free */
+            MiFreePageFileEntry(&Pfn1->OriginalPte);
+        }
         /* Insert it into the free list, there's nothing left to do */
         MiInsertPageInFreeList(PageFrameIndex);
         return;

Modified: branches/TransitionPte/ntoskrnl/mm/ARM3/section.c
URL:
http://svn.reactos.org/svn/reactos/branches/TransitionPte/ntoskrnl/mm/ARM3/…
==============================================================================
--- branches/TransitionPte/ntoskrnl/mm/ARM3/section.c	[iso-8859-1] (original)
+++ branches/TransitionPte/ntoskrnl/mm/ARM3/section.c	[iso-8859-1] Mon Sep 22 17:20:38
2014
@@ -680,25 +680,55 @@
                 PageFrameIndex = PFN_FROM_PTE(&TempPte);
                 Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 
-                /* As this is a paged-backed section, nobody should reference it anymore
(no cache or whatever) */
-                ASSERT(Pfn1->u3.ReferenceCount == 0);
-
-                /* And it should be in standby or modified list */
-                ASSERT((Pfn1->u3.e1.PageLocation == ModifiedPageList) ||
(Pfn1->u3.e1.PageLocation == StandbyPageList));
-
-                /* Unlink it and put it back in free list */
-                MiUnlinkPageFromList(Pfn1);
-
-                /* Temporarily mark this as active and make it free again */
-                Pfn1->u3.e1.PageLocation = ActiveAndValid;
-                MI_SET_PFN_DELETED(Pfn1);
-
-                MiInsertPageInFreeList(PageFrameIndex);
+                if (Pfn1->u3.e1.WriteInProgress == 1)
+                {
+                    KEVENT WriteEvent;
+
+                    DPRINT1("Setting the page free while it is being paged
out!\n");
+
+                    /* Only the page writer thread should hold a reference to it */
+                    ASSERT(Pfn1->u3.ReferenceCount == 1);
+
+                    /* Set the page event */
+                    KeInitializeEvent(&WriteEvent, NotificationEvent, FALSE);
+                    /* We must be the only ones waiting */
+                    ASSERT(Pfn1->u1.Event == NULL);
+                    Pfn1->u1.Event = &WriteEvent;
+
+                    /* Tell the page-out thread to abort */
+                    Pfn1->u3.e1.WriteInProgress = 0;
+                    /* We canceled the write operation, so the page-write-thread will
simply dereference
+                     * the Pfn. It has no way to know whether the page will be re-used
again (page fault case)
+                     * or if it should be made free. So mark it as deleted here */
+                    MI_SET_PFN_DELETED(Pfn1);
+
+                    /* Let the writer thread finish and go along */
+                    KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+                    KeWaitForSingleObject(&WriteEvent, FreePage, KernelMode, FALSE,
NULL);
+                    OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+                }
+                else
+                {
+                    /* As this is a paged-backed section, nobody should reference it
anymore (no cache or whatever) */
+                    ASSERT(Pfn1->u3.ReferenceCount == 0);
+
+                    /* And it should be in standby or modified list */
+                    ASSERT((Pfn1->u3.e1.PageLocation == ModifiedPageList) ||
+                        (Pfn1->u3.e1.PageLocation == StandbyPageList));
+
+                    /* Unlink it and temporarily take a reference */
+                    MiUnlinkPageFromList(Pfn1);
+                    Pfn1->u3.e2.ReferenceCount++;
+
+                    /* This will put it back in free list and clean up properly */
+                    MI_SET_PFN_DELETED(Pfn1);
+                    MiDecrementReferenceCount(Pfn1, PageFrameIndex);
+                }
             }
             else if (TempPte.u.Soft.PageFileHigh != 0)
             {
-                /* Should not happen for now */
-                ASSERT(FALSE);
+                /* There isn't much to do */
+                MiFreePageFileEntry(&TempPte);
             }
         }
         else

Modified: branches/TransitionPte/ntoskrnl/mm/ARM3/virtual.c
URL:
http://svn.reactos.org/svn/reactos/branches/TransitionPte/ntoskrnl/mm/ARM3/…
==============================================================================
--- branches/TransitionPte/ntoskrnl/mm/ARM3/virtual.c	[iso-8859-1] (original)
+++ branches/TransitionPte/ntoskrnl/mm/ARM3/virtual.c	[iso-8859-1] Mon Sep 22 17:20:38
2014
@@ -317,7 +317,6 @@
         {
             /* As always, only handle current ARM3 scenarios */
             ASSERT(PointerPte->u.Soft.Prototype == 0);
-            ASSERT(PointerPte->u.Soft.Transition == 0);
 
             /* Normally this is one possibility -- freeing a valid page */
             if (PointerPte->u.Hard.Valid)
@@ -345,6 +344,57 @@
 
                 /* Decrement the page table too */
                 MiDecrementShareCount(Pfn2, PageTableIndex);
+
+                /* Release the PFN database */
+                KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+
+                /* Destroy the PTE */
+                MI_ERASE_PTE(PointerPte);
+            }
+            else if (PointerPte->u.Soft.Transition == 1)
+            {
+                /* Get the page PFN */
+                PageFrameIndex = PFN_FROM_PTE(PointerPte);
+                Pfn1 = MiGetPfnEntry(PageFrameIndex);
+
+                /* The page should be out of any working set */
+                ASSERT(Pfn1->u1.WsIndex == 0);
+
+                /* Get the page table entry */
+                PageTableIndex = Pfn1->u4.PteFrame;
+                Pfn2 = MiGetPfnEntry(PageTableIndex);
+
+                /* Lock the PFN database */
+                OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+
+                /* Delete the page */
+                MI_SET_PFN_DELETED(Pfn1);
+
+                /* Decrement the page table */
+                MiDecrementShareCount(Pfn2, PageTableIndex);
+
+                ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
+                /* Unlink it and temporarily take a reference */
+                MiUnlinkPageFromList(Pfn1);
+                Pfn1->u3.e2.ReferenceCount++;
+
+                /* This will put it back in free list and clean properly up */
+                MI_SET_PFN_DELETED(Pfn1);
+                MiDecrementReferenceCount(Pfn1, PageFrameIndex);
+
+                /* Release the PFN database */
+                KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+
+                /* Destroy the PTE */
+                MI_ERASE_PTE(PointerPte);
+            }
+            else if (PointerPte->u.Soft.PageFileHigh != 0)
+            {
+                /* The PFN lock also protects the page files */
+                OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+
+                /* Pretty easy: free the page file slot and go look elsewhere */
+                MiFreePageFileEntry(PointerPte);
 
                 /* Release the PFN database */
                 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
@@ -358,10 +408,7 @@
                  * The only other ARM3 possibility is a demand zero page, which would
                  * mean freeing some of the paged pool pages that haven't even been
                  * touched yet, as part of a larger allocation.
-                 *
-                 * Right now, we shouldn't expect any page file information in the
PTE
                  */
-                ASSERT(PointerPte->u.Soft.PageFileHigh == 0);
 
                 /* Destroy the PTE */
                 MI_ERASE_PTE(PointerPte);
@@ -407,9 +454,8 @@
     /* See if the PTE is valid */
     if (TempPte.u.Hard.Valid == 0)
     {
-        /* Prototype and paged out PTEs not supported yet */
+        /* Prototype PTEs not supported yet */
         ASSERT(TempPte.u.Soft.Prototype == 0);
-        ASSERT((TempPte.u.Soft.PageFileHigh == 0) || (TempPte.u.Soft.Transition == 1));
 
         if (TempPte.u.Soft.Transition)
         {
@@ -435,15 +481,24 @@
                 /* And it should be in standby or modified list */
                 ASSERT((Pfn1->u3.e1.PageLocation == ModifiedPageList) ||
(Pfn1->u3.e1.PageLocation == StandbyPageList));
 
-                /* Unlink it and temporarily mark it as active */
+                /* Unlink it and temporarily take a reference */
                 MiUnlinkPageFromList(Pfn1);
                 Pfn1->u3.e2.ReferenceCount++;
-                Pfn1->u3.e1.PageLocation = ActiveAndValid;
 
                 /* This will put it back in free list and clean properly up */
                 MI_SET_PFN_DELETED(Pfn1);
                 MiDecrementReferenceCount(Pfn1, PageFrameIndex);
             }
+            return;
+        }
+
+        if (TempPte.u.Soft.PageFileHigh != 0)
+        {
+            /* Release the page file entry */
+            MiFreePageFileEntry(&TempPte);
+
+            /* And that's pretty much it */
+            MI_ERASE_PTE(PointerPte);
             return;
         }
     }
@@ -475,7 +530,6 @@
         }
 #endif
         /* Drop the share count on the page table */
-        PointerPde = MiPteToPde(PointerPte);
         MiDecrementShareCount(MiGetPfnEntry(PointerPde->u.Hard.PageFrameNumber),
             PointerPde->u.Hard.PageFrameNumber);
 

Modified: branches/TransitionPte/ntoskrnl/mm/balance.c
URL:
http://svn.reactos.org/svn/reactos/branches/TransitionPte/ntoskrnl/mm/balan…
==============================================================================
--- branches/TransitionPte/ntoskrnl/mm/balance.c	[iso-8859-1] (original)
+++ branches/TransitionPte/ntoskrnl/mm/balance.c	[iso-8859-1] Mon Sep 22 17:20:38 2014
@@ -22,22 +22,14 @@
 #pragma alloc_text(INIT, MiInitBalancerThread)
 #endif
 
-
-/* TYPES ********************************************************************/
-typedef struct _MM_ALLOCATION_REQUEST
-{
-   PFN_NUMBER Page;
-   LIST_ENTRY ListEntry;
-   KEVENT Event;
-}
-MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
+#define TEST_PAGING
+
+
 /* GLOBALS ******************************************************************/
 
 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
 static ULONG MiMinimumAvailablePages;
 static ULONG MiNrTotalPages;
-static LIST_ENTRY AllocationListHead;
-static KSPIN_LOCK AllocationListLock;
 static ULONG MiMinimumPagesPerRun;
 
 static CLIENT_ID MiBalancerThreadId;
@@ -53,8 +45,6 @@
 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
 {
    memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
-   InitializeListHead(&AllocationListHead);
-   KeInitializeSpinLock(&AllocationListLock);
 
    MiNrTotalPages = NrAvailablePages;
 
@@ -86,18 +76,10 @@
    MiMemoryConsumers[Consumer].Trim = Trim;
 }
 
-VOID
-NTAPI
-MiZeroPhysicalPage(
-    IN PFN_NUMBER PageFrameIndex
-);
-
 NTSTATUS
 NTAPI
 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
 {
-   PMM_ALLOCATION_REQUEST Request;
-   PLIST_ENTRY Entry;
    KIRQL OldIrql;
 
    if (Page == 0)
@@ -110,26 +92,11 @@
    {
       if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
       (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
-      if ((Entry = ExInterlockedRemoveHeadList(&AllocationListHead,
&AllocationListLock)) == NULL)
-      {
-         OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
-         MmDereferencePage(Page);
-         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
-      }
-      else
-      {
-         Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
-         MiZeroPhysicalPage(Page);
-         Request->Page = Page;
-         KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
-      }
    }
-   else
-   {
-      OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
-      MmDereferencePage(Page);
-      KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
-   }
+
+   OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+   MmDereferencePage(Page);
+   KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 
    return(STATUS_SUCCESS);
 }
@@ -149,10 +116,13 @@
         return InitialTarget;
     }
 
-    if (MiMemoryConsumers[Consumer].PagesUsed >
MiMemoryConsumers[Consumer].PagesTarget)
-    {
-        /* Consumer page limit exceeded */
-        Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed -
MiMemoryConsumers[Consumer].PagesTarget);
+    // if (MiFreeSwapPages == 0)
+    {
+        if (MiMemoryConsumers[Consumer].PagesUsed >
MiMemoryConsumers[Consumer].PagesTarget)
+        {
+            /* Consumer page limit exceeded */
+            Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed -
MiMemoryConsumers[Consumer].PagesTarget);
+        }
     }
     if (MmAvailablePages < MiMinimumAvailablePages)
     {
@@ -162,13 +132,6 @@
 
     if (Target)
     {
-        if (!InitialTarget)
-        {
-            /* If there was no initial target,
-             * swap at least MiMinimumPagesPerRun */
-            Target = max(Target, MiMinimumPagesPerRun);
-        }
-
         /* Now swap the pages out */
         Status = MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
 
@@ -265,8 +228,9 @@
     * Make sure we don't exceed our individual target.
     */
    PagesUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
-   if (PagesUsed > MiMemoryConsumers[Consumer].PagesTarget &&
-       !MiIsBalancerThread())
+   if ((PagesUsed > MiMemoryConsumers[Consumer].PagesTarget) &&
+       !MiIsBalancerThread() &&
+       (MmNumberOfPagingFiles == 0))
    {
       MmRebalanceMemoryConsumers();
    }
@@ -285,52 +249,6 @@
       }
       if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
       *AllocatedPage = Page;
-      if (MmAvailablePages < MiMinimumAvailablePages)
-          MmRebalanceMemoryConsumers();
-      return(STATUS_SUCCESS);
-   }
-
-   /*
-    * Make sure we don't exceed global targets.
-    */
-   if (MmAvailablePages < MiMinimumAvailablePages)
-   {
-      MM_ALLOCATION_REQUEST Request;
-
-      if (!CanWait)
-      {
-         (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
-         MmRebalanceMemoryConsumers();
-         return(STATUS_NO_MEMORY);
-      }
-
-      /* Insert an allocation request. */
-      Request.Page = 0;
-      KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
-
-      ExInterlockedInsertTailList(&AllocationListHead, &Request.ListEntry,
&AllocationListLock);
-      MmRebalanceMemoryConsumers();
-
-      KeWaitForSingleObject(&Request.Event,
-                            0,
-                            KernelMode,
-                            FALSE,
-                            NULL);
-
-      Page = Request.Page;
-      if (Page == 0)
-      {
-         KeBugCheck(NO_PAGES_AVAILABLE);
-      }
-
-      if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
-      *AllocatedPage = Page;
-
-      if (MmAvailablePages < MiMinimumAvailablePages)
-      {
-          MmRebalanceMemoryConsumers();
-      }
-
       return(STATUS_SUCCESS);
    }
 
@@ -347,91 +265,246 @@
    if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
    *AllocatedPage = Page;
 
-   if (MmAvailablePages < MiMinimumAvailablePages)
-   {
-       MmRebalanceMemoryConsumers();
-   }
-
    return(STATUS_SUCCESS);
 }
 
+/* Old implementation of the balancer thread */
+static
+VOID
+NTAPI
+MiRosBalancerThread(VOID)
+{
+    ULONG InitialTarget = 0;
+
+    DPRINT1("Legacy MM balancer in action!\n");
+
+#if (_MI_PAGING_LEVELS == 2)
+    if (!MiIsBalancerThread())
+    {
+        /* Clean up the unused PDEs */
+        ULONG_PTR Address;
+        PEPROCESS Process = PsGetCurrentProcess();
+
+        /* Acquire PFN lock */
+        KIRQL OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+        PMMPDE pointerPde;
+        for (Address = (ULONG_PTR) MI_LOWEST_VAD_ADDRESS ;
+                Address < (ULONG_PTR) MM_HIGHEST_VAD_ADDRESS; Address += (PAGE_SIZE *
PTE_COUNT))
+        {
+            if (MiQueryPageTableReferences((PVOID) Address) == 0)
+            {
+                pointerPde = MiAddressToPde(Address);
+                if (pointerPde->u.Hard.Valid)
+                    MiDeletePte(pointerPde, MiPdeToPte(pointerPde), Process, NULL);
+                ASSERT(pointerPde->u.Hard.Valid == 0);
+            }
+        }
+        /* Release lock */
+        KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+    }
+#endif
+    do
+    {
+        ULONG OldTarget = InitialTarget;
+        ULONG i;
+
+        /* Trim each consumer */
+        for (i = 0; i < MC_MAXIMUM; i++)
+        {
+            InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
+        }
+
+        /* No pages left to swap! */
+        if (InitialTarget != 0 && InitialTarget == OldTarget)
+        {
+            /* Game over */
+            KeBugCheck(NO_PAGES_AVAILABLE);
+        }
+    } while (InitialTarget != 0);
+}
 
 VOID NTAPI
 MiBalancerThread(PVOID Unused)
 {
-   PVOID WaitObjects[2];
-   NTSTATUS Status;
-   ULONG i;
-
-   WaitObjects[0] = &MiBalancerEvent;
-   WaitObjects[1] = &MiBalancerTimer;
-
-   while (1)
-   {
-      Status = KeWaitForMultipleObjects(2,
-                                        WaitObjects,
-                                        WaitAny,
-                                        Executive,
-                                        KernelMode,
-                                        FALSE,
-                                        NULL,
-                                        NULL);
-
-      if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
-      {
-        ULONG InitialTarget = 0;
-
-#if (_MI_PAGING_LEVELS == 2)
-        if (!MiIsBalancerThread())
-        {
-            /* Clean up the unused PDEs */
-            ULONG_PTR Address;
-            PEPROCESS Process = PsGetCurrentProcess();
-
-            /* Acquire PFN lock */
-            KIRQL OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
-            PMMPDE pointerPde;
-            for (Address = (ULONG_PTR)MI_LOWEST_VAD_ADDRESS;
-                 Address < (ULONG_PTR)MM_HIGHEST_VAD_ADDRESS;
-                 Address += (PAGE_SIZE * PTE_COUNT))
-            {
-                if (MiQueryPageTableReferences((PVOID)Address) == 0)
+    PVOID WaitObjects[2];
+    NTSTATUS Status;
+    ULONG i;
+    PVOID LastPagedPoolAddress = MmPagedPoolStart;
+#ifdef TEST_PAGING
+    PULONG PagingTestBuffer;
+    BOOLEAN TestPage = FALSE;
+#endif
+
+    WaitObjects[0] = &MiBalancerTimer;
+    WaitObjects[1] = &MiBalancerEvent;
+
+#ifdef TEST_PAGING
+    PagingTestBuffer = ExAllocatePoolWithTag(PagedPool, PAGE_SIZE, 'tseT');
+    DPRINT1("PagingTestBuffer: %p, PTE %p\n", PagingTestBuffer,
MiAddressToPte(PagingTestBuffer));
+    for (i = 0; i < (PAGE_SIZE / sizeof(ULONG)); i++)
+        PagingTestBuffer[i] = i;
+#endif
+
+    while (1)
+    {
+        /* For now, we only age the paged pool, with 100 pages per run. */
+        LONG Count = 100;
+        PVOID Address = LastPagedPoolAddress;
+        KIRQL OldIrql;
+        PMMPTE PointerPte;
+        PMMPDE PointerPde;
+        ULONG PdeIndex;
+        MMPTE TempPte;
+        PMMPFN Pfn1;
+        BOOLEAN Flush;
+
+        Status = KeWaitForMultipleObjects(
+            2,
+            WaitObjects,
+            WaitAny,
+            Executive,
+            KernelMode,
+            FALSE,
+            NULL,
+            NULL);
+        if (!NT_SUCCESS(Status))
+        {
+            DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
+            KeBugCheck(MEMORY_MANAGEMENT);
+        }
+
+        DPRINT("MM Balancer: Starting to loop.\n");
+
+        while (Count-- > 0)
+        {
+            Address = (PVOID)((ULONG_PTR) Address + PAGE_SIZE);
+
+            /* Acquire PFN lock while we are cooking */
+            OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+
+            if (Address > MmPagedPoolEnd)
+            {
+                Address = MmPagedPoolStart;
+            }
+
+            PointerPde = MiAddressToPde(Address);
+            PointerPte = MiAddressToPte(Address);
+
+            PdeIndex = ((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) /
sizeof(MMPTE);
+
+            if (MmSystemPagePtes[PdeIndex].u.Hard.Valid == 0)
+                TempPte.u.Long = 0;
+            else
+                TempPte = *PointerPte;
+
+            if (TempPte.u.Hard.Valid == 0)
+            {
+                /* Bad luck, not a paged-in address */
+#ifdef TEST_PAGING
+                if (PointerPte == MiAddressToPte(PagingTestBuffer))
                 {
-                    pointerPde = MiAddressToPde(Address);
-                    if (pointerPde->u.Hard.Valid)
-                        MiDeletePte(pointerPde, MiPdeToPte(pointerPde), Process, NULL);
-                    ASSERT(pointerPde->u.Hard.Valid == 0);
+                    /* Of course it should not magically become a prototype pte */
+                    ASSERT(TempPte.u.Soft.Prototype == 0);
+                    if (TempPte.u.Soft.Transition == 0)
+                    {
+                        /* It was paged out! */
+                        ASSERT(TempPte.u.Soft.PageFileHigh != 0);
+                        TestPage = TRUE;
+                    }
                 }
-            }
-            /* Release lock */
+#endif
+                KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+                continue;
+            }
+
+            Flush = FALSE;
+
+            /* Get the Pfn */
+            Pfn1 = MI_PFN_ELEMENT(PFN_FROM_PTE(&TempPte));
+            ASSERT(Pfn1->PteAddress == PointerPte);
+            ASSERT(Pfn1->u3.e1.PrototypePte == 0);
+
+            /* First check if it was written to */
+            if (TempPte.u.Hard.Dirty)
+                Pfn1->u3.e1.Modified = 1;
+
+            /* See if it was accessed since the last time we looked */
+            if (TempPte.u.Hard.Accessed)
+            {
+                /* Yes! Mark the page as young and fresh again */
+                Pfn1->Wsle.u1.e1.Age = 0;
+                /* Tell the CPU we want to know for the next time */
+                TempPte.u.Hard.Accessed = 0;
+                MI_UPDATE_VALID_PTE(PointerPte, TempPte);
+                Flush = TRUE;
+            }
+            else if (Pfn1->Wsle.u1.e1.Age == 3)
+            {
+                /* Page is getting old: Mark the PTE as transition */
+                DPRINT1("MM Balancer: putting %p (page %x) as transition.\n",
PointerPte,
+                    PFN_FROM_PTE(&TempPte));
+                TempPte.u.Hard.Valid = 0;
+                TempPte.u.Soft.Transition = 1;
+                TempPte.u.Trans.Protection = MM_READWRITE;
+                MiDecrementShareCount(Pfn1, PFN_FROM_PTE(&TempPte));
+                MI_WRITE_INVALID_PTE(PointerPte, TempPte);
+                Flush = TRUE;
+            }
+            else
+            {
+                /* The page is just getting older */
+                Pfn1->Wsle.u1.e1.Age++;
+            }
+
+            /* Flush the TLB entry if we have to */
+            if (Flush)
+            {
+#ifdef CONFIG_SMP
+                // FIXME: Should invalidate entry in every CPU TLB
+                ASSERT(FALSE);
+#endif
+                KeInvalidateTlbEntry(Address);
+            }
+
+            /* Unlock, we're done for this address */
             KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
         }
-#endif
-          do
-          {
-              ULONG OldTarget = InitialTarget;
-
-              /* Trim each consumer */
-              for (i = 0; i < MC_MAXIMUM; i++)
-              {
-                  InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
-              }
-
-              /* No pages left to swap! */
-              if (InitialTarget != 0 &&
-                  InitialTarget == OldTarget)
-              {
-                  /* Game over */
-                  KeBugCheck(NO_PAGES_AVAILABLE);
-              }
-          } while (InitialTarget != 0);
-      }
-      else
-      {
-         DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
-         KeBugCheck(MEMORY_MANAGEMENT);
-      }
-   }
+
+        DPRINT("MM Balancer: End of the loop.\n");
+
+#ifdef TEST_PAGING
+        if (TestPage)
+        {
+            BOOLEAN Passed = TRUE;
+            DPRINT1("Verifying data integrity of paged out data!\n");
+            for (i = 0; i < (PAGE_SIZE / sizeof(ULONG)); i++)
+            {
+                if (PagingTestBuffer[i] != i)
+                    Passed = FALSE;
+            }
+            if (Passed)
+                DPRINT1("PASSED! \\o/\n");
+            else
+                DPRINT1("FAILED /o\\\n");
+            TestPage = FALSE;
+        }
+#endif
+        /* Remember this for the next run */
+        LastPagedPoolAddress = Address;
+
+        if (Status == STATUS_WAIT_1)
+        {
+            /* Balancer thread was woken up by legacy MM
+             * This means we are really starving. Let's see if this still a problem
*/
+            if ((MmAvailablePages >= MmMinimumFreePages) &&
(MmNumberOfPagingFiles != 0))
+            {
+                /* Yay! No need to trust the old balancer */
+                continue;
+            }
+
+            MiRosBalancerThread();
+        }
+    }
 }
 
 VOID

Modified: branches/TransitionPte/ntoskrnl/mm/mminit.c
URL:
http://svn.reactos.org/svn/reactos/branches/TransitionPte/ntoskrnl/mm/mmini…
==============================================================================
--- branches/TransitionPte/ntoskrnl/mm/mminit.c	[iso-8859-1] (original)
+++ branches/TransitionPte/ntoskrnl/mm/mminit.c	[iso-8859-1] Mon Sep 22 17:20:38 2014
@@ -299,36 +299,179 @@
 NTAPI
 MmMpwThreadMain(PVOID Parameter)
 {
-   NTSTATUS Status;
-   ULONG PagesWritten;
-   LARGE_INTEGER Timeout;
-
-   UNREFERENCED_PARAMETER(Parameter);
-
-   Timeout.QuadPart = -50000000;
-
-   for(;;)
-   {
-      Status = KeWaitForSingleObject(&MpwThreadEvent,
-                                     0,
-                                     KernelMode,
-                                     FALSE,
-                                     &Timeout);
-      if (!NT_SUCCESS(Status))
-      {
-         DbgPrint("MpwThread: Wait failed\n");
-         KeBugCheck(MEMORY_MANAGEMENT);
-         return;
-      }
-
-      PagesWritten = 0;
-
-#ifndef NEWCC
-      // XXX arty -- we flush when evicting pages or destorying cache
-      // sections.
-      CcRosFlushDirtyPages(128, &PagesWritten, FALSE);
-#endif
-   }
+    NTSTATUS Status;
+    PMMPTE SysPte, PteTable;
+
+    UNREFERENCED_PARAMETER(Parameter);
+
+    /* Reserve a PTE for the page table */
+    SysPte = MiReserveSystemPtes(1, SystemPteSpace);
+    ASSERT(SysPte != NULL);
+    PteTable = MiPteToAddress(SysPte);
+
+    for(;;)
+    {
+        KIRQL OldIrql;
+        PFN_NUMBER PageFrameIndex;
+        PMMPFN Pfn;
+        MMPTE TempPte, PdePte;
+        PMMPTE PointerPte;
+        BOOLEAN PageFileEntryFromPage = FALSE;
+        /*
+         * To start working, we wait for two conditions:
+         *  - there are pages to be paged out.
+         *  - We are in a low memory situation.
+         */
+        Status = KeWaitForSingleObject(
+            &MpwThreadEvent,
+            WrPageOut,
+            KernelMode,
+            FALSE,
+            NULL);
+
+        DPRINT("THE KRAKEN WAS RELEASED AND WILL PAGE YOUR ASS OUT!\n");
+
+        if (!NT_SUCCESS(Status))
+        {
+            DPRINT1("MpwThread: Wait failed\n");
+            KeBugCheck(MEMORY_MANAGEMENT);
+            return;
+        }
+
+        /* Lock the PFN database */
+        OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+
+        /* The true main loop */
+        while ((MmTotalPagesForPagingFile != 0) && (MiFreeSwapPages != 0))
+        {
+            /* Get the first page from the list */
+            PageFrameIndex = MmModifiedPageListByColor[0].Flink;
+
+            /* Get The Pfn */
+            Pfn = MI_PFN_ELEMENT(PageFrameIndex);
+
+            /* Some things which must always hold */
+            ASSERT(Pfn->OriginalPte.u.Soft.Transition == 0);
+            ASSERT(Pfn->u3.ReferenceCount == 0);
+
+            /* And some that are not yet supported */
+            ASSERT(Pfn->OriginalPte.u.Soft.Prototype == 0);
+
+            /* Maybe this is not the first time */
+            if ((Pfn->OriginalPte.u.Soft.Prototype == 0) &&
+                    (Pfn->OriginalPte.u.Soft.PageFileHigh != 0))
+            {
+                /* Use that */
+                TempPte = Pfn->OriginalPte;
+                PageFileEntryFromPage = TRUE;
+            }
+            else
+            {
+                MiReservePageFileEntry(&TempPte);
+            }
+
+            /* Get it out of the list and reference it */
+            MiUnlinkPageFromList(Pfn);
+            MiReferenceUnusedPageAndBumpLockCount(Pfn);
+
+            ASSERT(Pfn->u3.e1.PageLocation == ModifiedPageList);
+
+            /* Mark it as write in progress */
+            Pfn->u3.e1.WriteInProgress = 1;
+            Pfn->u1.Event = NULL;
+
+            /* Release the PFN lock while we are writing */
+            KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+
+            /* Do the actual write */
+            Status = MiWritePageFile(PageFrameIndex, &TempPte);
+
+            /* Get the lock again */
+            OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+
+            /* Get a mapping to the page table */
+            MI_MAKE_HARDWARE_PTE_KERNEL(&PdePte,
+                                        SysPte,
+                                        MM_READWRITE,
+                                        Pfn->u4.PteFrame);
+            MI_WRITE_VALID_PTE(SysPte, PdePte);
+
+            /* Finally get a pointer to the PTE this page represents */
+            PointerPte =
&PteTable[MiAddressToPteOffset(MiPteToAddress(Pfn->PteAddress))];
+
+            /* Get relevant values from the original PTE */
+            TempPte.u.Soft.Protection = PointerPte->u.Soft.Protection;
+
+            /* Maybe someone aborted the operation */
+            if (Pfn->u3.e1.WriteInProgress == 0)
+            {
+                DPRINT1("Someone aborted the page-out operation!\n");
+                /* Just set the event and let the waiter go along */
+                ASSERT(Pfn->u1.Event != NULL);
+                KeSetEvent(Pfn->u1.Event, IO_NO_INCREMENT, FALSE);
+                Pfn->u1.Event = NULL;
+                /* This is now useless */
+                if (!PageFileEntryFromPage)
+                    MiFreePageFileEntry(&TempPte);
+            }
+            /* Maybe someone tried to access the page while we were not looking */
+            else if (Pfn->u1.Event != NULL)
+            {
+                DPRINT1("Page fault occured while we were paging out!\n");
+
+                /* So the page fault handler marked the page as unmodified */
+                ASSERT(Pfn->u3.e1.Modified == 0);
+
+                /* Save the pagefile entry for this page */
+                Pfn->OriginalPte = TempPte;
+                KeSetEvent(Pfn->u1.Event, IO_NO_INCREMENT, FALSE);
+                Pfn->u1.Event = NULL;
+            }
+            else if (!NT_SUCCESS(Status))
+            {
+                DPRINT1("Failed to write page to pagefile!\n");
+                /* This is now useless */
+                if (!PageFileEntryFromPage)
+                    MiFreePageFileEntry(&TempPte);
+                /* MiDereferencePfnAndDropLockCount will put it back on the tail of the
list */
+            }
+            else
+            {
+                DPRINT1("Page %x successfully paged out. PTE pointer %p (->
%x)\n",
+                    PageFrameIndex, Pfn->PteAddress, TempPte.u.Long);
+
+                /* Of course it must already be invalid */
+                ASSERT(PointerPte->u.Hard.Valid == 0);
+
+                /* And be in transition */
+                ASSERT(PointerPte->u.Soft.Transition == 1);
+
+                /* So the PTE is now officially paged out */
+                MI_WRITE_INVALID_PTE(PointerPte, TempPte);
+
+                /* And the pagefile entry belongs to the PTE, not to the page! */
+                Pfn->OriginalPte.u.Long = 0;
+
+                /* And dereference the page table frame */
+                MiDecrementShareCount(MI_PFN_ELEMENT(Pfn->u4.PteFrame),
Pfn->u4.PteFrame);
+                /* We can finally make it available for real */
+                MI_SET_PFN_DELETED(Pfn);
+            }
+
+            /* We're done with this */
+            Pfn->u3.e1.WriteInProgress = 0;
+
+            /* This will put it back in the free list */
+            MiDereferencePfnAndDropLockCount(Pfn);
+
+            /* Unmap the Page Table */
+            MI_ERASE_PTE(SysPte);
+            KeInvalidateTlbEntry(PteTable);
+        }
+
+        /* We're done for this run */
+        KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+    }
 }
 
 NTSTATUS

Modified: branches/TransitionPte/ntoskrnl/mm/pagefile.c
URL:
http://svn.reactos.org/svn/reactos/branches/TransitionPte/ntoskrnl/mm/pagef…
==============================================================================
--- branches/TransitionPte/ntoskrnl/mm/pagefile.c	[iso-8859-1] (original)
+++ branches/TransitionPte/ntoskrnl/mm/pagefile.c	[iso-8859-1] Mon Sep 22 17:20:38 2014
@@ -46,13 +46,14 @@
 {
    LIST_ENTRY PagingFileListEntry;
    PFILE_OBJECT FileObject;
-   LARGE_INTEGER MaximumSize;
-   LARGE_INTEGER CurrentSize;
+   ULONG_PTR MaximumSize;
+   ULONG_PTR CurrentSize;
    PFN_NUMBER FreePages;
    PFN_NUMBER UsedPages;
-   PULONG AllocMap;
+   PVOID AllocMap;
+   RTL_BITMAP AllocBitMap;
+   ULONG_PTR AllocBitMapHint;
    KSPIN_LOCK AllocMapLock;
-   ULONG AllocMapSize;
    PRETRIEVAL_POINTERS_BUFFER RetrievalPointers;
 }
 PAGINGFILE, *PPAGINGFILE;
@@ -72,9 +73,6 @@
 
 /* List of paging files, both used and free */
 static PPAGINGFILE PagingFileList[MAX_PAGING_FILES];
-
-/* Lock for examining the list of paging files */
-static KSPIN_LOCK PagingFileListLock;
 
 /* Number of paging files */
 ULONG MmNumberOfPagingFiles;
@@ -221,30 +219,62 @@
 NTAPI
 MmWriteToSwapPage(SWAPENTRY SwapEntry, PFN_NUMBER Page)
 {
-   ULONG i;
-   ULONG_PTR offset;
-   LARGE_INTEGER file_offset;
+    MMPTE TempPte;
+
+    TempPte.u.Long = 0;
+    TempPte.u.Soft.PageFileHigh = OFFSET_FROM_ENTRY(SwapEntry);
+    TempPte.u.Soft.PageFileLow = FILE_FROM_ENTRY(SwapEntry);
+    return MiWritePageFile(Page, &TempPte);
+}
+
+
+NTSTATUS
+NTAPI
+MmReadFromSwapPage(SWAPENTRY SwapEntry, PFN_NUMBER Page)
+{
+    MMPTE TempPte;
+
+    TempPte.u.Long = 0;
+    TempPte.u.Soft.PageFileHigh = OFFSET_FROM_ENTRY(SwapEntry);
+    TempPte.u.Soft.PageFileLow = FILE_FROM_ENTRY(SwapEntry);
+    return MiReadPageFile(Page, &TempPte);
+}
+
+NTSTATUS
+NTAPI
+MiReadPageFile(
+	_In_ PFN_NUMBER Page,
+	_In_ const MMPTE* PointerPte
+)
+{
+   LARGE_INTEGER FileOffset;
    IO_STATUS_BLOCK Iosb;
+   ULONG PageFileIndex = PointerPte->u.Soft.PageFileLow;
+   ULONG_PTR PageFileOffset = PointerPte->u.Soft.PageFileHigh;
    NTSTATUS Status;
    KEVENT Event;
-   UCHAR MdlBase[sizeof(MDL) + sizeof(ULONG)];
+   UCHAR MdlBase[sizeof(MDL) + sizeof(PFN_NUMBER)];
    PMDL Mdl = (PMDL)MdlBase;
-
-   DPRINT("MmWriteToSwapPage\n");
-
-   if (SwapEntry == 0)
+   PPAGINGFILE PagingFile;
+
+   DPRINT("MiReadPageFile\n");
+
+   if (PageFileOffset == 0)
    {
       KeBugCheck(MEMORY_MANAGEMENT);
       return(STATUS_UNSUCCESSFUL);
    }
 
-   i = FILE_FROM_ENTRY(SwapEntry);
-   offset = OFFSET_FROM_ENTRY(SwapEntry);
-
-   if (PagingFileList[i]->FileObject == NULL ||
-         PagingFileList[i]->FileObject->DeviceObject == NULL)
-   {
-      DPRINT1("Bad paging file 0x%.8X\n", SwapEntry);
+   /* Normalize it */
+   PageFileOffset--;
+
+   ASSERT(PageFileIndex < MAX_PAGING_FILES);
+
+   PagingFile = PagingFileList[PageFileIndex];
+
+   if (PagingFile->FileObject == NULL || PagingFile->FileObject->DeviceObject ==
NULL)
+   {
+      DPRINT1("Bad paging file %u\n", PageFileIndex);
       KeBugCheck(MEMORY_MANAGEMENT);
    }
 
@@ -252,80 +282,13 @@
    MmBuildMdlFromPages(Mdl, &Page);
    Mdl->MdlFlags |= MDL_PAGES_LOCKED;
 
-   file_offset.QuadPart = offset * PAGE_SIZE;
-   file_offset = MmGetOffsetPageFile(PagingFileList[i]->RetrievalPointers,
file_offset);
-
-   KeInitializeEvent(&Event, NotificationEvent, FALSE);
-   Status = IoSynchronousPageWrite(PagingFileList[i]->FileObject,
-                                   Mdl,
-                                   &file_offset,
-                                   &Event,
-                                   &Iosb);
-   if (Status == STATUS_PENDING)
-   {
-      KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL);
-      Status = Iosb.Status;
-   }
-
-   if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA)
-   {
-      MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl);
-   }
-   return(Status);
-}
-
-
-NTSTATUS
-NTAPI
-MmReadFromSwapPage(SWAPENTRY SwapEntry, PFN_NUMBER Page)
-{
-	return MiReadPageFile(Page, FILE_FROM_ENTRY(SwapEntry), OFFSET_FROM_ENTRY(SwapEntry));
-}
-
-NTSTATUS
-NTAPI
-MiReadPageFile(
-	_In_ PFN_NUMBER Page,
-	_In_ ULONG PageFileIndex,
-	_In_ ULONG_PTR PageFileOffset)
-{
-   LARGE_INTEGER file_offset;
-   IO_STATUS_BLOCK Iosb;
-   NTSTATUS Status;
-   KEVENT Event;
-   UCHAR MdlBase[sizeof(MDL) + sizeof(ULONG)];
-   PMDL Mdl = (PMDL)MdlBase;
-   PPAGINGFILE PagingFile;
-
-   DPRINT("MiReadSwapFile\n");
-
-   if (PageFileOffset == 0)
-   {
-      KeBugCheck(MEMORY_MANAGEMENT);
-      return(STATUS_UNSUCCESSFUL);
-   }
-
-   ASSERT(PageFileIndex < MAX_PAGING_FILES);
-
-   PagingFile = PagingFileList[PageFileIndex];
-
-   if (PagingFile->FileObject == NULL || PagingFile->FileObject->DeviceObject ==
NULL)
-   {
-      DPRINT1("Bad paging file %u\n", PageFileIndex);
-      KeBugCheck(MEMORY_MANAGEMENT);
-   }
-
-   MmInitializeMdl(Mdl, NULL, PAGE_SIZE);
-   MmBuildMdlFromPages(Mdl, &Page);
-   Mdl->MdlFlags |= MDL_PAGES_LOCKED;
-
-   file_offset.QuadPart = PageFileOffset * PAGE_SIZE;
-   file_offset = MmGetOffsetPageFile(PagingFile->RetrievalPointers, file_offset);
+   FileOffset.QuadPart = PageFileOffset * PAGE_SIZE;
+   FileOffset = MmGetOffsetPageFile(PagingFile->RetrievalPointers, FileOffset);
 
    KeInitializeEvent(&Event, NotificationEvent, FALSE);
    Status = IoPageRead(PagingFile->FileObject,
                        Mdl,
-                       &file_offset,
+                       &FileOffset,
                        &Event,
                        &Iosb);
    if (Status == STATUS_PENDING)
@@ -340,14 +303,182 @@
    return(Status);
 }
 
+NTSTATUS
+NTAPI
+MiWritePageFile(
+    _In_ PFN_NUMBER Page,
+    _In_ const MMPTE* PointerPte
+)
+{
+    ULONG PageFileIndex = PointerPte->u.Soft.PageFileLow;
+    ULONG_PTR PageFileOffset = PointerPte->u.Soft.PageFileHigh;
+    LARGE_INTEGER FileOffset;
+    IO_STATUS_BLOCK Iosb;
+    NTSTATUS Status;
+    KEVENT Event;
+    UCHAR MdlBase[sizeof(MDL) + sizeof(PFN_NUMBER)];
+    PMDL Mdl = (PMDL) MdlBase;
+    PPAGINGFILE PagingFile;
+
+    /* The PTE must already be setup to point to a pagefile entry */
+    ASSERT(PointerPte->u.Hard.Valid == 0);
+    ASSERT(PointerPte->u.Soft.Transition == 0);
+    ASSERT(PointerPte->u.Soft.Prototype == 0);
+
+    DPRINT("MiWriteSwapFile\n");
+
+    if (PageFileOffset == 0)
+    {
+        KeBugCheck(MEMORY_MANAGEMENT);
+        return (STATUS_UNSUCCESSFUL);
+    }
+
+    /* Normalize it */
+    PageFileOffset--;
+
+    ASSERT(PageFileIndex < MAX_PAGING_FILES);
+
+    PagingFile = PagingFileList[PageFileIndex];
+
+    if (PagingFile->FileObject == NULL || PagingFile->FileObject->DeviceObject
== NULL)
+    {
+        DPRINT1("Bad paging file %u\n", PageFileIndex);
+        KeBugCheck(MEMORY_MANAGEMENT);
+    }
+
+    MmInitializeMdl(Mdl, NULL, PAGE_SIZE);
+    MmBuildMdlFromPages(Mdl, &Page);
+    Mdl->MdlFlags |= MDL_PAGES_LOCKED;
+
+    FileOffset.QuadPart = PageFileOffset * PAGE_SIZE;
+    FileOffset = MmGetOffsetPageFile(PagingFile->RetrievalPointers, FileOffset);
+
+    KeInitializeEvent(&Event, NotificationEvent, FALSE);
+    Status = IoSynchronousPageWrite(PagingFile->FileObject,
+                                    Mdl,
+                                    &FileOffset,
+                                    &Event,
+                                    &Iosb);
+    if (Status == STATUS_PENDING)
+    {
+        KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL);
+        Status = Iosb.Status;
+    }
+    if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA)
+    {
+        MmUnmapLockedPages(Mdl->MappedSystemVa, Mdl);
+    }
+    return Status;
+}
+
+VOID
+NTAPI
+MiFreePageFileEntry(
+    _In_ PMMPTE PointerPte
+)
+{
+    ULONG PageFileIndex;
+    ULONG_PTR PageFileOffset;
+    PPAGINGFILE PageFile;
+
+    /* Some sanity checks */
+    ASSERT(PointerPte->u.Hard.Valid == 0);
+    ASSERT(PointerPte->u.Soft.Transition == 0);
+    ASSERT(PointerPte->u.Soft.Prototype == 0);
+
+    /* We must hold the PFN lock */
+    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
+
+    PageFileIndex = PointerPte->u.Soft.PageFileLow;
+    PageFileOffset = PointerPte->u.Soft.PageFileHigh;
+
+    /* Fix up the value */
+    ASSERT(PageFileOffset != 0);
+    PageFileOffset--;
+
+    DPRINT1("Releasing page file entry %u, %lu\n", PageFileIndex,
PageFileOffset);
+
+    ASSERT(PageFileIndex < MAX_PAGING_FILES);
+
+    PageFile = PagingFileList[PageFileIndex];
+
+    /* And also lock this pagefile */
+    KeAcquireSpinLockAtDpcLevel(&PageFile->AllocMapLock);
+
+    /* It must be already in use */
+    ASSERT(RtlTestBit(&PageFile->AllocBitMap, PageFileOffset));
+    RtlClearBit(&PageFile->AllocBitMap, PageFileOffset);
+
+    /* One more */
+    PageFile->FreePages++;
+    PageFile->UsedPages--;
+
+    MiFreeSwapPages++;
+    MiUsedSwapPages--;
+
+    /* Done */
+    KeReleaseSpinLockFromDpcLevel(&PageFile->AllocMapLock);
+}
+
+NTSTATUS
+NTAPI
+MiReservePageFileEntry(
+    _Out_ PMMPTE PointerPte
+)
+{
+    ULONG PageFileIndex;
+    ULONG_PTR PageFileOffset;
+    PPAGINGFILE PageFile;
+
+    /* We must hold the PFN lock */
+    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
+
+    for (PageFileIndex = 0; PageFileIndex < MAX_PAGING_FILES; PageFileIndex++)
+    {
+        if (PagingFileList[PageFileIndex] == NULL)
+            continue;
+        if (PagingFileList[PageFileIndex]->FreePages != 0)
+            break;
+    }
+
+    if (PageFileIndex == MAX_PAGING_FILES)
+    {
+        DPRINT1("Pagefiles are full!\n");
+        return STATUS_PAGEFILE_QUOTA;
+    }
+
+    PointerPte->u.Long = 0;
+
+    PageFile = PagingFileList[PageFileIndex];
+
+    /* And also lock this page file */
+    KeAcquireSpinLockAtDpcLevel(&PageFile->AllocMapLock);
+
+    /* Find a free entry in this page file */
+    PageFileOffset = RtlFindClearBitsAndSet(&PageFile->AllocBitMap, 1,
PageFile->AllocBitMapHint);
+
+    /* We must have found something as the file was marked as having free entries */
+    ASSERT(PageFileOffset != ~((ULONG_PTR)0));
+    PageFile->AllocBitMapHint = PageFileOffset & ~((PageFile->MaximumSize /
PAGE_SIZE) - 1);
+
+    DPRINT1("Reserving page file entry %lu, %Iu\n", PageFileIndex,
PageFileOffset);
+
+    /* Done */
+    KeReleaseSpinLockFromDpcLevel(&PageFile->AllocMapLock);
+
+    /* We rely on PageFileHigh being != 0 */
+    PointerPte->u.Soft.PageFileHigh = PageFileOffset + 1;
+    PointerPte->u.Soft.PageFileLow = PageFileIndex;
+
+    return STATUS_SUCCESS;
+}
+
 VOID
 INIT_FUNCTION
 NTAPI
 MmInitPagingFile(VOID)
 {
    ULONG i;
-
-   KeInitializeSpinLock(&PagingFileListLock);
 
    MiFreeSwapPages = 0;
    MiUsedSwapPages = 0;
@@ -360,104 +491,41 @@
    MmNumberOfPagingFiles = 0;
 }
 
-static ULONG
-MiAllocPageFromPagingFile(PPAGINGFILE PagingFile)
-{
-   KIRQL oldIrql;
-   ULONG i, j;
-
-   KeAcquireSpinLock(&PagingFile->AllocMapLock, &oldIrql);
-
-   for (i = 0; i < PagingFile->AllocMapSize; i++)
-   {
-      for (j = 0; j < 32; j++)
-      {
-         if (!(PagingFile->AllocMap[i] & (1 << j)))
-         {
-            PagingFile->AllocMap[i] |= (1 << j);
-            PagingFile->UsedPages++;
-            PagingFile->FreePages--;
-            KeReleaseSpinLock(&PagingFile->AllocMapLock, oldIrql);
-            return((i * 32) + j);
-         }
-      }
-   }
-
-   KeReleaseSpinLock(&PagingFile->AllocMapLock, oldIrql);
-   return(0xFFFFFFFF);
-}
-
 VOID
 NTAPI
 MmFreeSwapPage(SWAPENTRY Entry)
 {
-   ULONG i;
-   ULONG_PTR off;
-   KIRQL oldIrql;
-
-   i = FILE_FROM_ENTRY(Entry);
-   off = OFFSET_FROM_ENTRY(Entry);
-
-   KeAcquireSpinLock(&PagingFileListLock, &oldIrql);
-   if (PagingFileList[i] == NULL)
-   {
-      KeBugCheck(MEMORY_MANAGEMENT);
-   }
-   KeAcquireSpinLockAtDpcLevel(&PagingFileList[i]->AllocMapLock);
-
-   PagingFileList[i]->AllocMap[off >> 5] &= (~(1 << (off % 32)));
-
-   PagingFileList[i]->FreePages++;
-   PagingFileList[i]->UsedPages--;
-
-   MiFreeSwapPages++;
-   MiUsedSwapPages--;
-
-   KeReleaseSpinLockFromDpcLevel(&PagingFileList[i]->AllocMapLock);
-   KeReleaseSpinLock(&PagingFileListLock, oldIrql);
+    MMPTE TempPte;
+    KIRQL OldIrql;
+
+    TempPte.u.Long = 0;
+    TempPte.u.Soft.PageFileLow = FILE_FROM_ENTRY(Entry);
+    TempPte.u.Soft.PageFileHigh = OFFSET_FROM_ENTRY(Entry);
+
+    OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+    MiFreePageFileEntry(&TempPte);
+    KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 }
 
 SWAPENTRY
 NTAPI
 MmAllocSwapPage(VOID)
 {
-   KIRQL oldIrql;
-   ULONG i;
-   ULONG off;
-   SWAPENTRY entry;
-
-   KeAcquireSpinLock(&PagingFileListLock, &oldIrql);
-
-   if (MiFreeSwapPages == 0)
-   {
-      KeReleaseSpinLock(&PagingFileListLock, oldIrql);
-      return(0);
-   }
-
-   for (i = 0; i < MAX_PAGING_FILES; i++)
-   {
-      if (PagingFileList[i] != NULL &&
-            PagingFileList[i]->FreePages >= 1)
-      {
-         off = MiAllocPageFromPagingFile(PagingFileList[i]);
-         if (off == 0xFFFFFFFF)
-         {
-            KeBugCheck(MEMORY_MANAGEMENT);
-            KeReleaseSpinLock(&PagingFileListLock, oldIrql);
-            return(STATUS_UNSUCCESSFUL);
-         }
-         MiUsedSwapPages++;
-         MiFreeSwapPages--;
-         KeReleaseSpinLock(&PagingFileListLock, oldIrql);
-
-         entry = ENTRY_FROM_FILE_OFFSET(i, off);
-         return(entry);
-      }
-   }
-
-   KeReleaseSpinLock(&PagingFileListLock, oldIrql);
-   KeBugCheck(MEMORY_MANAGEMENT);
-   return(0);
+    MMPTE TempPte;
+    SWAPENTRY entry;
+    NTSTATUS Status;
+    KIRQL OldIrql;
+
+    OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+    Status = MiReservePageFileEntry(&TempPte);
+    KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+
+    if (!NT_SUCCESS(Status))
+        return 0;
+
+    entry = ENTRY_FROM_FILE_OFFSET(TempPte.u.Soft.PageFileLow,
TempPte.u.Soft.PageFileHigh);
+
+    return entry;
 }
 
 static PRETRIEVEL_DESCRIPTOR_LIST FASTCALL
@@ -488,7 +556,7 @@
    IO_STATUS_BLOCK IoStatus;
    PFILE_OBJECT FileObject;
    PPAGINGFILE PagingFile;
-   KIRQL oldIrql;
+   KIRQL OldIrql;
    ULONG AllocMapSize;
    FILE_FS_SIZE_INFORMATION FsSizeInformation;
    PRETRIEVEL_DESCRIPTOR_LIST RetDescList;
@@ -646,7 +714,6 @@
 #if defined(__GNUC__)
    Vcn.QuadPart = 0LL;
 #else
-
    Vcn.QuadPart = 0;
 #endif
 
@@ -718,16 +785,15 @@
    RtlZeroMemory(PagingFile, sizeof(*PagingFile));
 
    PagingFile->FileObject = FileObject;
-   PagingFile->MaximumSize.QuadPart = SafeMaximumSize.QuadPart;
-   PagingFile->CurrentSize.QuadPart = SafeInitialSize.QuadPart;
-   PagingFile->FreePages = (ULONG)(SafeInitialSize.QuadPart / PAGE_SIZE);
+   PagingFile->MaximumSize = SafeMaximumSize.LowPart;
+   PagingFile->CurrentSize = SafeInitialSize.LowPart;
+   PagingFile->FreePages = (ULONG)(SafeMaximumSize.LowPart / PAGE_SIZE);
    PagingFile->UsedPages = 0;
    KeInitializeSpinLock(&PagingFile->AllocMapLock);
 
    AllocMapSize = (PagingFile->FreePages / 32) + 1;
    PagingFile->AllocMap = ExAllocatePool(NonPagedPool,
                                          AllocMapSize * sizeof(ULONG));
-   PagingFile->AllocMapSize = AllocMapSize;
 
    if (PagingFile->AllocMap == NULL)
    {
@@ -760,8 +826,10 @@
       return(STATUS_NO_MEMORY);
    }
 
-   RtlZeroMemory(PagingFile->AllocMap, AllocMapSize * sizeof(ULONG));
    RtlZeroMemory(PagingFile->RetrievalPointers, Size);
+
+   RtlInitializeBitMap(&PagingFile->AllocBitMap, PagingFile->AllocMap,
AllocMapSize * 32);
+   RtlClearAllBits(&PagingFile->AllocBitMap);
 
    Count = 0;
    PagingFile->RetrievalPointers->ExtentCount = ExtentCount;
@@ -799,7 +867,7 @@
       PagingFile->RetrievalPointers->Extents[i].NextVcn.QuadPart *=
BytesPerAllocationUnit;
    }
 
-   KeAcquireSpinLock(&PagingFileListLock, &oldIrql);
+   OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
    for (i = 0; i < MAX_PAGING_FILES; i++)
    {
       if (PagingFileList[i] == NULL)
@@ -810,7 +878,7 @@
    }
    MiFreeSwapPages = MiFreeSwapPages + PagingFile->FreePages;
    MmNumberOfPagingFiles++;
-   KeReleaseSpinLock(&PagingFileListLock, oldIrql);
+   KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 
    ZwClose(FileHandle);