[cgutman] 54544: [NTOSKRNL] - Many balancer fixes for concurrency, improved swapping efficiency, and reduced code duplication - Move the low memory case back under the PFN lock - Debugging is on fo...

Author: cgutman
Date: Tue Nov 29 23:59:25 2011
New Revision: 54544

URL: http://svn.reactos.org/svn/reactos?rev=54544&view=rev
Log:
[NTOSKRNL]
- Many balancer fixes for concurrency, improved swapping efficiency, and reduced code
duplication
- Move the low memory case back under the PFN lock
- Debugging is on for now because I don't trust the paging code all that much (it was
not used very much until recently)

Modified:
    trunk/reactos/ntoskrnl/mm/ARM3/pfnlist.c
    trunk/reactos/ntoskrnl/mm/balance.c

Modified: trunk/reactos/ntoskrnl/mm/ARM3/pfnlist.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/ARM3/pfnlist.c…
==============================================================================
--- trunk/reactos/ntoskrnl/mm/ARM3/pfnlist.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/mm/ARM3/pfnlist.c [iso-8859-1] Tue Nov 29 23:59:25 2011
@@ -848,7 +848,10 @@
 
     /* Make an empty software PTE */
     MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE);
-    
+
+    /* Lock the PFN database */
+    OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+
     /* Check if we're running low on pages */
     if (MmAvailablePages < 128)
     {
@@ -858,12 +861,7 @@
 
         /* Call RosMm and see if it can release any pages for us */
         MmRebalanceMemoryConsumers();
-
-        DPRINT1("Rebalance complete: %d pages left\n", MmAvailablePages);
-    }
-
-    /* Lock the PFN database */
-    OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+    }
 
     /* Grab a page */
     ASSERT_LIST_INVARIANT(&MmFreePageListHead);

Modified: trunk/reactos/ntoskrnl/mm/balance.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/balance.c?rev=…
==============================================================================
--- trunk/reactos/ntoskrnl/mm/balance.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/mm/balance.c [iso-8859-1] Tue Nov 29 23:59:25 2011
@@ -5,6 +5,7 @@
  * PURPOSE:         kernel memory managment functions
  *
  * PROGRAMMERS:     David Welch (welch(a)cwcom.net)
+ *                  Cameron Gutman (cameron.gutman(a)reactos.org)
  */
 
 /* INCLUDES *****************************************************************/
@@ -28,7 +29,6 @@
    KEVENT Event;
 }
 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
-
 /* GLOBALS ******************************************************************/
 
 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
@@ -36,14 +36,12 @@
 static ULONG MiNrTotalPages;
 static LIST_ENTRY AllocationListHead;
 static KSPIN_LOCK AllocationListLock;
-static ULONG MiPagesRequired = 0;
-static ULONG MiMinimumPagesPerRun = 10;
+static ULONG MiMinimumPagesPerRun;
 
 static CLIENT_ID MiBalancerThreadId;
 static HANDLE MiBalancerThreadHandle = NULL;
 static KEVENT MiBalancerEvent;
 static KTIMER MiBalancerTimer;
-static LONG MiBalancerWork = 0;
 
 /* FUNCTIONS ****************************************************************/
 
@@ -59,7 +57,8 @@
    MiNrTotalPages = NrAvailablePages;
 
    /* Set up targets. */
-   MiMinimumAvailablePages = 64;
+   MiMinimumAvailablePages = 128;
+   MiMinimumPagesPerRun = 256;
     if ((NrAvailablePages + NrSystemPages) >= 8192)
     {
         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
@@ -105,24 +104,20 @@
       KeBugCheck(MEMORY_MANAGEMENT);
    }
 
-   KeAcquireSpinLock(&AllocationListLock, &OldIrql);
    if (MmGetReferenceCountPage(Page) == 1)
    {
+      if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
       (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
-      if (IsListEmpty(&AllocationListHead) || MmAvailablePages <
MiMinimumAvailablePages)
-      {
-         KeReleaseSpinLock(&AllocationListLock, OldIrql);
-         if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
+      if (MmAvailablePages < MiMinimumAvailablePages ||
+	     (Entry = ExInterlockedRemoveHeadList(&AllocationListHead,
&AllocationListLock)) == NULL)
+      {
          OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
          MmDereferencePage(Page);
          KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
       }
       else
       {
-         Entry = RemoveHeadList(&AllocationListHead);
          Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
-         KeReleaseSpinLock(&AllocationListLock, OldIrql);
-         if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
          MiZeroPhysicalPage(Page);
          Request->Page = Page;
          KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
@@ -130,7 +125,6 @@
    }
    else
    {
-      KeReleaseSpinLock(&AllocationListLock, OldIrql);
       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
       MmDereferencePage(Page);
       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
@@ -143,17 +137,44 @@
 NTAPI
 MiTrimMemoryConsumer(ULONG Consumer)
 {
-   LONG Target;
-   ULONG NrFreedPages;
-
-   Target = max(MiMinimumPagesPerRun,
-                MiMemoryConsumers[Consumer].PagesUsed -
-                MiMemoryConsumers[Consumer].PagesTarget);
-
-   if (MiMemoryConsumers[Consumer].Trim != NULL)
-   {
-      MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
-   }
+    LONG Target = 0;
+    ULONG NrFreedPages = 0;
+    NTSTATUS Status;
+
+    /* Make sure we can trim this consumer */
+    if (!MiMemoryConsumers[Consumer].Trim)
+        return;
+
+    if (MiMemoryConsumers[Consumer].PagesUsed >
MiMemoryConsumers[Consumer].PagesTarget)
+    {
+        /* Consumer page limit exceeded */
+        Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed -
MiMemoryConsumers[Consumer].PagesTarget);
+    }
+    if (MmAvailablePages < MiMinimumAvailablePages)
+    {
+        /* Global page limit exceeded */
+        Target = max(Target, MiMinimumAvailablePages - MmAvailablePages);
+    }
+
+    if (Target)
+    {
+        /* Swap at least MiMinimumPagesPerRun */
+        Target = max(Target, MiMinimumPagesPerRun);
+
+        /* Now swap the pages out */
+        Status = MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
+
+        if (!ExpInTextModeSetup)
+            DPRINT1("Trimming consumer %d: Freed %d pages with a target of %d
pages\n", Consumer, NrFreedPages, Target);
+			
+		if (NrFreedPages == 0)
+		    DPRINT1("Ran out of pages to swap! Complete memory exhaustion is
imminent!\n");
+
+        if (!NT_SUCCESS(Status))
+        {
+            KeBugCheck(MEMORY_MANAGEMENT);
+        }
+    }
 }
 
 NTSTATUS
@@ -188,32 +209,6 @@
     return STATUS_SUCCESS;
 }
 
-VOID
-NTAPI
-MmRebalanceMemoryConsumers(VOID)
-{
-   LONG Target;
-   ULONG i;
-   ULONG NrFreedPages;
-   NTSTATUS Status;
-
-   Target = (ULONG)(MiMinimumAvailablePages - MmAvailablePages) + MiPagesRequired;
-   Target = max(Target, (LONG) MiMinimumPagesPerRun);
-
-   for (i = 0; i < MC_MAXIMUM && Target > 0; i++)
-   {
-      if (MiMemoryConsumers[i].Trim != NULL)
-      {
-         Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
-         if (!NT_SUCCESS(Status))
-         {
-            KeBugCheck(MEMORY_MANAGEMENT);
-         }
-         Target = Target - NrFreedPages;
-      }
-   }
-}
-
 static BOOLEAN
 MiIsBalancerThread(VOID)
 {
@@ -221,28 +216,34 @@
           (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
 }
 
+VOID
+NTAPI
+MmRebalanceMemoryConsumers(VOID)
+{
+    if (MiBalancerThreadHandle != NULL &&
+        !MiIsBalancerThread())
+    {
+        KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
+    }
+}
+
 NTSTATUS
 NTAPI
 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
                             PPFN_NUMBER AllocatedPage)
 {
-   ULONG OldUsed;
+   ULONG PagesUsed;
    PFN_NUMBER Page;
    KIRQL OldIrql;
 
    /*
     * Make sure we don't exceed our individual target.
     */
-   OldUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
-   if (OldUsed >= (MiMemoryConsumers[Consumer].PagesTarget - 1) &&
-         !MiIsBalancerThread())
-   {
-      if (!CanWait)
-      {
-         (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
-         return(STATUS_NO_MEMORY);
-      }
-      MiTrimMemoryConsumer(Consumer);
+   PagesUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed) + 1;
+   if (PagesUsed > MiMemoryConsumers[Consumer].PagesTarget &&
+       !MiIsBalancerThread())
+   {
+      MmRebalanceMemoryConsumers();
    }
 
    /*
@@ -259,19 +260,15 @@
       }
       if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
       *AllocatedPage = Page;
-      if (MmAvailablePages <= MiMinimumAvailablePages &&
-          MiBalancerThreadHandle != NULL &&
-          !MiIsBalancerThread())
-      {
-         KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
-      }
+      if (MmAvailablePages < MiMinimumAvailablePages)
+          MmRebalanceMemoryConsumers();
       return(STATUS_SUCCESS);
    }
 
    /*
     * Make sure we don't exceed global targets.
     */
-   if (MmAvailablePages <= MiMinimumAvailablePages)
+   if (MmAvailablePages < MiMinimumAvailablePages)
    {
       MM_ALLOCATION_REQUEST Request;
 
@@ -283,18 +280,10 @@
 
       /* Insert an allocation request. */
       Request.Page = 0;
-
       KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
-      (void)InterlockedIncrementUL(&MiPagesRequired);
-
-      KeAcquireSpinLock(&AllocationListLock, &OldIrql);
-
-      if (MiBalancerThreadHandle != NULL)
-      {
-         KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
-      }
-      InsertTailList(&AllocationListHead, &Request.ListEntry);
-      KeReleaseSpinLock(&AllocationListLock, OldIrql);
+
+      ExInterlockedInsertTailList(&AllocationListHead, &Request.ListEntry,
&AllocationListLock);
+      MmRebalanceMemoryConsumers();
 
       KeWaitForSingleObject(&Request.Event,
                             0,
@@ -310,13 +299,10 @@
 
       if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
       *AllocatedPage = Page;
-      (void)InterlockedDecrementUL(&MiPagesRequired);
-       
-      if (MmAvailablePages <= MiMinimumAvailablePages &&
-          MiBalancerThreadHandle != NULL &&
-          !MiIsBalancerThread())
-      {
-          KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
+
+      if (MmAvailablePages < MiMinimumAvailablePages)
+      {
+          MmRebalanceMemoryConsumers();
       }
 
       return(STATUS_SUCCESS);
@@ -334,12 +320,10 @@
    }
    if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
    *AllocatedPage = Page;
-    
-   if (MmAvailablePages <= MiMinimumAvailablePages &&
-       MiBalancerThreadHandle != NULL &&
-       !MiIsBalancerThread())
-   {
-       KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
+
+   if (MmAvailablePages < MiMinimumAvailablePages)
+   {
+       MmRebalanceMemoryConsumers();
    }
 
    return(STATUS_SUCCESS);
@@ -351,11 +335,6 @@
    PVOID WaitObjects[2];
    NTSTATUS Status;
    ULONG i;
-   ULONG NrFreedPages;
-   ULONG NrPagesUsed;
-   ULONG Target;
-   BOOLEAN ShouldRun;
-
 
    WaitObjects[0] = &MiBalancerEvent;
    WaitObjects[1] = &MiBalancerTimer;
@@ -371,55 +350,12 @@
                                         NULL,
                                         NULL);
 
-      if (Status == STATUS_SUCCESS)
-      {
-         /* MiBalancerEvent */
-         while (MmAvailablePages < MiMinimumAvailablePages + 5)
-         {
-            for (i = 0; i < MC_MAXIMUM; i++)
-            {
-               if (MiMemoryConsumers[i].Trim != NULL)
-               {
-                  NrFreedPages = 0;
-                  Status = MiMemoryConsumers[i].Trim(MiMinimumPagesPerRun, 0,
&NrFreedPages);
-                  if (!NT_SUCCESS(Status))
-                  {
-                     KeBugCheck(MEMORY_MANAGEMENT);
-                  }
-               }
-            }
-         }
-         InterlockedExchange(&MiBalancerWork, 0);
-      }
-      else if (Status == STATUS_SUCCESS + 1)
-      {
-         /* MiBalancerTimer */
-         ShouldRun = MmAvailablePages < MiMinimumAvailablePages + 5 ? TRUE : FALSE;
-         for (i = 0; i < MC_MAXIMUM; i++)
-         {
-            if (MiMemoryConsumers[i].Trim != NULL)
-            {
-               NrPagesUsed = MiMemoryConsumers[i].PagesUsed;
-               if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget || ShouldRun)
-               {
-                  if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget)
-                  {
-                     Target = max (NrPagesUsed - MiMemoryConsumers[i].PagesTarget,
-                                   MiMinimumPagesPerRun);
-                  }
-                  else
-                  {
-                     Target = MiMinimumPagesPerRun;
-                  }
-                  NrFreedPages = 0;
-                  Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
-                  if (!NT_SUCCESS(Status))
-                  {
-                     KeBugCheck(MEMORY_MANAGEMENT);
-                  }
-               }
-            }
-         }
+      if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
+      {
+          for (i = 0; i < MC_MAXIMUM; i++)
+          {
+              MiTrimMemoryConsumer(i);
+          }
       }
       else
       {