[ros-diffs] [reactos] 83/100: [NTOS:MM] Make the page LRU list a real LRU list.

[Jérôme Gardou]

https://git.reactos.org/?p=reactos.git;a=commitdiff;h=8a8b4db447c36232addbe…

commit 8a8b4db447c36232addbe4693953182d355c0408
Author:     Jérôme Gardou <jerome.gardou(a)reactos.org>
AuthorDate: Wed Jan 27 10:29:07 2021 +0100
Commit:     Jérôme Gardou <jerome.gardou(a)reactos.org>
CommitDate: Wed Feb 3 09:41:23 2021 +0100

    [NTOS:MM] Make the page LRU list a real LRU list.
    
    Also, implement flushing mapped sections to disk on shutdown.
---
 ntoskrnl/cache/section/newmm.h |  57 --------------
 ntoskrnl/include/internal/mm.h |  69 ++++++++++++++---
 ntoskrnl/mm/balance.c          |  27 +++----
 ntoskrnl/mm/freelist.c         | 164 ++++++++++++++++++++++++++---------------
 ntoskrnl/mm/mminit.c           |   3 -
 ntoskrnl/mm/shutdown.c         |  38 ++++++++++
 6 files changed, 214 insertions(+), 144 deletions(-)

diff --git a/ntoskrnl/cache/section/newmm.h b/ntoskrnl/cache/section/newmm.h
index edff99f3d85..3fb731c6ace 100644
--- a/ntoskrnl/cache/section/newmm.h
+++ b/ntoskrnl/cache/section/newmm.h
@@ -3,22 +3,6 @@
 #include <internal/arch/mm.h>
 
 /* TYPES *********************************************************************/
-
-#define PFN_FROM_SSE(E)          ((PFN_NUMBER)((E) >> PAGE_SHIFT))
-#define IS_SWAP_FROM_SSE(E)      ((E) & 0x00000001)
-#define MM_IS_WAIT_PTE(E)        \
-    (IS_SWAP_FROM_SSE(E) && SWAPENTRY_FROM_SSE(E) == MM_WAIT_ENTRY)
-#define MAKE_PFN_SSE(P)          ((ULONG_PTR)((P) << PAGE_SHIFT))
-#define SWAPENTRY_FROM_SSE(E)    ((E) >> 1)
-#define MAKE_SWAP_SSE(S)         (((ULONG_PTR)(S) << 1) | 0x1)
-#define DIRTY_SSE(E)             ((E) | 2)
-#define CLEAN_SSE(E)             ((E) & ~2)
-#define IS_DIRTY_SSE(E)          ((E) & 2)
-#define PAGE_FROM_SSE(E)         ((E) & 0xFFFFF000)
-#define SHARE_COUNT_FROM_SSE(E)  (((E) & 0x00000FFC) >> 2)
-#define MAX_SHARE_COUNT          0x3FF
-#define MAKE_SSE(P, C)           ((ULONG_PTR)((P) | ((C) << 2)))
-
 #define MM_SEGMENT_FINALIZE (0x40000000)
 
 #define RMAP_SEGMENT_MASK ~((ULONG_PTR)0xff)
@@ -123,25 +107,6 @@ VOID
 NTAPI
 MiInitializeSectionPageTable(PMM_SECTION_SEGMENT Segment);
 
-NTSTATUS
-NTAPI
-_MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment,
-                              PLARGE_INTEGER Offset,
-                              ULONG_PTR Entry,
-                              const char *file,
-                              int line);
-
-ULONG_PTR
-NTAPI
-_MmGetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment,
-                              PLARGE_INTEGER Offset,
-                              const char *file,
-                              int line);
-
-#define MmSetPageEntrySectionSegment(S,O,E)
_MmSetPageEntrySectionSegment(S,O,E,__FILE__,__LINE__)
-
-#define MmGetPageEntrySectionSegment(S,O)
_MmGetPageEntrySectionSegment(S,O,__FILE__,__LINE__)
-
 typedef VOID (NTAPI *FREE_SECTION_PAGE_FUN)(
     PMM_SECTION_SEGMENT Segment,
     PLARGE_INTEGER Offset);
@@ -151,12 +116,6 @@ NTAPI
 MmFreePageTablesSectionSegment(PMM_SECTION_SEGMENT Segment,
                                FREE_SECTION_PAGE_FUN FreePage);
 
-/* Yields a lock */
-PMM_SECTION_SEGMENT
-NTAPI
-MmGetSectionAssociation(PFN_NUMBER Page,
-                        PLARGE_INTEGER Offset);
-
 NTSTATUS
 NTAPI
 MmSetSectionAssociation(PFN_NUMBER Page,
@@ -267,22 +226,6 @@ MmPageOutDeleteMapping(PVOID Context,
                        PEPROCESS Process,
                        PVOID Address);
 
-VOID
-NTAPI
-_MmLockSectionSegment(PMM_SECTION_SEGMENT Segment,
-                      const char *file,
-                      int line);
-
-#define MmLockSectionSegment(x) _MmLockSectionSegment(x,__FILE__,__LINE__)
-
-VOID
-NTAPI
-_MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment,
-                        const char *file,
-                        int line);
-
-#define MmUnlockSectionSegment(x) _MmUnlockSectionSegment(x,__FILE__,__LINE__)
-
 VOID
 MmFreeCacheSectionPage(PVOID Context,
                        MEMORY_AREA* MemoryArea,
diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h
index 4db21f72199..bd9ea25e0b7 100644
--- a/ntoskrnl/include/internal/mm.h
+++ b/ntoskrnl/include/internal/mm.h
@@ -357,6 +357,8 @@ typedef struct _MMPFN
 
     // HACK until WS lists are supported
     MMWSLE Wsle;
+    struct _MMPFN* NextLRU;
+    struct _MMPFN* PreviousLRU;
 } MMPFN, *PMMPFN;
 
 extern PMMPFN MmPfnDatabase;
@@ -877,6 +879,11 @@ NTSTATUS
 NTAPI
 MmPageOutPhysicalAddress(PFN_NUMBER Page);
 
+PMM_SECTION_SEGMENT
+NTAPI
+MmGetSectionAssociation(PFN_NUMBER Page,
+                        PLARGE_INTEGER Offset);
+
 /* freelist.c **********************************************************/
 
 FORCEINLINE
@@ -950,20 +957,12 @@ MiGetPfnEntryIndex(IN PMMPFN Pfn1)
 
 PFN_NUMBER
 NTAPI
-MmGetLRUNextUserPage(PFN_NUMBER PreviousPage);
+MmGetLRUNextUserPage(PFN_NUMBER PreviousPage, BOOLEAN MoveToLast);
 
 PFN_NUMBER
 NTAPI
 MmGetLRUFirstUserPage(VOID);
 
-VOID
-NTAPI
-MmInsertLRULastUserPage(PFN_NUMBER Page);
-
-VOID
-NTAPI
-MmRemoveLRUUserPage(PFN_NUMBER Page);
-
 VOID
 NTAPI
 MmDumpArmPfnDatabase(
@@ -1232,6 +1231,37 @@ MmFindRegion(
 
 /* section.c *****************************************************************/
 
+#define PFN_FROM_SSE(E)          ((PFN_NUMBER)((E) >> PAGE_SHIFT))
+#define IS_SWAP_FROM_SSE(E)      ((E) & 0x00000001)
+#define MM_IS_WAIT_PTE(E)        \
+    (IS_SWAP_FROM_SSE(E) && SWAPENTRY_FROM_SSE(E) == MM_WAIT_ENTRY)
+#define MAKE_PFN_SSE(P)          ((ULONG_PTR)((P) << PAGE_SHIFT))
+#define SWAPENTRY_FROM_SSE(E)    ((E) >> 1)
+#define MAKE_SWAP_SSE(S)         (((ULONG_PTR)(S) << 1) | 0x1)
+#define DIRTY_SSE(E)             ((E) | 2)
+#define CLEAN_SSE(E)             ((E) & ~2)
+#define IS_DIRTY_SSE(E)          ((E) & 2)
+#define PAGE_FROM_SSE(E)         ((E) & 0xFFFFF000)
+#define SHARE_COUNT_FROM_SSE(E)  (((E) & 0x00000FFC) >> 2)
+#define MAX_SHARE_COUNT          0x3FF
+#define MAKE_SSE(P, C)           ((ULONG_PTR)((P) | ((C) << 2)))
+
+VOID
+NTAPI
+_MmLockSectionSegment(PMM_SECTION_SEGMENT Segment,
+                      const char *file,
+                      int line);
+
+#define MmLockSectionSegment(x) _MmLockSectionSegment(x,__FILE__,__LINE__)
+
+VOID
+NTAPI
+_MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment,
+                        const char *file,
+                        int line);
+
+#define MmUnlockSectionSegment(x) _MmUnlockSectionSegment(x,__FILE__,__LINE__)
+
 VOID
 NTAPI
 MmGetImageInformation(
@@ -1372,6 +1402,27 @@ MmExtendSection(
     _In_ PVOID Section,
     _Inout_ PLARGE_INTEGER NewSize);
 
+/* sptab.c *******************************************************************/
+
+NTSTATUS
+NTAPI
+_MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment,
+                              PLARGE_INTEGER Offset,
+                              ULONG_PTR Entry,
+                              const char *file,
+                              int line);
+
+ULONG_PTR
+NTAPI
+_MmGetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment,
+                              PLARGE_INTEGER Offset,
+                              const char *file,
+                              int line);
+
+#define MmSetPageEntrySectionSegment(S,O,E)
_MmSetPageEntrySectionSegment(S,O,E,__FILE__,__LINE__)
+
+#define MmGetPageEntrySectionSegment(S,O)
_MmGetPageEntrySectionSegment(S,O,__FILE__,__LINE__)
+
 /* sysldr.c ******************************************************************/
 
 VOID
diff --git a/ntoskrnl/mm/balance.c b/ntoskrnl/mm/balance.c
index 34dddf5dbfa..f9067c58a13 100644
--- a/ntoskrnl/mm/balance.c
+++ b/ntoskrnl/mm/balance.c
@@ -80,11 +80,7 @@ MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
         KeBugCheck(MEMORY_MANAGEMENT);
     }
 
-    if (MmGetReferenceCountPage(Page) == 1)
-    {
-        if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
-        (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
-    }
+    (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 
     MmDereferencePage(Page);
 
@@ -142,7 +138,6 @@ NTSTATUS
 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
 {
     PFN_NUMBER CurrentPage;
-    PFN_NUMBER NextPage;
     NTSTATUS Status;
 
     (*NrFreedPages) = 0;
@@ -158,13 +153,14 @@ MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
             (*NrFreedPages)++;
         }
 
-        NextPage = MmGetLRUNextUserPage(CurrentPage);
-        if (NextPage <= CurrentPage)
-        {
-            /* We wrapped around, so we're done */
-            break;
-        }
-        CurrentPage = NextPage;
+        CurrentPage = MmGetLRUNextUserPage(CurrentPage, TRUE);
+    }
+
+    if (CurrentPage)
+    {
+        KIRQL OldIrql = MiAcquirePfnLock();
+        MmDereferencePage(CurrentPage);
+        MiReleasePfnLock(OldIrql);
     }
 
     return STATUS_SUCCESS;
@@ -209,14 +205,13 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
     /*
      * Allocate always memory for the non paged pool and for the pager thread.
      */
-    if ((Consumer == MC_SYSTEM) /* || MiIsBalancerThread() */)
+    if (Consumer == MC_SYSTEM)
     {
         Page = MmAllocPage(Consumer);
         if (Page == 0)
         {
             KeBugCheck(NO_PAGES_AVAILABLE);
         }
-        if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
         *AllocatedPage = Page;
         if (MmAvailablePages < MiMinimumAvailablePages)
             MmRebalanceMemoryConsumers();
@@ -257,7 +252,6 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
             KeBugCheck(NO_PAGES_AVAILABLE);
         }
 
-        if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
         *AllocatedPage = Page;
 
         if (MmAvailablePages < MiMinimumAvailablePages)
@@ -276,7 +270,6 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
     {
         KeBugCheck(NO_PAGES_AVAILABLE);
     }
-    if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
     *AllocatedPage = Page;
 
     if (MmAvailablePages < MiMinimumAvailablePages)
diff --git a/ntoskrnl/mm/freelist.c b/ntoskrnl/mm/freelist.c
index 5cc22ce38b1..35c6336dcc6 100644
--- a/ntoskrnl/mm/freelist.c
+++ b/ntoskrnl/mm/freelist.c
@@ -35,95 +35,127 @@ SIZE_T MmPagedPoolCommit;
 SIZE_T MmPeakCommitment;
 SIZE_T MmtotalCommitLimitMaximum;
 
-static RTL_BITMAP MiUserPfnBitMap;
+PMMPFN FirstUserLRUPfn;
+PMMPFN LastUserLRUPfn;
 
 /* FUNCTIONS *************************************************************/
 
-VOID
-NTAPI
-MiInitializeUserPfnBitmap(VOID)
-{
-    PVOID Bitmap;
-
-    /* Allocate enough buffer for the PFN bitmap and align it on 32-bits */
-    Bitmap = ExAllocatePoolWithTag(NonPagedPool,
-                                   (((MmHighestPhysicalPage + 1) + 31) / 32) * 4,
-                                   TAG_MM);
-    ASSERT(Bitmap);
-
-    /* Initialize it and clear all the bits to begin with */
-    RtlInitializeBitMap(&MiUserPfnBitMap,
-                        Bitmap,
-                        (ULONG)MmHighestPhysicalPage + 1);
-    RtlClearAllBits(&MiUserPfnBitMap);
-}
-
 PFN_NUMBER
 NTAPI
 MmGetLRUFirstUserPage(VOID)
 {
-    ULONG Position;
+    PFN_NUMBER Page;
     KIRQL OldIrql;
 
     /* Find the first user page */
     OldIrql = MiAcquirePfnLock();
-    Position = RtlFindSetBits(&MiUserPfnBitMap, 1, 0);
+
+    if (FirstUserLRUPfn == NULL)
+    {
+        MiReleasePfnLock(OldIrql);
+        return 0;
+    }
+
+    Page = MiGetPfnEntryIndex(FirstUserLRUPfn);
+    MmReferencePage(Page);
+
     MiReleasePfnLock(OldIrql);
-    if (Position == 0xFFFFFFFF) return 0;
 
-    /* Return it */
-    ASSERT(Position != 0);
-    ASSERT_IS_ROS_PFN(MiGetPfnEntry(Position));
-    return Position;
+    return Page;
 }
 
+static
 VOID
-NTAPI
-MmInsertLRULastUserPage(PFN_NUMBER Pfn)
+MmInsertLRULastUserPage(PFN_NUMBER Page)
 {
-    KIRQL OldIrql;
+    MI_ASSERT_PFN_LOCK_HELD();
 
-    /* Set the page as a user page */
-    ASSERT(Pfn != 0);
-    ASSERT_IS_ROS_PFN(MiGetPfnEntry(Pfn));
-    ASSERT(!RtlCheckBit(&MiUserPfnBitMap, (ULONG)Pfn));
-    OldIrql = MiAcquirePfnLock();
-    RtlSetBit(&MiUserPfnBitMap, (ULONG)Pfn);
-    MiReleasePfnLock(OldIrql);
-}
+    PMMPFN Pfn = MiGetPfnEntry(Page);
 
-PFN_NUMBER
-NTAPI
-MmGetLRUNextUserPage(PFN_NUMBER PreviousPfn)
-{
-    ULONG Position;
-    KIRQL OldIrql;
+    if (FirstUserLRUPfn == NULL)
+        FirstUserLRUPfn = Pfn;
 
-    /* Find the next user page */
-    OldIrql = MiAcquirePfnLock();
-    Position = RtlFindSetBits(&MiUserPfnBitMap, 1, (ULONG)PreviousPfn + 1);
-    MiReleasePfnLock(OldIrql);
-    if (Position == 0xFFFFFFFF) return 0;
+    Pfn->PreviousLRU = LastUserLRUPfn;
 
-    /* Return it */
-    ASSERT(Position != 0);
-    ASSERT_IS_ROS_PFN(MiGetPfnEntry(Position));
-    return Position;
+    if (LastUserLRUPfn != NULL)
+        LastUserLRUPfn->NextLRU = Pfn;
+    LastUserLRUPfn = Pfn;
 }
 
+static
 VOID
-NTAPI
 MmRemoveLRUUserPage(PFN_NUMBER Page)
 {
-    KIRQL OldIrql;
+    MI_ASSERT_PFN_LOCK_HELD();
 
     /* Unset the page as a user page */
     ASSERT(Page != 0);
-    ASSERT_IS_ROS_PFN(MiGetPfnEntry(Page));
-    ASSERT(RtlCheckBit(&MiUserPfnBitMap, (ULONG)Page));
+
+    PMMPFN Pfn = MiGetPfnEntry(Page);
+
+    ASSERT_IS_ROS_PFN(Pfn);
+
+    if (Pfn->PreviousLRU)
+    {
+        ASSERT(Pfn->PreviousLRU->NextLRU == Pfn);
+        Pfn->PreviousLRU->NextLRU = Pfn->NextLRU;
+    }
+    else
+    {
+        ASSERT(FirstUserLRUPfn == Pfn);
+        FirstUserLRUPfn = Pfn->NextLRU;
+    }
+
+    if (Pfn->NextLRU)
+    {
+        ASSERT(Pfn->NextLRU->PreviousLRU == Pfn);
+        Pfn->NextLRU->PreviousLRU = Pfn->PreviousLRU;
+    }
+    else
+    {
+        ASSERT(Pfn == LastUserLRUPfn);
+        LastUserLRUPfn = Pfn->PreviousLRU;
+    }
+
+    Pfn->PreviousLRU = Pfn->NextLRU = NULL;
+}
+
+PFN_NUMBER
+NTAPI
+MmGetLRUNextUserPage(PFN_NUMBER PreviousPage, BOOLEAN MoveToLast)
+{
+    PFN_NUMBER Page = 0;
+    KIRQL OldIrql;
+
+    /* Find the next user page */
     OldIrql = MiAcquirePfnLock();
-    RtlClearBit(&MiUserPfnBitMap, (ULONG)Page);
+
+    PMMPFN PreviousPfn = MiGetPfnEntry(PreviousPage);
+    PMMPFN NextPfn = PreviousPfn->NextLRU;
+
+    /*
+     * Move this one at the end of the list.
+     * It may be freed by MmDereferencePage below.
+     * If it's not, then it means it is still hanging in some process address space.
+     * This avoids paging-out e.g. ntdll early just because it's mapped first time.
+     */
+    if (MoveToLast)
+    {
+        MmRemoveLRUUserPage(PreviousPage);
+        MmInsertLRULastUserPage(PreviousPage);
+    }
+
+    if (NextPfn)
+    {
+        Page = MiGetPfnEntryIndex(NextPfn);
+        MmReferencePage(Page);
+    }
+
+    MmDereferencePage(PreviousPage);
+
     MiReleasePfnLock(OldIrql);
+
+    return Page;
 }
 
 BOOLEAN
@@ -548,6 +580,13 @@ MmDereferencePage(PFN_NUMBER Pfn)
     Pfn1->u3.e2.ReferenceCount--;
     if (Pfn1->u3.e2.ReferenceCount == 0)
     {
+        /* Apply LRU hack */
+        if (Pfn1->u4.MustBeCached)
+        {
+            MmRemoveLRUUserPage(Pfn);
+            Pfn1->u4.MustBeCached = 0;
+        }
+
         /* Mark the page temporarily as valid, we're going to make it free soon */
         Pfn1->u3.e1.PageLocation = ActiveAndValid;
 
@@ -590,6 +629,15 @@ MmAllocPage(ULONG Type)
     Pfn1->u1.SwapEntry = 0;
     Pfn1->RmapListHead = NULL;
 
+    Pfn1->NextLRU = NULL;
+    Pfn1->PreviousLRU = NULL;
+
+    if (Type == MC_USER)
+    {
+        Pfn1->u4.MustBeCached = 1; /* HACK again */
+        MmInsertLRULastUserPage(PfnOffset);
+    }
+
     MiReleasePfnLock(OldIrql);
     return PfnOffset;
 }
diff --git a/ntoskrnl/mm/mminit.c b/ntoskrnl/mm/mminit.c
index 6c1a5f274d6..3e58ad9bbc8 100644
--- a/ntoskrnl/mm/mminit.c
+++ b/ntoskrnl/mm/mminit.c
@@ -17,8 +17,6 @@
 
 /* GLOBALS *******************************************************************/
 
-VOID NTAPI MiInitializeUserPfnBitmap(VOID);
-
 BOOLEAN Mm64BitPhysicalAddress = FALSE;
 ULONG MmReadClusterSize;
 //
@@ -235,7 +233,6 @@ MmInitSystem(IN ULONG Phase,
     MiDbgDumpAddressSpace();
 
     MmInitGlobalKernelPageDirectory();
-    MiInitializeUserPfnBitmap();
     MmInitializeMemoryConsumer(MC_USER, MmTrimUserMemory);
     MmInitializeRmapList();
     MmInitSectionImplementation();
diff --git a/ntoskrnl/mm/shutdown.c b/ntoskrnl/mm/shutdown.c
index c80cc527a30..89beca88fac 100644
--- a/ntoskrnl/mm/shutdown.c
+++ b/ntoskrnl/mm/shutdown.c
@@ -21,6 +21,44 @@ VOID
 MiShutdownSystem(VOID)
 {
     ULONG i;
+    PFN_NUMBER Page;
+    BOOLEAN Dirty;
+
+    /* Loop through all the pages owned by the legacy Mm and page them out, if needed.
*/
+    /* We do it twice, since flushing can cause the FS to dirtify new pages */
+    do
+    {
+        Dirty = FALSE;
+
+        Page = MmGetLRUFirstUserPage();
+        while (Page)
+        {
+            LARGE_INTEGER SegmentOffset;
+            PMM_SECTION_SEGMENT Segment = MmGetSectionAssociation(Page,
&SegmentOffset);
+
+            if (Segment)
+            {
+                if ((*Segment->Flags) & MM_DATAFILE_SEGMENT)
+                {
+                    MmLockSectionSegment(Segment);
+
+                    ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment,
&SegmentOffset);
+
+                    if (!IS_SWAP_FROM_SSE(Entry) && IS_DIRTY_SSE(Entry))
+                    {
+                        Dirty = TRUE;
+                        MmCheckDirtySegment(Segment, &SegmentOffset, FALSE, TRUE);
+                    }
+
+                    MmUnlockSectionSegment(Segment);
+                }
+
+                MmDereferenceSegment(Segment);
+            }
+
+            Page = MmGetLRUNextUserPage(Page, FALSE);
+        }
+    } while (Dirty);
 
     /* Loop through all the paging files */
     for (i = 0; i < MmNumberOfPagingFiles; i++)