[arty] 56441: [NTOSKRNL] Remove PAGEOPs and use the argument to pass relevant information where needed, including generally a section page table entry or a PFN_NUMBER. In its place, the code looks ...

Author: arty
Date: Sat Apr 28 02:56:31 2012
New Revision: 56441

URL: http://svn.reactos.org/svn/reactos?rev=56441&view=rev
Log:
[NTOSKRNL]
Remove PAGEOPs and use the argument to pass relevant information where needed,
including generally a section page table entry or a PFN_NUMBER.

In its place, the code looks for an MM_WAIT_ENTRY in the appropriate place and
waits, returning STATUS_MM_RESTART_OPERATION.  Rather than copying the pagein
and CoW operations in the PAGEOP contention case, we just punt back and restart
the operation, resulting in simpler code.

Modified:
    trunk/reactos/ntoskrnl/CMakeLists.txt
    trunk/reactos/ntoskrnl/cache/section/sptab.c
    trunk/reactos/ntoskrnl/include/internal/mm.h
    trunk/reactos/ntoskrnl/mm/mminit.c
    trunk/reactos/ntoskrnl/mm/rmap.c
    trunk/reactos/ntoskrnl/mm/section.c

Modified: trunk/reactos/ntoskrnl/CMakeLists.txt
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/CMakeLists.txt?re…
==============================================================================
--- trunk/reactos/ntoskrnl/CMakeLists.txt [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/CMakeLists.txt [iso-8859-1] Sat Apr 28 02:56:31 2012
@@ -229,7 +229,6 @@
     mm/mmfault.c
     mm/mminit.c
     mm/pagefile.c
-    mm/pageop.c
     mm/region.c
     mm/rmap.c
     mm/section.c

Modified: trunk/reactos/ntoskrnl/cache/section/sptab.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/cache/section/spt…
==============================================================================
--- trunk/reactos/ntoskrnl/cache/section/sptab.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/cache/section/sptab.c [iso-8859-1] Sat Apr 28 02:56:31 2012
@@ -215,8 +215,9 @@
         ASSERT(!Entry || IS_SWAP_FROM_SSE(Entry));
         MmDeleteSectionAssociation(PFN_FROM_SSE(OldEntry));
     } else if (IS_SWAP_FROM_SSE(Entry)) {
-        ASSERT(!IS_SWAP_FROM_SSE(OldEntry));
-        if (OldEntry)
+        ASSERT(!IS_SWAP_FROM_SSE(OldEntry) || 
+               SWAPENTRY_FROM_SSE(OldEntry) == MM_WAIT_ENTRY);
+        if (OldEntry && SWAPENTRY_FROM_SSE(OldEntry) != MM_WAIT_ENTRY)
             MmDeleteSectionAssociation(PFN_FROM_SSE(OldEntry));
     } else if (IS_SWAP_FROM_SSE(OldEntry)) {
         ASSERT(!IS_SWAP_FROM_SSE(Entry));

Modified: trunk/reactos/ntoskrnl/include/internal/mm.h
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/include/internal/…
==============================================================================
--- trunk/reactos/ntoskrnl/include/internal/mm.h [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/include/internal/mm.h [iso-8859-1] Sat Apr 28 02:56:31 2012
@@ -49,7 +49,6 @@
 struct _KTRAP_FRAME;
 struct _EPROCESS;
 struct _MM_RMAP_ENTRY;
-struct _MM_PAGEOP;
 typedef ULONG_PTR SWAPENTRY;
 
 //
@@ -83,12 +82,6 @@
 #define MM_CORE_DUMP_TYPE_NONE              (0x0)
 #define MM_CORE_DUMP_TYPE_MINIMAL           (0x1)
 #define MM_CORE_DUMP_TYPE_FULL              (0x2)
-
-#define MM_PAGEOP_PAGEIN                    (1)
-#define MM_PAGEOP_PAGEOUT                   (2)
-#define MM_PAGEOP_PAGESYNCH                 (3)
-#define MM_PAGEOP_ACCESSFAULT               (4)
-#define MM_PAGEOP_CHANGEPROTECT             (5)
 
 /* Number of list heads to use */
 #define MI_FREE_POOL_LISTS 4
@@ -415,37 +408,6 @@
 extern MMPFNLIST MmModifiedPageListHead;
 extern MMPFNLIST MmModifiedNoWritePageListHead;
 
-typedef struct _MM_PAGEOP
-{
-  /* Type of operation. */
-  ULONG OpType;
-  /* Number of threads interested in this operation. */
-  ULONG ReferenceCount;
-  /* Event that will be set when the operation is completed. */
-  KEVENT CompletionEvent;
-  /* Status of the operation once it is completed. */
-  NTSTATUS Status;
-  /* TRUE if the operation was abandoned. */
-  BOOLEAN Abandoned;
-  /* The memory area to be affected by the operation. */
-  PMEMORY_AREA MArea;
-  ULONG Hash;
-  struct _MM_PAGEOP* Next;
-  struct _ETHREAD* Thread;
-  /*
-   * These fields are used to identify the operation if it is against a
-   * virtual memory area.
-   */
-  HANDLE Pid;
-  PVOID Address;
-  /*
-   * These fields are used to identify the operation if it is against a
-   * section mapping.
-   */
-  PMM_SECTION_SEGMENT Segment;
-  ULONGLONG Offset;
-} MM_PAGEOP, *PMM_PAGEOP;
-
 typedef struct _MM_MEMORY_CONSUMER
 {
     ULONG PagesUsed;
@@ -937,7 +899,7 @@
     PMMSUPPORT AddressSpace,
     PMEMORY_AREA MemoryArea,
     PVOID Address,
-    struct _MM_PAGEOP* PageOp
+    PFN_NUMBER Page
 );
 
 NTSTATUS
@@ -973,7 +935,7 @@
     PMMSUPPORT AddressSpace,
     PMEMORY_AREA MArea,
     PVOID Address,
-    PMM_PAGEOP PageOp
+    PFN_NUMBER Page
 );
 
 /* kmap.c ********************************************************************/
@@ -1006,38 +968,6 @@
 PVOID
 FASTCALL
 MmSafeReadPtr(PVOID Source);
-
-/* pageop.c ******************************************************************/
-
-VOID
-NTAPI
-MmReleasePageOp(PMM_PAGEOP PageOp);
-
-PMM_PAGEOP
-NTAPI
-MmGetPageOp(
-    PMEMORY_AREA MArea,
-    HANDLE Pid,
-    PVOID Address,
-    PMM_SECTION_SEGMENT Segment,
-    ULONGLONG Offset,
-    ULONG OpType,
-    BOOLEAN First
-);
-
-PMM_PAGEOP
-NTAPI
-MmCheckForPageOp(
-    PMEMORY_AREA MArea,
-    HANDLE Pid,
-    PVOID Address,
-    PMM_SECTION_SEGMENT Segment,
-    ULONGLONG Offset
-);
-
-VOID
-NTAPI
-MmInitializePageOp(VOID);
 
 /* process.c *****************************************************************/
 
@@ -1657,7 +1587,7 @@
     PMMSUPPORT AddressSpace,
     PMEMORY_AREA MemoryArea,
     PVOID Address,
-    struct _MM_PAGEOP *PageOp
+    ULONG_PTR Entry
 );
 
 NTSTATUS

Modified: trunk/reactos/ntoskrnl/mm/mminit.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/mminit.c?rev=5…
==============================================================================
--- trunk/reactos/ntoskrnl/mm/mminit.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/mm/mminit.c [iso-8859-1] Sat Apr 28 02:56:31 2012
@@ -419,7 +419,6 @@
     MiInitializeUserPfnBitmap();
     MmInitializeMemoryConsumer(MC_USER, MmTrimUserMemory);
     MmInitializeRmapList();
-    MmInitializePageOp();
     MmInitSectionImplementation();
     MmInitPagingFile();
 

Modified: trunk/reactos/ntoskrnl/mm/rmap.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/rmap.c?rev=564…
==============================================================================
--- trunk/reactos/ntoskrnl/mm/rmap.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/mm/rmap.c [iso-8859-1] Sat Apr 28 02:56:31 2012
@@ -52,7 +52,6 @@
    ULONG Type;
    PVOID Address;
    PEPROCESS Process;
-   PMM_PAGEOP PageOp;
    ULONGLONG Offset;
    NTSTATUS Status = STATUS_SUCCESS;
 
@@ -126,17 +125,20 @@
    Type = MemoryArea->Type;
    if (Type == MEMORY_AREA_SECTION_VIEW)
    {
+      ULONG Entry;
       Offset = MemoryArea->Data.SectionData.ViewOffset.QuadPart +
                ((ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress);
+
+      MmLockSectionSegment(MemoryArea->Data.SectionData.Segment);
 
       /*
        * Get or create a pageop
        */
-      PageOp = MmGetPageOp(MemoryArea, NULL, 0,
-                           MemoryArea->Data.SectionData.Segment,
-                           Offset, MM_PAGEOP_PAGEOUT, TRUE);
-      if (PageOp == NULL)
-      {
+      Entry = MmGetPageEntrySectionSegment
+          (MemoryArea->Data.SectionData.Segment, (PLARGE_INTEGER)&Offset);
+      if (Entry && IS_SWAP_FROM_SSE(Entry) && SWAPENTRY_FROM_SSE(Entry)
== MM_WAIT_ENTRY)
+      {
+         MmUnlockSectionSegment(MemoryArea->Data.SectionData.Segment);
          MmUnlockAddressSpace(AddressSpace);
          if (Address < MmSystemRangeStart)
          {
@@ -145,17 +147,19 @@
          }
          return(STATUS_UNSUCCESSFUL);
       }
+
+      MmSetPageEntrySectionSegment(MemoryArea->Data.SectionData.Segment,
(PLARGE_INTEGER)&Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY));
 
       /*
        * Release locks now we have a page op.
        */
+      MmUnlockSectionSegment(MemoryArea->Data.SectionData.Segment);
       MmUnlockAddressSpace(AddressSpace);
 
       /*
        * Do the actual page out work.
        */
-      Status = MmPageOutSectionView(AddressSpace, MemoryArea,
-                                    Address, PageOp);
+      Status = MmPageOutSectionView(AddressSpace, MemoryArea, Address, Entry);
    }
    else if (Type == MEMORY_AREA_CACHE)
    {

Modified: trunk/reactos/ntoskrnl/mm/section.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/section.c?rev=…
==============================================================================
--- trunk/reactos/ntoskrnl/mm/section.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/mm/section.c [iso-8859-1] Sat Apr 28 02:56:31 2012
@@ -127,6 +127,7 @@
    BOOLEAN WasDirty;
    BOOLEAN Private;
    PEPROCESS CallingProcess;
+   ULONG_PTR SectionEntry;
 }
 MM_SECTION_PAGEOUT_CONTEXT;
 
@@ -704,43 +705,6 @@
     return nStatus;
 }
 
-/* Note: Mmsp prefix denotes "Memory Manager Section Private". */
-
-/*
- * FUNCTION:  Waits in kernel mode up to ten seconds for an MM_PAGEOP event.
- * ARGUMENTS: PMM_PAGEOP which event we should wait for.
- * RETURNS:   Status of the wait.
- */
-static NTSTATUS
-MmspWaitForPageOpCompletionEvent(PMM_PAGEOP PageOp)
-{
-   LARGE_INTEGER Timeout;
-#ifdef __GNUC__ /* TODO: Use other macro to check for suffix to use? */
-
-   Timeout.QuadPart = -100000000LL; // 10 sec
-#else
-
-   Timeout.QuadPart = -100000000; // 10 sec
-#endif
-
-   return KeWaitForSingleObject(&PageOp->CompletionEvent, 0, KernelMode, FALSE,
&Timeout);
-}
-
-
-/*
- * FUNCTION:  Sets the page op completion event and releases the page op.
- * ARGUMENTS: PMM_PAGEOP.
- * RETURNS:   In shorter time than it takes you to even read this
- *            description, so don't even think about geting a mug of coffee.
- */
-static void
-MmspCompleteAndReleasePageOp(PMM_PAGEOP PageOp)
-{
-   KeSetEvent(&PageOp->CompletionEvent, IO_NO_INCREMENT, FALSE);
-   MmReleasePageOp(PageOp);
-}
-
-
 /*
  * FUNCTION:  Waits in kernel mode indefinitely for a file object lock.
  * ARGUMENTS: PFILE_OBJECT to wait for.
@@ -831,12 +795,12 @@
                                  PMM_SECTION_SEGMENT Segment,
                                  PLARGE_INTEGER Offset,
                                  BOOLEAN Dirty,
-                                 BOOLEAN PageOut)
-{
-   ULONG_PTR Entry;
+                                 BOOLEAN PageOut,
+                                 ULONG_PTR *InEntry)
+{
+   ULONG_PTR Entry = InEntry ? *InEntry : MmGetPageEntrySectionSegment(Segment, Offset);
    BOOLEAN IsDirectMapped = FALSE;
 
-   Entry = MmGetPageEntrySectionSegment(Segment, Offset);
    if (Entry == 0)
    {
       DPRINT1("Entry == 0 for MmUnsharePageEntrySectionSegment\n");
@@ -844,7 +808,7 @@
    }
    if (SHARE_COUNT_FROM_SSE(Entry) == 0)
    {
-      DPRINT1("Zero share count for unshare\n");
+       DPRINT1("Zero share count for unshare (Seg %p Offset %x Page %x)\n",
Segment, Offset->LowPart, PFN_FROM_SSE(Entry));
        KeBugCheck(MEMORY_MANAGEMENT);
    }
    if (IS_SWAP_FROM_SSE(Entry))
@@ -913,10 +877,14 @@
              *   page without a rmap entry.
              */
             MmSetPageEntrySectionSegment(Segment, Offset, Entry);
+            if (InEntry) *InEntry = Entry;
+            MiSetPageEvent(NULL, NULL);
          }
          else
          {
             MmSetPageEntrySectionSegment(Segment, Offset, 0);
+            if (InEntry) *InEntry = 0;
+            MiSetPageEvent(NULL, NULL);
             if (!IsDirectMapped)
             {
                MmReleasePageMemoryConsumer(MC_USER, Page);
@@ -946,7 +914,9 @@
                   }
                }
                MmSetPageEntrySectionSegment(Segment, Offset,
MAKE_SWAP_SSE(SavedSwapEntry));
+               if (InEntry) *InEntry = MAKE_SWAP_SSE(SavedSwapEntry);
                MmSetSavedSwapEntryPage(Page, 0);
+               MiSetPageEvent(NULL, NULL);
             }
             MmReleasePageMemoryConsumer(MC_USER, Page);
          }
@@ -959,7 +929,10 @@
    }
    else
    {
-      MmSetPageEntrySectionSegment(Segment, Offset, Entry);
+      if (InEntry)
+          *InEntry = Entry;
+      else
+          MmSetPageEntrySectionSegment(Segment, Offset, Entry);
    }
    return(SHARE_COUNT_FROM_SSE(Entry) > 0);
 }
@@ -1232,7 +1205,6 @@
    ULONG_PTR Entry;
    ULONG_PTR Entry1;
    ULONG Attributes;
-   PMM_PAGEOP PageOp;
    PMM_REGION Region;
    BOOLEAN HasSwapEntry;
    PVOID PAddress;
@@ -1270,7 +1242,7 @@
     * Lock the segment
     */
    MmLockSectionSegment(Segment);
-
+   Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
    /*
     * Check if this page needs to be mapped COW
     */
@@ -1286,111 +1258,41 @@
    }
 
    /*
-    * Get or create a page operation descriptor
-    */
-   PageOp = MmGetPageOp(MemoryArea, NULL, 0, Segment, Offset.LowPart, MM_PAGEOP_PAGEIN,
FALSE);
-   if (PageOp == NULL)
-   {
-      DPRINT1("MmGetPageOp failed\n");
-       KeBugCheck(MEMORY_MANAGEMENT);
-   }
-
-   /*
     * Check if someone else is already handling this fault, if so wait
     * for them
     */
-   if (PageOp->Thread != PsGetCurrentThread())
+   if (Entry && IS_SWAP_FROM_SSE(Entry) && SWAPENTRY_FROM_SSE(Entry) ==
MM_WAIT_ENTRY)
    {
       MmUnlockSectionSegment(Segment);
       MmUnlockAddressSpace(AddressSpace);
-      Status = MmspWaitForPageOpCompletionEvent(PageOp);
+      MiWaitForPageEvent(NULL, NULL);
+      MmLockAddressSpace(AddressSpace);
+      DPRINT("Address 0x%.8X\n", Address);
+      return(STATUS_MM_RESTART_OPERATION);
+   }
+
+   HasSwapEntry = MmIsPageSwapEntry(Process, Address);
+
+   if (HasSwapEntry)
+   {
+      SWAPENTRY SwapEntry;
       /*
-       * Check for various strange conditions
+       * Is it a wait entry?
        */
-      if (Status != STATUS_SUCCESS)
-      {
-         DPRINT1("Failed to wait for page op, status = %x\n", Status);
-          KeBugCheck(MEMORY_MANAGEMENT);
-      }
-      if (PageOp->Status == STATUS_PENDING)
-      {
-         DPRINT1("Woke for page op before completion\n");
-          KeBugCheck(MEMORY_MANAGEMENT);
-      }
-      MmLockAddressSpace(AddressSpace);
-      /*
-       * If this wasn't a pagein then restart the operation
-       */
-      if (PageOp->OpType != MM_PAGEOP_PAGEIN)
-      {
-         MmspCompleteAndReleasePageOp(PageOp);
-         DPRINT("Address 0x%.8X\n", Address);
-         return(STATUS_MM_RESTART_OPERATION);
-      }
-
-      /*
-      * If the thread handling this fault has failed then we don't retry
-      */
-      if (!NT_SUCCESS(PageOp->Status))
-      {
-         Status = PageOp->Status;
-         MmspCompleteAndReleasePageOp(PageOp);
-         DPRINT("Address 0x%.8X\n", Address);
-         return(Status);
-      }
-      MmLockSectionSegment(Segment);
-      /*
-       * If the completed fault was for another address space then set the
-       * page in this one.
-       */
-      if (!MmIsPagePresent(Process, Address))
-      {
-         Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
-         HasSwapEntry = MmIsPageSwapEntry(Process, (PVOID)PAddress);
-
-         if (PAGE_FROM_SSE(Entry) == 0 || HasSwapEntry)
-         {
-            /*
-             * The page was a private page in another or in our address space
-             */
-            MmUnlockSectionSegment(Segment);
-            MmspCompleteAndReleasePageOp(PageOp);
-            return(STATUS_MM_RESTART_OPERATION);
-         }
-
-         Page = PFN_FROM_SSE(Entry);
-
-         MmSharePageEntrySectionSegment(Segment, &Offset);
-
-         /* FIXME: Should we call MmCreateVirtualMappingUnsafe if
-          * (Section->AllocationAttributes & SEC_PHYSICALMEMORY) is true?
-          */
-         Status = MmCreateVirtualMapping(Process,
-                                         PAddress,
-                                         Attributes,
-                                         &Page,
-                                         1);
-         if (!NT_SUCCESS(Status))
-         {
-            DPRINT1("Unable to create virtual mapping\n");
-            KeBugCheck(MEMORY_MANAGEMENT);
-         }
-         MmInsertRmap(Page, Process, Address);
-      }
-      MmUnlockSectionSegment(Segment);
-      PageOp->Status = STATUS_SUCCESS;
-      MmspCompleteAndReleasePageOp(PageOp);
-      DPRINT("Address 0x%.8X\n", Address);
-      return(STATUS_SUCCESS);
-   }
-
-   HasSwapEntry = MmIsPageSwapEntry(Process, Address);
-   if (HasSwapEntry)
-   {
+      MmGetPageFileMapping(Process, Address, &SwapEntry);
+
+      if (SwapEntry == MM_WAIT_ENTRY)
+      {
+         MmUnlockSectionSegment(Segment);
+         MmUnlockAddressSpace(AddressSpace);
+         MiWaitForPageEvent(NULL, NULL);
+         MmLockAddressSpace(AddressSpace);
+         return STATUS_MM_RESTART_OPERATION;
+      }
+
       /*
        * Must be private page we have swapped out.
        */
-      SWAPENTRY SwapEntry;
 
       /*
        * Sanity check
@@ -1445,8 +1347,7 @@
       /*
        * Finish the operation
        */
-      PageOp->Status = STATUS_SUCCESS;
-      MmspCompleteAndReleasePageOp(PageOp);
+      MiSetPageEvent(Process, Address);
       DPRINT("Address 0x%.8X\n", Address);
       return(STATUS_SUCCESS);
    }
@@ -1476,8 +1377,7 @@
       /*
        * Cleanup and release locks
        */
-      PageOp->Status = STATUS_SUCCESS;
-      MmspCompleteAndReleasePageOp(PageOp);
+      MiSetPageEvent(Process, Address);
       DPRINT("Address 0x%.8X\n", Address);
       return(STATUS_SUCCESS);
    }
@@ -1518,8 +1418,7 @@
       /*
        * Cleanup and release locks
        */
-      PageOp->Status = STATUS_SUCCESS;
-      MmspCompleteAndReleasePageOp(PageOp);
+      MiSetPageEvent(Process, Address);
       DPRINT("Address 0x%.8X\n", Address);
       return(STATUS_SUCCESS);
    }
@@ -1539,6 +1438,7 @@
       /*
        * Release all our locks and read in the page from disk
        */
+      MmSetPageEntrySectionSegment(Segment, &Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY));
       MmUnlockSectionSegment(Segment);
       MmUnlockAddressSpace(AddressSpace);
 
@@ -1573,11 +1473,64 @@
           * Cleanup and release locks
           */
          MmLockAddressSpace(AddressSpace);
-         PageOp->Status = Status;
-         MmspCompleteAndReleasePageOp(PageOp);
+         MiSetPageEvent(Process, Address);
          DPRINT("Address 0x%.8X\n", Address);
          return(Status);
       }
+
+      /*
+       * Mark the offset within the section as having valid, in-memory
+       * data
+       */
+      MmLockAddressSpace(AddressSpace);
+      MmLockSectionSegment(Segment);
+      Entry = MAKE_SSE(Page << PAGE_SHIFT, 1);
+      MmSetPageEntrySectionSegment(Segment, &Offset, Entry);
+      MmUnlockSectionSegment(Segment);
+
+      Status = MmCreateVirtualMapping(Process,
+                                      PAddress,
+                                      Attributes,
+                                      &Page,
+                                      1);
+      if (!NT_SUCCESS(Status))
+      {
+         DPRINT1("Unable to create virtual mapping\n");
+          KeBugCheck(MEMORY_MANAGEMENT);
+      }
+      MmInsertRmap(Page, Process, Address);
+
+      MiSetPageEvent(Process, Address);
+      DPRINT("Address 0x%.8X\n", Address);
+      return(STATUS_SUCCESS);
+   }
+   else if (IS_SWAP_FROM_SSE(Entry))
+   {
+      SWAPENTRY SwapEntry;
+
+      SwapEntry = SWAPENTRY_FROM_SSE(Entry);
+
+      /*
+      * Release all our locks and read in the page from disk
+      */
+      MmUnlockSectionSegment(Segment);
+
+      MmUnlockAddressSpace(AddressSpace);
+      MI_SET_USAGE(MI_USAGE_SECTION);
+      if (Process) MI_SET_PROCESS2(Process->ImageFileName);
+      if (!Process) MI_SET_PROCESS2("Kernel Section");
+      Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &Page);
+      if (!NT_SUCCESS(Status))
+      {
+          KeBugCheck(MEMORY_MANAGEMENT);
+      }
+
+      Status = MmReadFromSwapPage(SwapEntry, Page);
+      if (!NT_SUCCESS(Status))
+      {
+          KeBugCheck(MEMORY_MANAGEMENT);
+      }
+
       /*
        * Relock the address space and segment
        */
@@ -1591,76 +1544,7 @@
       Entry1 = MmGetPageEntrySectionSegment(Segment, &Offset);
       if (Entry != Entry1)
       {
-         DPRINT1("Someone changed ppte entry while we slept\n");
-         KeBugCheck(MEMORY_MANAGEMENT);
-      }
-
-      /*
-       * Mark the offset within the section as having valid, in-memory
-       * data
-       */
-      Entry = MAKE_SSE(Page << PAGE_SHIFT, 1);
-      MmSetPageEntrySectionSegment(Segment, &Offset, Entry);
-      MmUnlockSectionSegment(Segment);
-
-      Status = MmCreateVirtualMapping(Process,
-                                      PAddress,
-                                      Attributes,
-                                      &Page,
-                                      1);
-      if (!NT_SUCCESS(Status))
-      {
-         DPRINT1("Unable to create virtual mapping\n");
-          KeBugCheck(MEMORY_MANAGEMENT);
-      }
-      MmInsertRmap(Page, Process, Address);
-
-      PageOp->Status = STATUS_SUCCESS;
-      MmspCompleteAndReleasePageOp(PageOp);
-      DPRINT("Address 0x%.8X\n", Address);
-      return(STATUS_SUCCESS);
-   }
-   else if (IS_SWAP_FROM_SSE(Entry))
-   {
-      SWAPENTRY SwapEntry;
-
-      SwapEntry = SWAPENTRY_FROM_SSE(Entry);
-
-      /*
-      * Release all our locks and read in the page from disk
-      */
-      MmUnlockSectionSegment(Segment);
-
-      MmUnlockAddressSpace(AddressSpace);
-      MI_SET_USAGE(MI_USAGE_SECTION);
-      if (Process) MI_SET_PROCESS2(Process->ImageFileName);
-      if (!Process) MI_SET_PROCESS2("Kernel Section");
-      Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &Page);
-      if (!NT_SUCCESS(Status))
-      {
-          KeBugCheck(MEMORY_MANAGEMENT);
-      }
-
-      Status = MmReadFromSwapPage(SwapEntry, Page);
-      if (!NT_SUCCESS(Status))
-      {
-          KeBugCheck(MEMORY_MANAGEMENT);
-      }
-
-      /*
-       * Relock the address space and segment
-       */
-      MmLockAddressSpace(AddressSpace);
-      MmLockSectionSegment(Segment);
-
-      /*
-       * Check the entry. No one should change the status of a page
-       * that has a pending page-in.
-       */
-      Entry1 = MmGetPageEntrySectionSegment(Segment, &Offset);
-      if (Entry != Entry1)
-      {
-         DPRINT1("Someone changed ppte entry while we slept\n");
+          DPRINT1("Someone changed ppte entry while we slept (%x vs %x)\n",
Entry, Entry1);
           KeBugCheck(MEMORY_MANAGEMENT);
       }
 
@@ -1687,8 +1571,7 @@
           KeBugCheck(MEMORY_MANAGEMENT);
       }
       MmInsertRmap(Page, Process, Address);
-      PageOp->Status = STATUS_SUCCESS;
-      MmspCompleteAndReleasePageOp(PageOp);
+      MiSetPageEvent(Process, Address);
       DPRINT("Address 0x%.8X\n", Address);
       return(STATUS_SUCCESS);
    }
@@ -1715,8 +1598,7 @@
           KeBugCheck(MEMORY_MANAGEMENT);
       }
       MmInsertRmap(Page, Process, Address);
-      PageOp->Status = STATUS_SUCCESS;
-      MmspCompleteAndReleasePageOp(PageOp);
+      MiSetPageEvent(Process, Address);
       DPRINT("Address 0x%.8X\n", Address);
       return(STATUS_SUCCESS);
    }
@@ -1735,7 +1617,6 @@
    NTSTATUS Status;
    PVOID PAddress;
    LARGE_INTEGER Offset;
-   PMM_PAGEOP PageOp;
    PMM_REGION Region;
    ULONG_PTR Entry;
    PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace);
@@ -1799,46 +1680,29 @@
    /*
     * Get or create a pageop
     */
-   PageOp = MmGetPageOp(MemoryArea, NULL, 0, Segment, Offset.LowPart,
-                        MM_PAGEOP_ACCESSFAULT, FALSE);
-   if (PageOp == NULL)
-   {
-      DPRINT1("MmGetPageOp failed\n");
-       KeBugCheck(MEMORY_MANAGEMENT);
-   }
-
+   MmLockSectionSegment(Segment);
+   Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
+   
    /*
     * Wait for any other operations to complete
     */
-   if (PageOp->Thread != PsGetCurrentThread())
-   {
-      MmUnlockAddressSpace(AddressSpace);
-      Status = MmspWaitForPageOpCompletionEvent(PageOp);
-      /*
-      * Check for various strange conditions
-      */
-      if (Status == STATUS_TIMEOUT)
-      {
-         DPRINT1("Failed to wait for page op, status = %x\n", Status);
-          KeBugCheck(MEMORY_MANAGEMENT);
-      }
-      if (PageOp->Status == STATUS_PENDING)
-      {
-         DPRINT1("Woke for page op before completion\n");
-          KeBugCheck(MEMORY_MANAGEMENT);
-      }
-      /*
-      * Restart the operation
-      */
-      MmLockAddressSpace(AddressSpace);
-      MmspCompleteAndReleasePageOp(PageOp);
-      DPRINT("Address 0x%.8X\n", Address);
-      return(STATUS_MM_RESTART_OPERATION);
+   if (Entry == SWAPENTRY_FROM_SSE(MM_WAIT_ENTRY))
+   {
+       MmUnlockSectionSegment(Segment);
+       MmUnlockAddressSpace(AddressSpace);
+       MiWaitForPageEvent(NULL, NULL);
+       /*
+        * Restart the operation
+        */
+       MmLockAddressSpace(AddressSpace);
+       DPRINT("Address 0x%.8X\n", Address);
+       return(STATUS_MM_RESTART_OPERATION);
    }
 
    /*
     * Release locks now we have the pageop
     */
+   MmUnlockSectionSegment(Segment);
    MmUnlockAddressSpace(AddressSpace);
 
    /*
@@ -1890,11 +1754,10 @@
    MmDeleteRmap(OldPage, Process, PAddress);
    MmInsertRmap(NewPage, Process, PAddress);
    MmLockSectionSegment(Segment);
-   MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, FALSE, FALSE);
+   MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, FALSE, FALSE, NULL);
    MmUnlockSectionSegment(Segment);
 
-   PageOp->Status = STATUS_SUCCESS;
-   MmspCompleteAndReleasePageOp(PageOp);
+   MiSetPageEvent(Process, Address);
    DPRINT("Address 0x%.8X\n", Address);
    return(STATUS_SUCCESS);
 }
@@ -1904,7 +1767,7 @@
 {
    MM_SECTION_PAGEOUT_CONTEXT* PageOutContext;
    BOOLEAN WasDirty;
-   PFN_NUMBER Page;
+   PFN_NUMBER Page = 0;
 
    PageOutContext = (MM_SECTION_PAGEOUT_CONTEXT*)Context;
    if (Process)
@@ -1928,7 +1791,8 @@
                                        PageOutContext->Segment,
                                        &PageOutContext->Offset,
                                        PageOutContext->WasDirty,
-                                       TRUE);
+                                       TRUE,
+                                       &PageOutContext->SectionEntry);
       MmUnlockSectionSegment(PageOutContext->Segment);
    }
    if (Process)
@@ -1940,21 +1804,17 @@
    {
       MmReleasePageMemoryConsumer(MC_USER, Page);
    }
-
-   DPRINT("PhysicalAddress %x, Address %x\n", Page << PAGE_SHIFT,
Address);
 }
 
 NTSTATUS
 NTAPI
 MmPageOutSectionView(PMMSUPPORT AddressSpace,
                      MEMORY_AREA* MemoryArea,
-                     PVOID Address,
-                     PMM_PAGEOP PageOp)
+                     PVOID Address, ULONG_PTR Entry)
 {
    PFN_NUMBER Page;
    MM_SECTION_PAGEOUT_CONTEXT Context;
    SWAPENTRY SwapEntry;
-   ULONG_PTR Entry;
    ULONGLONG FileOffset;
    NTSTATUS Status;
    PFILE_OBJECT FileObject;
@@ -1973,6 +1833,7 @@
     */
    Context.Segment = MemoryArea->Data.SectionData.Segment;
    Context.Section = MemoryArea->Data.SectionData.Section;
+   Context.SectionEntry = Entry;
    Context.CallingProcess = Process;
 
    Context.Offset.QuadPart = (ULONG_PTR)Address -
(ULONG_PTR)MemoryArea->StartingAddress
@@ -1983,6 +1844,8 @@
 
    FileObject = Context.Section->FileObject;
    DirectMapped = FALSE;
+
+   MmLockSectionSegment(Context.Segment);
 
 #ifndef NEWCC
    if (FileObject != NULL &&
@@ -2035,16 +1898,14 @@
    {
        DPRINT("Cannot page out locked section page: 0x%p (RefCount: %d)\n",
                Page, MmGetReferenceCountPage(Page));
-       PageOp->Status = STATUS_UNSUCCESSFUL;
-       MmspCompleteAndReleasePageOp(PageOp);
+       MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry);
+       MmUnlockSectionSegment(Context.Segment);
        return STATUS_UNSUCCESSFUL;
    }
 
    /*
     * Prepare the context structure for the rmap delete call.
     */
-   MmLockSectionSegment(Context.Segment);
-   Entry = MmGetPageEntrySectionSegment(Context.Segment, &Context.Offset);
    MmUnlockSectionSegment(Context.Segment);
    Context.WasDirty = FALSE;
    if (Context.Segment->Image.Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA
||
@@ -2077,9 +1938,12 @@
    }
 
    MmDeleteAllRmaps(Page, (PVOID)&Context, MmPageOutDeleteMapping);
-   MmLockSectionSegment(Context.Segment);
-   Entry = MmGetPageEntrySectionSegment(Context.Segment, &Context.Offset);
-   MmUnlockSectionSegment(Context.Segment);
+
+   /* Since we passed in a surrogate, we'll get back the page entry
+    * state in our context.  This is intended to make intermediate
+    * decrements of share count not release the wait entry.
+    */
+   Entry = Context.SectionEntry;
 
    /*
     * If this wasn't a private page then we should have reduced the entry to
@@ -2117,8 +1981,7 @@
          MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset,
MAKE_SWAP_SSE(SwapEntry));
          MmUnlockSectionSegment(Context.Segment);
          MmReleasePageMemoryConsumer(MC_USER, Page);
-         PageOp->Status = STATUS_SUCCESS;
-         MmspCompleteAndReleasePageOp(PageOp);
+         MiSetPageEvent(NULL, NULL);
          return(STATUS_SUCCESS);
       }
    }
@@ -2140,8 +2003,7 @@
             MmUnlockSectionSegment(Context.Segment);
          }
          MmReleasePageMemoryConsumer(MC_USER, Page);
-         PageOp->Status = STATUS_SUCCESS;
-         MmspCompleteAndReleasePageOp(PageOp);
+         MiSetPageEvent(NULL, NULL);
          return(STATUS_SUCCESS);
       }
    }
@@ -2165,8 +2027,7 @@
          KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)Bcb, (ULONG_PTR)FileOffset,
(ULONG_PTR)Address);
       }
 #endif
-      PageOp->Status = STATUS_SUCCESS;
-      MmspCompleteAndReleasePageOp(PageOp);
+      MiSetPageEvent(NULL, NULL);
       return(STATUS_SUCCESS);
    }
    else if (!Context.WasDirty && !DirectMapped && !Context.Private)
@@ -2178,13 +2039,12 @@
          KeBugCheckEx(MEMORY_MANAGEMENT, SwapEntry, Page, (ULONG_PTR)Process,
(ULONG_PTR)Address);
       }
       MmReleasePageMemoryConsumer(MC_USER, Page);
-      PageOp->Status = STATUS_SUCCESS;
-      MmspCompleteAndReleasePageOp(PageOp);
+      MiSetPageEvent(NULL, NULL);
       return(STATUS_SUCCESS);
    }
    else if (!Context.WasDirty && Context.Private && SwapEntry != 0)
    {
-      DPRINT1("Not dirty and private and not swapped (%p:%p)\n", Process,
Address);
+      DPRINT("Not dirty and private and not swapped (%p:%p)\n", Process,
Address);
       MmSetSavedSwapEntryPage(Page, 0);
       MmLockAddressSpace(AddressSpace);
       Status = MmCreatePageFileMapping(Process,
@@ -2197,8 +2057,7 @@
          KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)Process, (ULONG_PTR)Address,
SwapEntry);
       }
       MmReleasePageMemoryConsumer(MC_USER, Page);
-      PageOp->Status = STATUS_SUCCESS;
-      MmspCompleteAndReleasePageOp(PageOp);
+      MiSetPageEvent(NULL, NULL);
       return(STATUS_SUCCESS);
    }
 
@@ -2253,8 +2112,7 @@
 	    MmUnlockSectionSegment(Context.Segment);
          }
          MmUnlockAddressSpace(AddressSpace);
-         PageOp->Status = STATUS_UNSUCCESSFUL;
-         MmspCompleteAndReleasePageOp(PageOp);
+         MiSetPageEvent(NULL, NULL);
          return(STATUS_PAGEFILE_QUOTA);
       }
    }
@@ -2299,8 +2157,7 @@
          MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry);
       }
       MmUnlockAddressSpace(AddressSpace);
-      PageOp->Status = STATUS_UNSUCCESSFUL;
-      MmspCompleteAndReleasePageOp(PageOp);
+      MiSetPageEvent(NULL, NULL);
       return(STATUS_UNSUCCESSFUL);
    }
 
@@ -2308,12 +2165,13 @@
     * Otherwise we have succeeded.
     */
    DPRINT("MM: Wrote section page 0x%.8X to swap!\n", Page <<
PAGE_SHIFT);
-   MmLockSectionSegment(Context.Segment);
    MmSetSavedSwapEntryPage(Page, 0);
    if (Context.Segment->Flags & MM_PAGEFILE_SEGMENT ||
          Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)
    {
+      MmLockSectionSegment(Context.Segment);
       MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset,
MAKE_SWAP_SSE(SwapEntry));
+      MmUnlockSectionSegment(Context.Segment);
    }
    else
    {
@@ -2323,9 +2181,11 @@
    if (Context.Private)
    {
       MmLockAddressSpace(AddressSpace);
+      MmLockSectionSegment(Context.Segment);
       Status = MmCreatePageFileMapping(Process,
                                        Address,
                                        SwapEntry);
+      MmUnlockSectionSegment(Context.Segment);
       MmUnlockAddressSpace(AddressSpace);
       if (!NT_SUCCESS(Status))
       {
@@ -2335,13 +2195,15 @@
    }
    else
    {
+      MmLockAddressSpace(AddressSpace);
+      MmLockSectionSegment(Context.Segment);
       Entry = MAKE_SWAP_SSE(SwapEntry);
       MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry);
-   }
-
-   MmUnlockSectionSegment(Context.Segment);
-   PageOp->Status = STATUS_SUCCESS;
-   MmspCompleteAndReleasePageOp(PageOp);
+      MmUnlockSectionSegment(Context.Segment);
+      MmUnlockAddressSpace(AddressSpace);
+   }
+
+   MiSetPageEvent(NULL, NULL);
    return(STATUS_SUCCESS);
 }
 
@@ -2350,7 +2212,7 @@
 MmWritePageSectionView(PMMSUPPORT AddressSpace,
                        PMEMORY_AREA MemoryArea,
                        PVOID Address,
-                       PMM_PAGEOP PageOp)
+                       ULONG PageEntry)
 {
    LARGE_INTEGER Offset;
    PROS_SECTION_OBJECT Section;
@@ -2447,12 +2309,16 @@
     */
    if (DirectMapped && !Private)
    {
+      LARGE_INTEGER SOffset;
       ASSERT(SwapEntry == 0);
+      SOffset.QuadPart = Offset.QuadPart + Segment->Image.FileOffset;
 #ifndef NEWCC
-      CcRosMarkDirtyCacheSegment(Bcb, (ULONG)(Offset.QuadPart +
Segment->Image.FileOffset));
+      CcRosMarkDirtyCacheSegment(Bcb, Offset.LowPart);
 #endif
-      PageOp->Status = STATUS_SUCCESS;
-      MmspCompleteAndReleasePageOp(PageOp);
+      MmLockSectionSegment(Segment);
+      MmSetPageEntrySectionSegment(Segment, &Offset, PageEntry);
+      MmUnlockSectionSegment(Segment);
+      MiSetPageEvent(NULL, NULL);
       return(STATUS_SUCCESS);
    }
 
@@ -2465,8 +2331,7 @@
       if (SwapEntry == 0)
       {
          MmSetDirtyAllRmaps(Page);
-         PageOp->Status = STATUS_UNSUCCESSFUL;
-         MmspCompleteAndReleasePageOp(PageOp);
+         MiSetPageEvent(NULL, NULL);
          return(STATUS_PAGEFILE_QUOTA);
       }
       MmSetSavedSwapEntryPage(Page, SwapEntry);
@@ -2481,8 +2346,7 @@
       DPRINT1("MM: Failed to write to swap page (Status was 0x%.8X)\n",
               Status);
       MmSetDirtyAllRmaps(Page);
-      PageOp->Status = STATUS_UNSUCCESSFUL;
-      MmspCompleteAndReleasePageOp(PageOp);
+      MiSetPageEvent(NULL, NULL);
       return(STATUS_UNSUCCESSFUL);
    }
 
@@ -2490,8 +2354,7 @@
     * Otherwise we have succeeded.
     */
    DPRINT("MM: Wrote section page 0x%.8X to swap!\n", Page <<
PAGE_SHIFT);
-   PageOp->Status = STATUS_SUCCESS;
-   MmspCompleteAndReleasePageOp(PageOp);
+   MiSetPageEvent(NULL, NULL);
    return(STATUS_SUCCESS);
 }
 
@@ -4039,8 +3902,6 @@
    PBCB Bcb;
    LARGE_INTEGER Offset;
    SWAPENTRY SavedSwapEntry;
-   PMM_PAGEOP PageOp;
-   NTSTATUS Status;
    PROS_SECTION_OBJECT Section;
    PMM_SECTION_SEGMENT Segment;
    PMMSUPPORT AddressSpace;
@@ -4057,27 +3918,18 @@
    Section = MemoryArea->Data.SectionData.Section;
    Segment = MemoryArea->Data.SectionData.Segment;
 
-   PageOp = MmCheckForPageOp(MemoryArea, NULL, NULL, Segment, Offset.LowPart);
-
-   while (PageOp)
+   Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
+   while (Entry && IS_SWAP_FROM_SSE(Entry) && SWAPENTRY_FROM_SSE(Entry)
== MM_WAIT_ENTRY)
    {
       MmUnlockSectionSegment(Segment);
       MmUnlockAddressSpace(AddressSpace);
 
-      Status = MmspWaitForPageOpCompletionEvent(PageOp);
-      if (Status != STATUS_SUCCESS)
-      {
-         DPRINT1("Failed to wait for page op, status = %x\n", Status);
-         KeBugCheck(MEMORY_MANAGEMENT);
-      }
+      MiWaitForPageEvent(NULL, NULL);
 
       MmLockAddressSpace(AddressSpace);
       MmLockSectionSegment(Segment);
-      MmspCompleteAndReleasePageOp(PageOp);
-      PageOp = MmCheckForPageOp(MemoryArea, NULL, NULL, Segment, Offset.LowPart);
-   }
-
-   Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
+      Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
+   }
 
    /*
     * For a dirty, datafile, non-private page mark it as dirty in the
@@ -4136,7 +3988,7 @@
       else
       {
          MmDeleteRmap(Page, Process, Address);
-         MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, Dirty, FALSE);
+         MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, Dirty, FALSE,
NULL);
       }
    }
 }
@@ -4208,9 +4060,7 @@
    PMEMORY_AREA MemoryArea;
    PMMSUPPORT AddressSpace;
    PROS_SECTION_OBJECT Section;
-   PMM_PAGEOP PageOp;
-   ULONG_PTR Offset;
-    PVOID ImageBaseAddress = 0;
+   PVOID ImageBaseAddress = 0;
 
    DPRINT("Opening memory area Process %x BaseAddress %x\n",
           Process, BaseAddress);
@@ -4232,40 +4082,6 @@
    }
 
    MemoryArea->DeleteInProgress = TRUE;
-
-   while (MemoryArea->PageOpCount)
-   {
-      Offset = PAGE_ROUND_UP((ULONG_PTR)MemoryArea->EndingAddress -
(ULONG_PTR)MemoryArea->StartingAddress);
-
-      while (Offset)
-      {
-         Offset -= PAGE_SIZE;
-         PageOp = MmCheckForPageOp(MemoryArea, NULL, NULL,
-                                   MemoryArea->Data.SectionData.Segment,
-                                   Offset +
MemoryArea->Data.SectionData.ViewOffset.QuadPart);
-         if (PageOp)
-         {
-            MmUnlockAddressSpace(AddressSpace);
-            Status = MmspWaitForPageOpCompletionEvent(PageOp);
-            if (Status != STATUS_SUCCESS)
-            {
-               DPRINT1("Failed to wait for page op, status = %x\n", Status);
-               KeBugCheck(MEMORY_MANAGEMENT);
-            }
-            MmLockAddressSpace(AddressSpace);
-            MmspCompleteAndReleasePageOp(PageOp);
-            MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace,
-                                                     BaseAddress);
-            if (MemoryArea == NULL ||
-                MemoryArea->Type != MEMORY_AREA_SECTION_VIEW)
-            {
-               MmUnlockAddressSpace(AddressSpace);
-               return STATUS_NOT_MAPPED_VIEW;
-            }
-            break;
-         }
-      }
-   }
 
    Section = MemoryArea->Data.SectionData.Section;