Best regards,
Alex Ionescu

On Sat, Apr 28, 2012 at 10:56 AM, <arty@svn.reactos.org> wrote:

Author: arty
Date: Sat Apr 28 02:56:31 2012
New Revision: 56441

URL: http://svn.reactos.org/svn/reactos?rev=56441&view=rev
Log:
[NTOSKRNL]
Remove PAGEOPs and use the argument to pass relevant information where needed,
including generally a section page table entry or a PFN_NUMBER.

In its place, the code looks for an MM_WAIT_ENTRY in the appropriate place and
waits, returning STATUS_MM_RESTART_OPERATION. Rather than copying the pagein
and CoW operations in the PAGEOP contention case, we just punt back and restart
the operation, resulting in simpler code.

Modified:
trunk/reactos/ntoskrnl/CMakeLists.txt
trunk/reactos/ntoskrnl/cache/section/sptab.c
trunk/reactos/ntoskrnl/include/internal/mm.h
trunk/reactos/ntoskrnl/mm/mminit.c
trunk/reactos/ntoskrnl/mm/rmap.c
trunk/reactos/ntoskrnl/mm/section.c

Modified: trunk/reactos/ntoskrnl/CMakeLists.txt
URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/CMakeLists.txt?rev=56441&r1=56440&r2=56441&view=diff
==============================================================================
--- trunk/reactos/ntoskrnl/CMakeLists.txt [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/CMakeLists.txt [iso-8859-1] Sat Apr 28 02:56:31 2012
@@ -229,7 +229,6 @@
mm/mmfault.c
mm/mminit.c
mm/pagefile.c
- mm/pageop.c
mm/region.c
mm/rmap.c
mm/section.c

Modified: trunk/reactos/ntoskrnl/cache/section/sptab.c
URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/cache/section/sptab.c?rev=56441&r1=56440&r2=56441&view=diff
==============================================================================
--- trunk/reactos/ntoskrnl/cache/section/sptab.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/cache/section/sptab.c [iso-8859-1] Sat Apr 28 02:56:31 2012
@@ -215,8 +215,9 @@
ASSERT(!Entry || IS_SWAP_FROM_SSE(Entry));
MmDeleteSectionAssociation(PFN_FROM_SSE(OldEntry));
} else if (IS_SWAP_FROM_SSE(Entry)) {
- ASSERT(!IS_SWAP_FROM_SSE(OldEntry));
- if (OldEntry)
+ ASSERT(!IS_SWAP_FROM_SSE(OldEntry) ||
+ SWAPENTRY_FROM_SSE(OldEntry) == MM_WAIT_ENTRY);
+ if (OldEntry && SWAPENTRY_FROM_SSE(OldEntry) != MM_WAIT_ENTRY)
MmDeleteSectionAssociation(PFN_FROM_SSE(OldEntry));
} else if (IS_SWAP_FROM_SSE(OldEntry)) {
ASSERT(!IS_SWAP_FROM_SSE(Entry));

Modified: trunk/reactos/ntoskrnl/include/internal/mm.h
URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/include/internal/mm.h?rev=56441&r1=56440&r2=56441&view=diff
==============================================================================
--- trunk/reactos/ntoskrnl/include/internal/mm.h [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/include/internal/mm.h [iso-8859-1] Sat Apr 28 02:56:31 2012
@@ -49,7 +49,6 @@
struct _KTRAP_FRAME;
struct _EPROCESS;
struct _MM_RMAP_ENTRY;
-struct _MM_PAGEOP;
typedef ULONG_PTR SWAPENTRY;

//
@@ -83,12 +82,6 @@
#define MM_CORE_DUMP_TYPE_NONE (0x0)
#define MM_CORE_DUMP_TYPE_MINIMAL (0x1)
#define MM_CORE_DUMP_TYPE_FULL (0x2)
-
-#define MM_PAGEOP_PAGEIN (1)
-#define MM_PAGEOP_PAGEOUT (2)
-#define MM_PAGEOP_PAGESYNCH (3)
-#define MM_PAGEOP_ACCESSFAULT (4)
-#define MM_PAGEOP_CHANGEPROTECT (5)

/* Number of list heads to use */
#define MI_FREE_POOL_LISTS 4
@@ -415,37 +408,6 @@
extern MMPFNLIST MmModifiedPageListHead;
extern MMPFNLIST MmModifiedNoWritePageListHead;

-typedef struct _MM_PAGEOP
-{
- /* Type of operation. */
- ULONG OpType;
- /* Number of threads interested in this operation. */
- ULONG ReferenceCount;
- /* Event that will be set when the operation is completed. */
- KEVENT CompletionEvent;
- /* Status of the operation once it is completed. */
- NTSTATUS Status;
- /* TRUE if the operation was abandoned. */
- BOOLEAN Abandoned;
- /* The memory area to be affected by the operation. */
- PMEMORY_AREA MArea;
- ULONG Hash;
- struct _MM_PAGEOP* Next;
- struct _ETHREAD* Thread;
- /*
- * These fields are used to identify the operation if it is against a
- * virtual memory area.
- */
- HANDLE Pid;
- PVOID Address;
- /*
- * These fields are used to identify the operation if it is against a
- * section mapping.
- */
- PMM_SECTION_SEGMENT Segment;
- ULONGLONG Offset;
-} MM_PAGEOP, *PMM_PAGEOP;
-
typedef struct _MM_MEMORY_CONSUMER
{
ULONG PagesUsed;
@@ -937,7 +899,7 @@
PMMSUPPORT AddressSpace,
PMEMORY_AREA MemoryArea,
PVOID Address,
- struct _MM_PAGEOP* PageOp
+ PFN_NUMBER Page
);

NTSTATUS
@@ -973,7 +935,7 @@
PMMSUPPORT AddressSpace,
PMEMORY_AREA MArea,
PVOID Address,
- PMM_PAGEOP PageOp
+ PFN_NUMBER Page
);

/* kmap.c ********************************************************************/
@@ -1006,38 +968,6 @@
PVOID
FASTCALL
MmSafeReadPtr(PVOID Source);
-
-/* pageop.c ******************************************************************/
-
-VOID
-NTAPI
-MmReleasePageOp(PMM_PAGEOP PageOp);
-
-PMM_PAGEOP
-NTAPI
-MmGetPageOp(
- PMEMORY_AREA MArea,
- HANDLE Pid,
- PVOID Address,
- PMM_SECTION_SEGMENT Segment,
- ULONGLONG Offset,
- ULONG OpType,
- BOOLEAN First
-);
-
-PMM_PAGEOP
-NTAPI
-MmCheckForPageOp(
- PMEMORY_AREA MArea,
- HANDLE Pid,
- PVOID Address,
- PMM_SECTION_SEGMENT Segment,
- ULONGLONG Offset
-);
-
-VOID
-NTAPI
-MmInitializePageOp(VOID);

/* process.c *****************************************************************/

@@ -1657,7 +1587,7 @@
PMMSUPPORT AddressSpace,
PMEMORY_AREA MemoryArea,
PVOID Address,
- struct _MM_PAGEOP *PageOp
+ ULONG_PTR Entry
);

NTSTATUS

Modified: trunk/reactos/ntoskrnl/mm/mminit.c
URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/mminit.c?rev=56441&r1=56440&r2=56441&view=diff
==============================================================================
--- trunk/reactos/ntoskrnl/mm/mminit.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/mm/mminit.c [iso-8859-1] Sat Apr 28 02:56:31 2012
@@ -419,7 +419,6 @@
MiInitializeUserPfnBitmap();
MmInitializeMemoryConsumer(MC_USER, MmTrimUserMemory);
MmInitializeRmapList();
- MmInitializePageOp();
MmInitSectionImplementation();
MmInitPagingFile();

Modified: trunk/reactos/ntoskrnl/mm/rmap.c
URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/rmap.c?rev=56441&r1=56440&r2=56441&view=diff
==============================================================================
--- trunk/reactos/ntoskrnl/mm/rmap.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/mm/rmap.c [iso-8859-1] Sat Apr 28 02:56:31 2012
@@ -52,7 +52,6 @@
ULONG Type;
PVOID Address;
PEPROCESS Process;
- PMM_PAGEOP PageOp;
ULONGLONG Offset;
NTSTATUS Status = STATUS_SUCCESS;

@@ -126,17 +125,20 @@
Type = MemoryArea->Type;
if (Type == MEMORY_AREA_SECTION_VIEW)
{
+ ULONG Entry;
Offset = MemoryArea->Data.SectionData.ViewOffset.QuadPart +
((ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress);
+
+ MmLockSectionSegment(MemoryArea->Data.SectionData.Segment);

/*
* Get or create a pageop
*/
- PageOp = MmGetPageOp(MemoryArea, NULL, 0,
- MemoryArea->Data.SectionData.Segment,
- Offset, MM_PAGEOP_PAGEOUT, TRUE);
- if (PageOp == NULL)
- {
+ Entry = MmGetPageEntrySectionSegment
+ (MemoryArea->Data.SectionData.Segment, (PLARGE_INTEGER)&Offset);
+ if (Entry && IS_SWAP_FROM_SSE(Entry) && SWAPENTRY_FROM_SSE(Entry) == MM_WAIT_ENTRY)
+ {
+ MmUnlockSectionSegment(MemoryArea->Data.SectionData.Segment);
MmUnlockAddressSpace(AddressSpace);
if (Address < MmSystemRangeStart)
{
@@ -145,17 +147,19 @@
}
return(STATUS_UNSUCCESSFUL);
}
+
+ MmSetPageEntrySectionSegment(MemoryArea->Data.SectionData.Segment, (PLARGE_INTEGER)&Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY));

/*
* Release locks now we have a page op.
*/
+ MmUnlockSectionSegment(MemoryArea->Data.SectionData.Segment);
MmUnlockAddressSpace(AddressSpace);

/*
* Do the actual page out work.
*/
- Status = MmPageOutSectionView(AddressSpace, MemoryArea,
- Address, PageOp);
+ Status = MmPageOutSectionView(AddressSpace, MemoryArea, Address, Entry);
}
else if (Type == MEMORY_AREA_CACHE)
{

Modified: trunk/reactos/ntoskrnl/mm/section.c
URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/section.c?rev=56441&r1=56440&r2=56441&view=diff
==============================================================================
--- trunk/reactos/ntoskrnl/mm/section.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/mm/section.c [iso-8859-1] Sat Apr 28 02:56:31 2012
@@ -127,6 +127,7 @@
BOOLEAN WasDirty;
BOOLEAN Private;
PEPROCESS CallingProcess;
+ ULONG_PTR SectionEntry;
}
MM_SECTION_PAGEOUT_CONTEXT;

@@ -704,43 +705,6 @@
return nStatus;
}

-/* Note: Mmsp prefix denotes "Memory Manager Section Private". */
-
-/*
- * FUNCTION: Waits in kernel mode up to ten seconds for an MM_PAGEOP event.
- * ARGUMENTS: PMM_PAGEOP which event we should wait for.
- * RETURNS: Status of the wait.
- */
-static NTSTATUS
-MmspWaitForPageOpCompletionEvent(PMM_PAGEOP PageOp)
-{
- LARGE_INTEGER Timeout;
-#ifdef __GNUC__ /* TODO: Use other macro to check for suffix to use? */
-
- Timeout.QuadPart = -100000000LL; // 10 sec
-#else
-
- Timeout.QuadPart = -100000000; // 10 sec
-#endif
-
- return KeWaitForSingleObject(&PageOp->CompletionEvent, 0, KernelMode, FALSE, &Timeout);
-}
-
-
-/*
- * FUNCTION: Sets the page op completion event and releases the page op.
- * ARGUMENTS: PMM_PAGEOP.
- * RETURNS: In shorter time than it takes you to even read this
- * description, so don't even think about geting a mug of coffee.
- */
-static void
-MmspCompleteAndReleasePageOp(PMM_PAGEOP PageOp)
-{
- KeSetEvent(&PageOp->CompletionEvent, IO_NO_INCREMENT, FALSE);
- MmReleasePageOp(PageOp);
-}
-
-
/*
* FUNCTION: Waits in kernel mode indefinitely for a file object lock.
* ARGUMENTS: PFILE_OBJECT to wait for.
@@ -831,12 +795,12 @@
PMM_SECTION_SEGMENT Segment,
PLARGE_INTEGER Offset,
BOOLEAN Dirty,
- BOOLEAN PageOut)
-{
- ULONG_PTR Entry;
+ BOOLEAN PageOut,
+ ULONG_PTR *InEntry)
+{
+ ULONG_PTR Entry = InEntry ? *InEntry : MmGetPageEntrySectionSegment(Segment, Offset);
BOOLEAN IsDirectMapped = FALSE;

- Entry = MmGetPageEntrySectionSegment(Segment, Offset);
if (Entry == 0)
{
DPRINT1("Entry == 0 for MmUnsharePageEntrySectionSegment\n");
@@ -844,7 +808,7 @@
}
if (SHARE_COUNT_FROM_SSE(Entry) == 0)
{
- DPRINT1("Zero share count for unshare\n");
+ DPRINT1("Zero share count for unshare (Seg %p Offset %x Page %x)\n", Segment, Offset->LowPart, PFN_FROM_SSE(Entry));
KeBugCheck(MEMORY_MANAGEMENT);
}
if (IS_SWAP_FROM_SSE(Entry))
@@ -913,10 +877,14 @@
* page without a rmap entry.
*/
MmSetPageEntrySectionSegment(Segment, Offset, Entry);
+ if (InEntry) *InEntry = Entry;
+ MiSetPageEvent(NULL, NULL);
}
else
{
MmSetPageEntrySectionSegment(Segment, Offset, 0);
+ if (InEntry) *InEntry = 0;
+ MiSetPageEvent(NULL, NULL);
if (!IsDirectMapped)
{
MmReleasePageMemoryConsumer(MC_USER, Page);
@@ -946,7 +914,9 @@
}
}
MmSetPageEntrySectionSegment(Segment, Offset, MAKE_SWAP_SSE(SavedSwapEntry));
+ if (InEntry) *InEntry = MAKE_SWAP_SSE(SavedSwapEntry);
MmSetSavedSwapEntryPage(Page, 0);
+ MiSetPageEvent(NULL, NULL);
}
MmReleasePageMemoryConsumer(MC_USER, Page);
}
@@ -959,7 +929,10 @@
}
else
{
- MmSetPageEntrySectionSegment(Segment, Offset, Entry);
+ if (InEntry)
+ *InEntry = Entry;
+ else
+ MmSetPageEntrySectionSegment(Segment, Offset, Entry);
}
return(SHARE_COUNT_FROM_SSE(Entry) > 0);
}
@@ -1232,7 +1205,6 @@
ULONG_PTR Entry;
ULONG_PTR Entry1;
ULONG Attributes;
- PMM_PAGEOP PageOp;
PMM_REGION Region;
BOOLEAN HasSwapEntry;
PVOID PAddress;
@@ -1270,7 +1242,7 @@
* Lock the segment
*/
MmLockSectionSegment(Segment);
-
+ Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
/*
* Check if this page needs to be mapped COW
*/
@@ -1286,111 +1258,41 @@
}

/*
- * Get or create a page operation descriptor
- */
- PageOp = MmGetPageOp(MemoryArea, NULL, 0, Segment, Offset.LowPart, MM_PAGEOP_PAGEIN, FALSE);
- if (PageOp == NULL)
- {
- DPRINT1("MmGetPageOp failed\n");
- KeBugCheck(MEMORY_MANAGEMENT);
- }
-
- /*
* Check if someone else is already handling this fault, if so wait
* for them
*/
- if (PageOp->Thread != PsGetCurrentThread())
+ if (Entry && IS_SWAP_FROM_SSE(Entry) && SWAPENTRY_FROM_SSE(Entry) == MM_WAIT_ENTRY)
{
MmUnlockSectionSegment(Segment);
MmUnlockAddressSpace(AddressSpace);
- Status = MmspWaitForPageOpCompletionEvent(PageOp);
+ MiWaitForPageEvent(NULL, NULL);
+ MmLockAddressSpace(AddressSpace);
+ DPRINT("Address 0x%.8X\n", Address);
+ return(STATUS_MM_RESTART_OPERATION);
+ }
+
+ HasSwapEntry = MmIsPageSwapEntry(Process, Address);
+
+ if (HasSwapEntry)
+ {
+ SWAPENTRY SwapEntry;
/*
- * Check for various strange conditions
+ * Is it a wait entry?
*/
- if (Status != STATUS_SUCCESS)
- {
- DPRINT1("Failed to wait for page op, status = %x\n", Status);
- KeBugCheck(MEMORY_MANAGEMENT);
- }
- if (PageOp->Status == STATUS_PENDING)
- {
- DPRINT1("Woke for page op before completion\n");
- KeBugCheck(MEMORY_MANAGEMENT);
- }
- MmLockAddressSpace(AddressSpace);
- /*
- * If this wasn't a pagein then restart the operation
- */
- if (PageOp->OpType != MM_PAGEOP_PAGEIN)
- {
- MmspCompleteAndReleasePageOp(PageOp);
- DPRINT("Address 0x%.8X\n", Address);
- return(STATUS_MM_RESTART_OPERATION);
- }
-
- /*
- * If the thread handling this fault has failed then we don't retry
- */
- if (!NT_SUCCESS(PageOp->Status))
- {
- Status = PageOp->Status;
- MmspCompleteAndReleasePageOp(PageOp);
- DPRINT("Address 0x%.8X\n", Address);
- return(Status);
- }
- MmLockSectionSegment(Segment);
- /*
- * If the completed fault was for another address space then set the
- * page in this one.
- */
- if (!MmIsPagePresent(Process, Address))
- {
- Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
- HasSwapEntry = MmIsPageSwapEntry(Process, (PVOID)PAddress);
-
- if (PAGE_FROM_SSE(Entry) == 0 || HasSwapEntry)
- {
- /*
- * The page was a private page in another or in our address space
- */
- MmUnlockSectionSegment(Segment);
- MmspCompleteAndReleasePageOp(PageOp);
- return(STATUS_MM_RESTART_OPERATION);
- }
-
- Page = PFN_FROM_SSE(Entry);
-
- MmSharePageEntrySectionSegment(Segment, &Offset);
-
- /* FIXME: Should we call MmCreateVirtualMappingUnsafe if
- * (Section->AllocationAttributes & SEC_PHYSICALMEMORY) is true?
- */
- Status = MmCreateVirtualMapping(Process,
- PAddress,
- Attributes,
- &Page,
- 1);
- if (!NT_SUCCESS(Status))
- {
- DPRINT1("Unable to create virtual mapping\n");
- KeBugCheck(MEMORY_MANAGEMENT);
- }
- MmInsertRmap(Page, Process, Address);
- }
- MmUnlockSectionSegment(Segment);
- PageOp->Status = STATUS_SUCCESS;
- MmspCompleteAndReleasePageOp(PageOp);
- DPRINT("Address 0x%.8X\n", Address);
- return(STATUS_SUCCESS);
- }
-
- HasSwapEntry = MmIsPageSwapEntry(Process, Address);
- if (HasSwapEntry)
- {
+ MmGetPageFileMapping(Process, Address, &SwapEntry);
+
+ if (SwapEntry == MM_WAIT_ENTRY)
+ {
+ MmUnlockSectionSegment(Segment);
+ MmUnlockAddressSpace(AddressSpace);
+ MiWaitForPageEvent(NULL, NULL);
+ MmLockAddressSpace(AddressSpace);
+ return STATUS_MM_RESTART_OPERATION;
+ }
+
/*
* Must be private page we have swapped out.
*/
- SWAPENTRY SwapEntry;

/*
* Sanity check
@@ -1445,8 +1347,7 @@
/*
* Finish the operation
*/
- PageOp->Status = STATUS_SUCCESS;
- MmspCompleteAndReleasePageOp(PageOp);
+ MiSetPageEvent(Process, Address);
DPRINT("Address 0x%.8X\n", Address);
return(STATUS_SUCCESS);
}
@@ -1476,8 +1377,7 @@
/*
* Cleanup and release locks
*/
- PageOp->Status = STATUS_SUCCESS;
- MmspCompleteAndReleasePageOp(PageOp);
+ MiSetPageEvent(Process, Address);
DPRINT("Address 0x%.8X\n", Address);
return(STATUS_SUCCESS);
}
@@ -1518,8 +1418,7 @@
/*
* Cleanup and release locks
*/
- PageOp->Status = STATUS_SUCCESS;
- MmspCompleteAndReleasePageOp(PageOp);
+ MiSetPageEvent(Process, Address);
DPRINT("Address 0x%.8X\n", Address);
return(STATUS_SUCCESS);
}
@@ -1539,6 +1438,7 @@
/*
* Release all our locks and read in the page from disk
*/
+ MmSetPageEntrySectionSegment(Segment, &Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY));
MmUnlockSectionSegment(Segment);
MmUnlockAddressSpace(AddressSpace);

@@ -1573,11 +1473,64 @@
* Cleanup and release locks
*/
MmLockAddressSpace(AddressSpace);
- PageOp->Status = Status;
- MmspCompleteAndReleasePageOp(PageOp);
+ MiSetPageEvent(Process, Address);
DPRINT("Address 0x%.8X\n", Address);
return(Status);
}
+
+ /*
+ * Mark the offset within the section as having valid, in-memory
+ * data
+ */
+ MmLockAddressSpace(AddressSpace);
+ MmLockSectionSegment(Segment);
+ Entry = MAKE_SSE(Page << PAGE_SHIFT, 1);
+ MmSetPageEntrySectionSegment(Segment, &Offset, Entry);
+ MmUnlockSectionSegment(Segment);
+
+ Status = MmCreateVirtualMapping(Process,
+ PAddress,
+ Attributes,
+ &Page,
+ 1);
+ if (!NT_SUCCESS(Status))
+ {
+ DPRINT1("Unable to create virtual mapping\n");
+ KeBugCheck(MEMORY_MANAGEMENT);
+ }
+ MmInsertRmap(Page, Process, Address);
+
+ MiSetPageEvent(Process, Address);
+ DPRINT("Address 0x%.8X\n", Address);
+ return(STATUS_SUCCESS);
+ }
+ else if (IS_SWAP_FROM_SSE(Entry))
+ {
+ SWAPENTRY SwapEntry;
+
+ SwapEntry = SWAPENTRY_FROM_SSE(Entry);
+
+ /*
+ * Release all our locks and read in the page from disk
+ */
+ MmUnlockSectionSegment(Segment);
+
+ MmUnlockAddressSpace(AddressSpace);
+ MI_SET_USAGE(MI_USAGE_SECTION);
+ if (Process) MI_SET_PROCESS2(Process->ImageFileName);
+ if (!Process) MI_SET_PROCESS2("Kernel Section");
+ Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &Page);
+ if (!NT_SUCCESS(Status))
+ {
+ KeBugCheck(MEMORY_MANAGEMENT);
+ }
+
+ Status = MmReadFromSwapPage(SwapEntry, Page);
+ if (!NT_SUCCESS(Status))
+ {
+ KeBugCheck(MEMORY_MANAGEMENT);
+ }
+
/*
* Relock the address space and segment
*/
@@ -1591,76 +1544,7 @@
Entry1 = MmGetPageEntrySectionSegment(Segment, &Offset);
if (Entry != Entry1)
{
- DPRINT1("Someone changed ppte entry while we slept\n");
- KeBugCheck(MEMORY_MANAGEMENT);
- }
-
- /*
- * Mark the offset within the section as having valid, in-memory
- * data
- */
- Entry = MAKE_SSE(Page << PAGE_SHIFT, 1);
- MmSetPageEntrySectionSegment(Segment, &Offset, Entry);
- MmUnlockSectionSegment(Segment);
-
- Status = MmCreateVirtualMapping(Process,
- PAddress,
- Attributes,
- &Page,
- 1);
- if (!NT_SUCCESS(Status))
- {
- DPRINT1("Unable to create virtual mapping\n");
- KeBugCheck(MEMORY_MANAGEMENT);
- }
- MmInsertRmap(Page, Process, Address);
-
- PageOp->Status = STATUS_SUCCESS;
- MmspCompleteAndReleasePageOp(PageOp);
- DPRINT("Address 0x%.8X\n", Address);
- return(STATUS_SUCCESS);
- }
- else if (IS_SWAP_FROM_SSE(Entry))
- {
- SWAPENTRY SwapEntry;
-
- SwapEntry = SWAPENTRY_FROM_SSE(Entry);
-
- /*
- * Release all our locks and read in the page from disk
- */
- MmUnlockSectionSegment(Segment);
-
- MmUnlockAddressSpace(AddressSpace);
- MI_SET_USAGE(MI_USAGE_SECTION);
- if (Process) MI_SET_PROCESS2(Process->ImageFileName);
- if (!Process) MI_SET_PROCESS2("Kernel Section");
- Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &Page);
- if (!NT_SUCCESS(Status))
- {
- KeBugCheck(MEMORY_MANAGEMENT);
- }
-
- Status = MmReadFromSwapPage(SwapEntry, Page);
- if (!NT_SUCCESS(Status))
- {
- KeBugCheck(MEMORY_MANAGEMENT);
- }
-
- /*
- * Relock the address space and segment
- */
- MmLockAddressSpace(AddressSpace);
- MmLockSectionSegment(Segment);
-
- /*
- * Check the entry. No one should change the status of a page
- * that has a pending page-in.
- */
- Entry1 = MmGetPageEntrySectionSegment(Segment, &Offset);
- if (Entry != Entry1)
- {
- DPRINT1("Someone changed ppte entry while we slept\n");
+ DPRINT1("Someone changed ppte entry while we slept (%x vs %x)\n", Entry, Entry1);
KeBugCheck(MEMORY_MANAGEMENT);
}

@@ -1687,8 +1571,7 @@
KeBugCheck(MEMORY_MANAGEMENT);
}
MmInsertRmap(Page, Process, Address);
- PageOp->Status = STATUS_SUCCESS;
- MmspCompleteAndReleasePageOp(PageOp);
+ MiSetPageEvent(Process, Address);
DPRINT("Address 0x%.8X\n", Address);
return(STATUS_SUCCESS);
}
@@ -1715,8 +1598,7 @@
KeBugCheck(MEMORY_MANAGEMENT);
}
MmInsertRmap(Page, Process, Address);
- PageOp->Status = STATUS_SUCCESS;
- MmspCompleteAndReleasePageOp(PageOp);
+ MiSetPageEvent(Process, Address);
DPRINT("Address 0x%.8X\n", Address);
return(STATUS_SUCCESS);
}
@@ -1735,7 +1617,6 @@
NTSTATUS Status;
PVOID PAddress;
LARGE_INTEGER Offset;
- PMM_PAGEOP PageOp;
PMM_REGION Region;
ULONG_PTR Entry;
PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace);
@@ -1799,46 +1680,29 @@
/*
* Get or create a pageop
*/
- PageOp = MmGetPageOp(MemoryArea, NULL, 0, Segment, Offset.LowPart,
- MM_PAGEOP_ACCESSFAULT, FALSE);
- if (PageOp == NULL)
- {
- DPRINT1("MmGetPageOp failed\n");
- KeBugCheck(MEMORY_MANAGEMENT);
- }
-
+ MmLockSectionSegment(Segment);
+ Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
+
/*
* Wait for any other operations to complete
*/
- if (PageOp->Thread != PsGetCurrentThread())
- {
- MmUnlockAddressSpace(AddressSpace);
- Status = MmspWaitForPageOpCompletionEvent(PageOp);
- /*
- * Check for various strange conditions
- */
- if (Status == STATUS_TIMEOUT)
- {
- DPRINT1("Failed to wait for page op, status = %x\n", Status);
- KeBugCheck(MEMORY_MANAGEMENT);
- }
- if (PageOp->Status == STATUS_PENDING)
- {
- DPRINT1("Woke for page op before completion\n");
- KeBugCheck(MEMORY_MANAGEMENT);
- }
- /*
- * Restart the operation
- */
- MmLockAddressSpace(AddressSpace);
- MmspCompleteAndReleasePageOp(PageOp);
- DPRINT("Address 0x%.8X\n", Address);
- return(STATUS_MM_RESTART_OPERATION);
+ if (Entry == SWAPENTRY_FROM_SSE(MM_WAIT_ENTRY))
+ {
+ MmUnlockSectionSegment(Segment);
+ MmUnlockAddressSpace(AddressSpace);
+ MiWaitForPageEvent(NULL, NULL);
+ /*
+ * Restart the operation
+ */
+ MmLockAddressSpace(AddressSpace);
+ DPRINT("Address 0x%.8X\n", Address);
+ return(STATUS_MM_RESTART_OPERATION);
}

/*
* Release locks now we have the pageop
*/
+ MmUnlockSectionSegment(Segment);
MmUnlockAddressSpace(AddressSpace);

/*
@@ -1890,11 +1754,10 @@
MmDeleteRmap(OldPage, Process, PAddress);
MmInsertRmap(NewPage, Process, PAddress);
MmLockSectionSegment(Segment);
- MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, FALSE, FALSE);
+ MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, FALSE, FALSE, NULL);
MmUnlockSectionSegment(Segment);

- PageOp->Status = STATUS_SUCCESS;
- MmspCompleteAndReleasePageOp(PageOp);
+ MiSetPageEvent(Process, Address);
DPRINT("Address 0x%.8X\n", Address);
return(STATUS_SUCCESS);
}
@@ -1904,7 +1767,7 @@
{
MM_SECTION_PAGEOUT_CONTEXT* PageOutContext;
BOOLEAN WasDirty;
- PFN_NUMBER Page;
+ PFN_NUMBER Page = 0;

PageOutContext = (MM_SECTION_PAGEOUT_CONTEXT*)Context;
if (Process)
@@ -1928,7 +1791,8 @@
PageOutContext->Segment,
&PageOutContext->Offset,
PageOutContext->WasDirty,
- TRUE);
+ TRUE,
+ &PageOutContext->SectionEntry);
MmUnlockSectionSegment(PageOutContext->Segment);
}
if (Process)
@@ -1940,21 +1804,17 @@
{
MmReleasePageMemoryConsumer(MC_USER, Page);
}
-
- DPRINT("PhysicalAddress %x, Address %x\n", Page << PAGE_SHIFT, Address);
}

NTSTATUS
NTAPI
MmPageOutSectionView(PMMSUPPORT AddressSpace,
MEMORY_AREA* MemoryArea,
- PVOID Address,
- PMM_PAGEOP PageOp)
+ PVOID Address, ULONG_PTR Entry)
{
PFN_NUMBER Page;
MM_SECTION_PAGEOUT_CONTEXT Context;
SWAPENTRY SwapEntry;
- ULONG_PTR Entry;
ULONGLONG FileOffset;
NTSTATUS Status;
PFILE_OBJECT FileObject;
@@ -1973,6 +1833,7 @@
*/
Context.Segment = MemoryArea->Data.SectionData.Segment;
Context.Section = MemoryArea->Data.SectionData.Section;
+ Context.SectionEntry = Entry;
Context.CallingProcess = Process;

Context.Offset.QuadPart = (ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress
@@ -1983,6 +1844,8 @@

FileObject = Context.Section->FileObject;
DirectMapped = FALSE;
+
+ MmLockSectionSegment(Context.Segment);

#ifndef NEWCC
if (FileObject != NULL &&
@@ -2035,16 +1898,14 @@
{
DPRINT("Cannot page out locked section page: 0x%p (RefCount: %d)\n",
Page, MmGetReferenceCountPage(Page));
- PageOp->Status = STATUS_UNSUCCESSFUL;
- MmspCompleteAndReleasePageOp(PageOp);
+ MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry);
+ MmUnlockSectionSegment(Context.Segment);
return STATUS_UNSUCCESSFUL;
}

/*
* Prepare the context structure for the rmap delete call.
*/
- MmLockSectionSegment(Context.Segment);
- Entry = MmGetPageEntrySectionSegment(Context.Segment, &Context.Offset);
MmUnlockSectionSegment(Context.Segment);
Context.WasDirty = FALSE;
if (Context.Segment->Image.Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA ||
@@ -2077,9 +1938,12 @@
}

MmDeleteAllRmaps(Page, (PVOID)&Context, MmPageOutDeleteMapping);
- MmLockSectionSegment(Context.Segment);
- Entry = MmGetPageEntrySectionSegment(Context.Segment, &Context.Offset);
- MmUnlockSectionSegment(Context.Segment);
+
+ /* Since we passed in a surrogate, we'll get back the page entry
+ * state in our context. This is intended to make intermediate
+ * decrements of share count not release the wait entry.
+ */
+ Entry = Context.SectionEntry;

/*
* If this wasn't a private page then we should have reduced the entry to
@@ -2117,8 +1981,7 @@
MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry));
MmUnlockSectionSegment(Context.Segment);
MmReleasePageMemoryConsumer(MC_USER, Page);
- PageOp->Status = STATUS_SUCCESS;
- MmspCompleteAndReleasePageOp(PageOp);
+ MiSetPageEvent(NULL, NULL);
return(STATUS_SUCCESS);
}
}
@@ -2140,8 +2003,7 @@
MmUnlockSectionSegment(Context.Segment);
}
MmReleasePageMemoryConsumer(MC_USER, Page);
- PageOp->Status = STATUS_SUCCESS;
- MmspCompleteAndReleasePageOp(PageOp);
+ MiSetPageEvent(NULL, NULL);
return(STATUS_SUCCESS);
}
}
@@ -2165,8 +2027,7 @@
KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)Bcb, (ULONG_PTR)FileOffset, (ULONG_PTR)Address);
}
#endif
- PageOp->Status = STATUS_SUCCESS;
- MmspCompleteAndReleasePageOp(PageOp);
+ MiSetPageEvent(NULL, NULL);
return(STATUS_SUCCESS);
}
else if (!Context.WasDirty && !DirectMapped && !Context.Private)
@@ -2178,13 +2039,12 @@
KeBugCheckEx(MEMORY_MANAGEMENT, SwapEntry, Page, (ULONG_PTR)Process, (ULONG_PTR)Address);
}
MmReleasePageMemoryConsumer(MC_USER, Page);
- PageOp->Status = STATUS_SUCCESS;
- MmspCompleteAndReleasePageOp(PageOp);
+ MiSetPageEvent(NULL, NULL);
return(STATUS_SUCCESS);
}
else if (!Context.WasDirty && Context.Private && SwapEntry != 0)
{
- DPRINT1("Not dirty and private and not swapped (%p:%p)\n", Process, Address);
+ DPRINT("Not dirty and private and not swapped (%p:%p)\n", Process, Address);
MmSetSavedSwapEntryPage(Page, 0);
MmLockAddressSpace(AddressSpace);
Status = MmCreatePageFileMapping(Process,
@@ -2197,8 +2057,7 @@
KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)Process, (ULONG_PTR)Address, SwapEntry);
}
MmReleasePageMemoryConsumer(MC_USER, Page);
- PageOp->Status = STATUS_SUCCESS;
- MmspCompleteAndReleasePageOp(PageOp);
+ MiSetPageEvent(NULL, NULL);
return(STATUS_SUCCESS);
}

@@ -2253,8 +2112,7 @@
MmUnlockSectionSegment(Context.Segment);
}
MmUnlockAddressSpace(AddressSpace);
- PageOp->Status = STATUS_UNSUCCESSFUL;
- MmspCompleteAndReleasePageOp(PageOp);
+ MiSetPageEvent(NULL, NULL);
return(STATUS_PAGEFILE_QUOTA);
}
}
@@ -2299,8 +2157,7 @@
MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry);
}
MmUnlockAddressSpace(AddressSpace);
- PageOp->Status = STATUS_UNSUCCESSFUL;
- MmspCompleteAndReleasePageOp(PageOp);
+ MiSetPageEvent(NULL, NULL);
return(STATUS_UNSUCCESSFUL);
}

@@ -2308,12 +2165,13 @@
* Otherwise we have succeeded.
*/
DPRINT("MM: Wrote section page 0x%.8X to swap!\n", Page << PAGE_SHIFT);
- MmLockSectionSegment(Context.Segment);
MmSetSavedSwapEntryPage(Page, 0);
if (Context.Segment->Flags & MM_PAGEFILE_SEGMENT ||
Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)
{
+ MmLockSectionSegment(Context.Segment);
MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry));
+ MmUnlockSectionSegment(Context.Segment);
}
else
{
@@ -2323,9 +2181,11 @@
if (Context.Private)
{
MmLockAddressSpace(AddressSpace);
+ MmLockSectionSegment(Context.Segment);
Status = MmCreatePageFileMapping(Process,
Address,
SwapEntry);
+ MmUnlockSectionSegment(Context.Segment);
MmUnlockAddressSpace(AddressSpace);
if (!NT_SUCCESS(Status))
{
@@ -2335,13 +2195,15 @@
}
else
{
+ MmLockAddressSpace(AddressSpace);
+ MmLockSectionSegment(Context.Segment);
Entry = MAKE_SWAP_SSE(SwapEntry);
MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry);
- }
-
- MmUnlockSectionSegment(Context.Segment);
- PageOp->Status = STATUS_SUCCESS;
- MmspCompleteAndReleasePageOp(PageOp);
+ MmUnlockSectionSegment(Context.Segment);
+ MmUnlockAddressSpace(AddressSpace);
+ }
+
+ MiSetPageEvent(NULL, NULL);
return(STATUS_SUCCESS);
}

@@ -2350,7 +2212,7 @@
MmWritePageSectionView(PMMSUPPORT AddressSpace,
PMEMORY_AREA MemoryArea,
PVOID Address,
- PMM_PAGEOP PageOp)
+ ULONG PageEntry)
{
LARGE_INTEGER Offset;
PROS_SECTION_OBJECT Section;
@@ -2447,12 +2309,16 @@
*/
if (DirectMapped && !Private)
{
+ LARGE_INTEGER SOffset;
ASSERT(SwapEntry == 0);
+ SOffset.QuadPart = Offset.QuadPart + Segment->Image.FileOffset;
#ifndef NEWCC
- CcRosMarkDirtyCacheSegment(Bcb, (ULONG)(Offset.QuadPart + Segment->Image.FileOffset));
+ CcRosMarkDirtyCacheSegment(Bcb, Offset.LowPart);
#endif
- PageOp->Status = STATUS_SUCCESS;
- MmspCompleteAndReleasePageOp(PageOp);
+ MmLockSectionSegment(Segment);
+ MmSetPageEntrySectionSegment(Segment, &Offset, PageEntry);
+ MmUnlockSectionSegment(Segment);
+ MiSetPageEvent(NULL, NULL);
return(STATUS_SUCCESS);
}

@@ -2465,8 +2331,7 @@
if (SwapEntry == 0)
{
MmSetDirtyAllRmaps(Page);
- PageOp->Status = STATUS_UNSUCCESSFUL;
- MmspCompleteAndReleasePageOp(PageOp);
+ MiSetPageEvent(NULL, NULL);
return(STATUS_PAGEFILE_QUOTA);
}
MmSetSavedSwapEntryPage(Page, SwapEntry);
@@ -2481,8 +2346,7 @@
DPRINT1("MM: Failed to write to swap page (Status was 0x%.8X)\n",
Status);
MmSetDirtyAllRmaps(Page);
- PageOp->Status = STATUS_UNSUCCESSFUL;
- MmspCompleteAndReleasePageOp(PageOp);
+ MiSetPageEvent(NULL, NULL);
return(STATUS_UNSUCCESSFUL);
}

@@ -2490,8 +2354,7 @@
* Otherwise we have succeeded.
*/
DPRINT("MM: Wrote section page 0x%.8X to swap!\n", Page << PAGE_SHIFT);
- PageOp->Status = STATUS_SUCCESS;
- MmspCompleteAndReleasePageOp(PageOp);
+ MiSetPageEvent(NULL, NULL);
return(STATUS_SUCCESS);
}

@@ -4039,8 +3902,6 @@
PBCB Bcb;
LARGE_INTEGER Offset;
SWAPENTRY SavedSwapEntry;
- PMM_PAGEOP PageOp;
- NTSTATUS Status;
PROS_SECTION_OBJECT Section;
PMM_SECTION_SEGMENT Segment;
PMMSUPPORT AddressSpace;
@@ -4057,27 +3918,18 @@
Section = MemoryArea->Data.SectionData.Section;
Segment = MemoryArea->Data.SectionData.Segment;

- PageOp = MmCheckForPageOp(MemoryArea, NULL, NULL, Segment, Offset.LowPart);
-
- while (PageOp)
+ Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
+ while (Entry && IS_SWAP_FROM_SSE(Entry) && SWAPENTRY_FROM_SSE(Entry) == MM_WAIT_ENTRY)
{
MmUnlockSectionSegment(Segment);
MmUnlockAddressSpace(AddressSpace);

- Status = MmspWaitForPageOpCompletionEvent(PageOp);
- if (Status != STATUS_SUCCESS)
- {
- DPRINT1("Failed to wait for page op, status = %x\n", Status);
- KeBugCheck(MEMORY_MANAGEMENT);
- }
+ MiWaitForPageEvent(NULL, NULL);

MmLockAddressSpace(AddressSpace);
MmLockSectionSegment(Segment);
- MmspCompleteAndReleasePageOp(PageOp);
- PageOp = MmCheckForPageOp(MemoryArea, NULL, NULL, Segment, Offset.LowPart);
- }
-
- Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
+ Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
+ }

/*
* For a dirty, datafile, non-private page mark it as dirty in the
@@ -4136,7 +3988,7 @@
else
{
MmDeleteRmap(Page, Process, Address);
- MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, Dirty, FALSE);
+ MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, Dirty, FALSE, NULL);
}
}
}
@@ -4208,9 +4060,7 @@
PMEMORY_AREA MemoryArea;
PMMSUPPORT AddressSpace;
PROS_SECTION_OBJECT Section;
- PMM_PAGEOP PageOp;
- ULONG_PTR Offset;
- PVOID ImageBaseAddress = 0;
+ PVOID ImageBaseAddress = 0;

DPRINT("Opening memory area Process %x BaseAddress %x\n",
Process, BaseAddress);
@@ -4232,40 +4082,6 @@
}

MemoryArea->DeleteInProgress = TRUE;
-
- while (MemoryArea->PageOpCount)
- {
- Offset = PAGE_ROUND_UP((ULONG_PTR)MemoryArea->EndingAddress - (ULONG_PTR)MemoryArea->StartingAddress);
-
- while (Offset)
- {
- Offset -= PAGE_SIZE;
- PageOp = MmCheckForPageOp(MemoryArea, NULL, NULL,
- MemoryArea->Data.SectionData.Segment,
- Offset + MemoryArea->Data.SectionData.ViewOffset.QuadPart);
- if (PageOp)
- {
- MmUnlockAddressSpace(AddressSpace);
- Status = MmspWaitForPageOpCompletionEvent(PageOp);
- if (Status != STATUS_SUCCESS)
- {
- DPRINT1("Failed to wait for page op, status = %x\n", Status);
- KeBugCheck(MEMORY_MANAGEMENT);
- }
- MmLockAddressSpace(AddressSpace);
- MmspCompleteAndReleasePageOp(PageOp);
- MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace,
- BaseAddress);
- if (MemoryArea == NULL ||
- MemoryArea->Type != MEMORY_AREA_SECTION_VIEW)
- {
- MmUnlockAddressSpace(AddressSpace);
- return STATUS_NOT_MAPPED_VIEW;
- }
- break;
- }
- }
- }

Section = MemoryArea->Data.SectionData.Section;