Author: arty Date: Sat Apr 28 02:56:31 2012 New Revision: 56441
URL: http://svn.reactos.org/svn/reactos?rev=56441&view=rev Log: [NTOSKRNL] Remove PAGEOPs and use the argument to pass relevant information where needed, including generally a section page table entry or a PFN_NUMBER.
In its place, the code looks for an MM_WAIT_ENTRY in the appropriate place and waits, returning STATUS_MM_RESTART_OPERATION. Rather than copying the pagein and CoW operations in the PAGEOP contention case, we just punt back and restart the operation, resulting in simpler code.
Modified: trunk/reactos/ntoskrnl/CMakeLists.txt trunk/reactos/ntoskrnl/cache/section/sptab.c trunk/reactos/ntoskrnl/include/internal/mm.h trunk/reactos/ntoskrnl/mm/mminit.c trunk/reactos/ntoskrnl/mm/rmap.c trunk/reactos/ntoskrnl/mm/section.c
Modified: trunk/reactos/ntoskrnl/CMakeLists.txt URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/CMakeLists.txt?rev... ============================================================================== --- trunk/reactos/ntoskrnl/CMakeLists.txt [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/CMakeLists.txt [iso-8859-1] Sat Apr 28 02:56:31 2012 @@ -229,7 +229,6 @@ mm/mmfault.c mm/mminit.c mm/pagefile.c - mm/pageop.c mm/region.c mm/rmap.c mm/section.c
Modified: trunk/reactos/ntoskrnl/cache/section/sptab.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/cache/section/spta... ============================================================================== --- trunk/reactos/ntoskrnl/cache/section/sptab.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/cache/section/sptab.c [iso-8859-1] Sat Apr 28 02:56:31 2012 @@ -215,8 +215,9 @@ ASSERT(!Entry || IS_SWAP_FROM_SSE(Entry)); MmDeleteSectionAssociation(PFN_FROM_SSE(OldEntry)); } else if (IS_SWAP_FROM_SSE(Entry)) { - ASSERT(!IS_SWAP_FROM_SSE(OldEntry)); - if (OldEntry) + ASSERT(!IS_SWAP_FROM_SSE(OldEntry) || + SWAPENTRY_FROM_SSE(OldEntry) == MM_WAIT_ENTRY); + if (OldEntry && SWAPENTRY_FROM_SSE(OldEntry) != MM_WAIT_ENTRY) MmDeleteSectionAssociation(PFN_FROM_SSE(OldEntry)); } else if (IS_SWAP_FROM_SSE(OldEntry)) { ASSERT(!IS_SWAP_FROM_SSE(Entry));
Modified: trunk/reactos/ntoskrnl/include/internal/mm.h URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/include/internal/m... ============================================================================== --- trunk/reactos/ntoskrnl/include/internal/mm.h [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/include/internal/mm.h [iso-8859-1] Sat Apr 28 02:56:31 2012 @@ -49,7 +49,6 @@ struct _KTRAP_FRAME; struct _EPROCESS; struct _MM_RMAP_ENTRY; -struct _MM_PAGEOP; typedef ULONG_PTR SWAPENTRY;
// @@ -83,12 +82,6 @@ #define MM_CORE_DUMP_TYPE_NONE (0x0) #define MM_CORE_DUMP_TYPE_MINIMAL (0x1) #define MM_CORE_DUMP_TYPE_FULL (0x2) - -#define MM_PAGEOP_PAGEIN (1) -#define MM_PAGEOP_PAGEOUT (2) -#define MM_PAGEOP_PAGESYNCH (3) -#define MM_PAGEOP_ACCESSFAULT (4) -#define MM_PAGEOP_CHANGEPROTECT (5)
/* Number of list heads to use */ #define MI_FREE_POOL_LISTS 4 @@ -415,37 +408,6 @@ extern MMPFNLIST MmModifiedPageListHead; extern MMPFNLIST MmModifiedNoWritePageListHead;
-typedef struct _MM_PAGEOP -{ - /* Type of operation. */ - ULONG OpType; - /* Number of threads interested in this operation. */ - ULONG ReferenceCount; - /* Event that will be set when the operation is completed. */ - KEVENT CompletionEvent; - /* Status of the operation once it is completed. */ - NTSTATUS Status; - /* TRUE if the operation was abandoned. */ - BOOLEAN Abandoned; - /* The memory area to be affected by the operation. */ - PMEMORY_AREA MArea; - ULONG Hash; - struct _MM_PAGEOP* Next; - struct _ETHREAD* Thread; - /* - * These fields are used to identify the operation if it is against a - * virtual memory area. - */ - HANDLE Pid; - PVOID Address; - /* - * These fields are used to identify the operation if it is against a - * section mapping. - */ - PMM_SECTION_SEGMENT Segment; - ULONGLONG Offset; -} MM_PAGEOP, *PMM_PAGEOP; - typedef struct _MM_MEMORY_CONSUMER { ULONG PagesUsed; @@ -937,7 +899,7 @@ PMMSUPPORT AddressSpace, PMEMORY_AREA MemoryArea, PVOID Address, - struct _MM_PAGEOP* PageOp + PFN_NUMBER Page );
NTSTATUS @@ -973,7 +935,7 @@ PMMSUPPORT AddressSpace, PMEMORY_AREA MArea, PVOID Address, - PMM_PAGEOP PageOp + PFN_NUMBER Page );
/* kmap.c ********************************************************************/ @@ -1006,38 +968,6 @@ PVOID FASTCALL MmSafeReadPtr(PVOID Source); - -/* pageop.c ******************************************************************/ - -VOID -NTAPI -MmReleasePageOp(PMM_PAGEOP PageOp); - -PMM_PAGEOP -NTAPI -MmGetPageOp( - PMEMORY_AREA MArea, - HANDLE Pid, - PVOID Address, - PMM_SECTION_SEGMENT Segment, - ULONGLONG Offset, - ULONG OpType, - BOOLEAN First -); - -PMM_PAGEOP -NTAPI -MmCheckForPageOp( - PMEMORY_AREA MArea, - HANDLE Pid, - PVOID Address, - PMM_SECTION_SEGMENT Segment, - ULONGLONG Offset -); - -VOID -NTAPI -MmInitializePageOp(VOID);
/* process.c *****************************************************************/
@@ -1657,7 +1587,7 @@ PMMSUPPORT AddressSpace, PMEMORY_AREA MemoryArea, PVOID Address, - struct _MM_PAGEOP *PageOp + ULONG_PTR Entry );
NTSTATUS
Modified: trunk/reactos/ntoskrnl/mm/mminit.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/mminit.c?rev=56... ============================================================================== --- trunk/reactos/ntoskrnl/mm/mminit.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/mm/mminit.c [iso-8859-1] Sat Apr 28 02:56:31 2012 @@ -419,7 +419,6 @@ MiInitializeUserPfnBitmap(); MmInitializeMemoryConsumer(MC_USER, MmTrimUserMemory); MmInitializeRmapList(); - MmInitializePageOp(); MmInitSectionImplementation(); MmInitPagingFile();
Modified: trunk/reactos/ntoskrnl/mm/rmap.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/rmap.c?rev=5644... ============================================================================== --- trunk/reactos/ntoskrnl/mm/rmap.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/mm/rmap.c [iso-8859-1] Sat Apr 28 02:56:31 2012 @@ -52,7 +52,6 @@ ULONG Type; PVOID Address; PEPROCESS Process; - PMM_PAGEOP PageOp; ULONGLONG Offset; NTSTATUS Status = STATUS_SUCCESS;
@@ -126,17 +125,20 @@ Type = MemoryArea->Type; if (Type == MEMORY_AREA_SECTION_VIEW) { + ULONG Entry; Offset = MemoryArea->Data.SectionData.ViewOffset.QuadPart + ((ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress); + + MmLockSectionSegment(MemoryArea->Data.SectionData.Segment);
/* * Get or create a pageop */ - PageOp = MmGetPageOp(MemoryArea, NULL, 0, - MemoryArea->Data.SectionData.Segment, - Offset, MM_PAGEOP_PAGEOUT, TRUE); - if (PageOp == NULL) - { + Entry = MmGetPageEntrySectionSegment + (MemoryArea->Data.SectionData.Segment, (PLARGE_INTEGER)&Offset); + if (Entry && IS_SWAP_FROM_SSE(Entry) && SWAPENTRY_FROM_SSE(Entry) == MM_WAIT_ENTRY) + { + MmUnlockSectionSegment(MemoryArea->Data.SectionData.Segment); MmUnlockAddressSpace(AddressSpace); if (Address < MmSystemRangeStart) { @@ -145,17 +147,19 @@ } return(STATUS_UNSUCCESSFUL); } + + MmSetPageEntrySectionSegment(MemoryArea->Data.SectionData.Segment, (PLARGE_INTEGER)&Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY));
/* * Release locks now we have a page op. */ + MmUnlockSectionSegment(MemoryArea->Data.SectionData.Segment); MmUnlockAddressSpace(AddressSpace);
/* * Do the actual page out work. */ - Status = MmPageOutSectionView(AddressSpace, MemoryArea, - Address, PageOp); + Status = MmPageOutSectionView(AddressSpace, MemoryArea, Address, Entry); } else if (Type == MEMORY_AREA_CACHE) {
Modified: trunk/reactos/ntoskrnl/mm/section.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/section.c?rev=5... ============================================================================== --- trunk/reactos/ntoskrnl/mm/section.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/mm/section.c [iso-8859-1] Sat Apr 28 02:56:31 2012 @@ -127,6 +127,7 @@ BOOLEAN WasDirty; BOOLEAN Private; PEPROCESS CallingProcess; + ULONG_PTR SectionEntry; } MM_SECTION_PAGEOUT_CONTEXT;
@@ -704,43 +705,6 @@ return nStatus; }
-/* Note: Mmsp prefix denotes "Memory Manager Section Private". */ - -/* - * FUNCTION: Waits in kernel mode up to ten seconds for an MM_PAGEOP event. - * ARGUMENTS: PMM_PAGEOP which event we should wait for. - * RETURNS: Status of the wait. - */ -static NTSTATUS -MmspWaitForPageOpCompletionEvent(PMM_PAGEOP PageOp) -{ - LARGE_INTEGER Timeout; -#ifdef __GNUC__ /* TODO: Use other macro to check for suffix to use? */ - - Timeout.QuadPart = -100000000LL; // 10 sec -#else - - Timeout.QuadPart = -100000000; // 10 sec -#endif - - return KeWaitForSingleObject(&PageOp->CompletionEvent, 0, KernelMode, FALSE, &Timeout); -} - - -/* - * FUNCTION: Sets the page op completion event and releases the page op. - * ARGUMENTS: PMM_PAGEOP. - * RETURNS: In shorter time than it takes you to even read this - * description, so don't even think about geting a mug of coffee. - */ -static void -MmspCompleteAndReleasePageOp(PMM_PAGEOP PageOp) -{ - KeSetEvent(&PageOp->CompletionEvent, IO_NO_INCREMENT, FALSE); - MmReleasePageOp(PageOp); -} - - /* * FUNCTION: Waits in kernel mode indefinitely for a file object lock. * ARGUMENTS: PFILE_OBJECT to wait for. @@ -831,12 +795,12 @@ PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset, BOOLEAN Dirty, - BOOLEAN PageOut) -{ - ULONG_PTR Entry; + BOOLEAN PageOut, + ULONG_PTR *InEntry) +{ + ULONG_PTR Entry = InEntry ? *InEntry : MmGetPageEntrySectionSegment(Segment, Offset); BOOLEAN IsDirectMapped = FALSE;
- Entry = MmGetPageEntrySectionSegment(Segment, Offset); if (Entry == 0) { DPRINT1("Entry == 0 for MmUnsharePageEntrySectionSegment\n"); @@ -844,7 +808,7 @@ } if (SHARE_COUNT_FROM_SSE(Entry) == 0) { - DPRINT1("Zero share count for unshare\n"); + DPRINT1("Zero share count for unshare (Seg %p Offset %x Page %x)\n", Segment, Offset->LowPart, PFN_FROM_SSE(Entry)); KeBugCheck(MEMORY_MANAGEMENT); } if (IS_SWAP_FROM_SSE(Entry)) @@ -913,10 +877,14 @@ * page without a rmap entry. */ MmSetPageEntrySectionSegment(Segment, Offset, Entry); + if (InEntry) *InEntry = Entry; + MiSetPageEvent(NULL, NULL); } else { MmSetPageEntrySectionSegment(Segment, Offset, 0); + if (InEntry) *InEntry = 0; + MiSetPageEvent(NULL, NULL); if (!IsDirectMapped) { MmReleasePageMemoryConsumer(MC_USER, Page); @@ -946,7 +914,9 @@ } } MmSetPageEntrySectionSegment(Segment, Offset, MAKE_SWAP_SSE(SavedSwapEntry)); + if (InEntry) *InEntry = MAKE_SWAP_SSE(SavedSwapEntry); MmSetSavedSwapEntryPage(Page, 0); + MiSetPageEvent(NULL, NULL); } MmReleasePageMemoryConsumer(MC_USER, Page); } @@ -959,7 +929,10 @@ } else { - MmSetPageEntrySectionSegment(Segment, Offset, Entry); + if (InEntry) + *InEntry = Entry; + else + MmSetPageEntrySectionSegment(Segment, Offset, Entry); } return(SHARE_COUNT_FROM_SSE(Entry) > 0); } @@ -1232,7 +1205,6 @@ ULONG_PTR Entry; ULONG_PTR Entry1; ULONG Attributes; - PMM_PAGEOP PageOp; PMM_REGION Region; BOOLEAN HasSwapEntry; PVOID PAddress; @@ -1270,7 +1242,7 @@ * Lock the segment */ MmLockSectionSegment(Segment); - + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); /* * Check if this page needs to be mapped COW */ @@ -1286,111 +1258,41 @@ }
/* - * Get or create a page operation descriptor - */ - PageOp = MmGetPageOp(MemoryArea, NULL, 0, Segment, Offset.LowPart, MM_PAGEOP_PAGEIN, FALSE); - if (PageOp == NULL) - { - DPRINT1("MmGetPageOp failed\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - - /* * Check if someone else is already handling this fault, if so wait * for them */ - if (PageOp->Thread != PsGetCurrentThread()) + if (Entry && IS_SWAP_FROM_SSE(Entry) && SWAPENTRY_FROM_SSE(Entry) == MM_WAIT_ENTRY) { MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); - Status = MmspWaitForPageOpCompletionEvent(PageOp); + MiWaitForPageEvent(NULL, NULL); + MmLockAddressSpace(AddressSpace); + DPRINT("Address 0x%.8X\n", Address); + return(STATUS_MM_RESTART_OPERATION); + } + + HasSwapEntry = MmIsPageSwapEntry(Process, Address); + + if (HasSwapEntry) + { + SWAPENTRY SwapEntry; /* - * Check for various strange conditions + * Is it a wait entry? */ - if (Status != STATUS_SUCCESS) - { - DPRINT1("Failed to wait for page op, status = %x\n", Status); - KeBugCheck(MEMORY_MANAGEMENT); - } - if (PageOp->Status == STATUS_PENDING) - { - DPRINT1("Woke for page op before completion\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - MmLockAddressSpace(AddressSpace); - /* - * If this wasn't a pagein then restart the operation - */ - if (PageOp->OpType != MM_PAGEOP_PAGEIN) - { - MmspCompleteAndReleasePageOp(PageOp); - DPRINT("Address 0x%.8X\n", Address); - return(STATUS_MM_RESTART_OPERATION); - } - - /* - * If the thread handling this fault has failed then we don't retry - */ - if (!NT_SUCCESS(PageOp->Status)) - { - Status = PageOp->Status; - MmspCompleteAndReleasePageOp(PageOp); - DPRINT("Address 0x%.8X\n", Address); - return(Status); - } - MmLockSectionSegment(Segment); - /* - * If the completed fault was for another address space then set the - * page in this one. - */ - if (!MmIsPagePresent(Process, Address)) - { - Entry = MmGetPageEntrySectionSegment(Segment, &Offset); - HasSwapEntry = MmIsPageSwapEntry(Process, (PVOID)PAddress); - - if (PAGE_FROM_SSE(Entry) == 0 || HasSwapEntry) - { - /* - * The page was a private page in another or in our address space - */ - MmUnlockSectionSegment(Segment); - MmspCompleteAndReleasePageOp(PageOp); - return(STATUS_MM_RESTART_OPERATION); - } - - Page = PFN_FROM_SSE(Entry); - - MmSharePageEntrySectionSegment(Segment, &Offset); - - /* FIXME: Should we call MmCreateVirtualMappingUnsafe if - * (Section->AllocationAttributes & SEC_PHYSICALMEMORY) is true? - */ - Status = MmCreateVirtualMapping(Process, - PAddress, - Attributes, - &Page, - 1); - if (!NT_SUCCESS(Status)) - { - DPRINT1("Unable to create virtual mapping\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - MmInsertRmap(Page, Process, Address); - } - MmUnlockSectionSegment(Segment); - PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); - DPRINT("Address 0x%.8X\n", Address); - return(STATUS_SUCCESS); - } - - HasSwapEntry = MmIsPageSwapEntry(Process, Address); - if (HasSwapEntry) - { + MmGetPageFileMapping(Process, Address, &SwapEntry); + + if (SwapEntry == MM_WAIT_ENTRY) + { + MmUnlockSectionSegment(Segment); + MmUnlockAddressSpace(AddressSpace); + MiWaitForPageEvent(NULL, NULL); + MmLockAddressSpace(AddressSpace); + return STATUS_MM_RESTART_OPERATION; + } + /* * Must be private page we have swapped out. */ - SWAPENTRY SwapEntry;
/* * Sanity check @@ -1445,8 +1347,7 @@ /* * Finish the operation */ - PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); + MiSetPageEvent(Process, Address); DPRINT("Address 0x%.8X\n", Address); return(STATUS_SUCCESS); } @@ -1476,8 +1377,7 @@ /* * Cleanup and release locks */ - PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); + MiSetPageEvent(Process, Address); DPRINT("Address 0x%.8X\n", Address); return(STATUS_SUCCESS); } @@ -1518,8 +1418,7 @@ /* * Cleanup and release locks */ - PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); + MiSetPageEvent(Process, Address); DPRINT("Address 0x%.8X\n", Address); return(STATUS_SUCCESS); } @@ -1539,6 +1438,7 @@ /* * Release all our locks and read in the page from disk */ + MmSetPageEntrySectionSegment(Segment, &Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace);
@@ -1573,11 +1473,64 @@ * Cleanup and release locks */ MmLockAddressSpace(AddressSpace); - PageOp->Status = Status; - MmspCompleteAndReleasePageOp(PageOp); + MiSetPageEvent(Process, Address); DPRINT("Address 0x%.8X\n", Address); return(Status); } + + /* + * Mark the offset within the section as having valid, in-memory + * data + */ + MmLockAddressSpace(AddressSpace); + MmLockSectionSegment(Segment); + Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); + MmSetPageEntrySectionSegment(Segment, &Offset, Entry); + MmUnlockSectionSegment(Segment); + + Status = MmCreateVirtualMapping(Process, + PAddress, + Attributes, + &Page, + 1); + if (!NT_SUCCESS(Status)) + { + DPRINT1("Unable to create virtual mapping\n"); + KeBugCheck(MEMORY_MANAGEMENT); + } + MmInsertRmap(Page, Process, Address); + + MiSetPageEvent(Process, Address); + DPRINT("Address 0x%.8X\n", Address); + return(STATUS_SUCCESS); + } + else if (IS_SWAP_FROM_SSE(Entry)) + { + SWAPENTRY SwapEntry; + + SwapEntry = SWAPENTRY_FROM_SSE(Entry); + + /* + * Release all our locks and read in the page from disk + */ + MmUnlockSectionSegment(Segment); + + MmUnlockAddressSpace(AddressSpace); + MI_SET_USAGE(MI_USAGE_SECTION); + if (Process) MI_SET_PROCESS2(Process->ImageFileName); + if (!Process) MI_SET_PROCESS2("Kernel Section"); + Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &Page); + if (!NT_SUCCESS(Status)) + { + KeBugCheck(MEMORY_MANAGEMENT); + } + + Status = MmReadFromSwapPage(SwapEntry, Page); + if (!NT_SUCCESS(Status)) + { + KeBugCheck(MEMORY_MANAGEMENT); + } + /* * Relock the address space and segment */ @@ -1591,76 +1544,7 @@ Entry1 = MmGetPageEntrySectionSegment(Segment, &Offset); if (Entry != Entry1) { - DPRINT1("Someone changed ppte entry while we slept\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - - /* - * Mark the offset within the section as having valid, in-memory - * data - */ - Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); - MmSetPageEntrySectionSegment(Segment, &Offset, Entry); - MmUnlockSectionSegment(Segment); - - Status = MmCreateVirtualMapping(Process, - PAddress, - Attributes, - &Page, - 1); - if (!NT_SUCCESS(Status)) - { - DPRINT1("Unable to create virtual mapping\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - MmInsertRmap(Page, Process, Address); - - PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); - DPRINT("Address 0x%.8X\n", Address); - return(STATUS_SUCCESS); - } - else if (IS_SWAP_FROM_SSE(Entry)) - { - SWAPENTRY SwapEntry; - - SwapEntry = SWAPENTRY_FROM_SSE(Entry); - - /* - * Release all our locks and read in the page from disk - */ - MmUnlockSectionSegment(Segment); - - MmUnlockAddressSpace(AddressSpace); - MI_SET_USAGE(MI_USAGE_SECTION); - if (Process) MI_SET_PROCESS2(Process->ImageFileName); - if (!Process) MI_SET_PROCESS2("Kernel Section"); - Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &Page); - if (!NT_SUCCESS(Status)) - { - KeBugCheck(MEMORY_MANAGEMENT); - } - - Status = MmReadFromSwapPage(SwapEntry, Page); - if (!NT_SUCCESS(Status)) - { - KeBugCheck(MEMORY_MANAGEMENT); - } - - /* - * Relock the address space and segment - */ - MmLockAddressSpace(AddressSpace); - MmLockSectionSegment(Segment); - - /* - * Check the entry. No one should change the status of a page - * that has a pending page-in. - */ - Entry1 = MmGetPageEntrySectionSegment(Segment, &Offset); - if (Entry != Entry1) - { - DPRINT1("Someone changed ppte entry while we slept\n"); + DPRINT1("Someone changed ppte entry while we slept (%x vs %x)\n", Entry, Entry1); KeBugCheck(MEMORY_MANAGEMENT); }
@@ -1687,8 +1571,7 @@ KeBugCheck(MEMORY_MANAGEMENT); } MmInsertRmap(Page, Process, Address); - PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); + MiSetPageEvent(Process, Address); DPRINT("Address 0x%.8X\n", Address); return(STATUS_SUCCESS); } @@ -1715,8 +1598,7 @@ KeBugCheck(MEMORY_MANAGEMENT); } MmInsertRmap(Page, Process, Address); - PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); + MiSetPageEvent(Process, Address); DPRINT("Address 0x%.8X\n", Address); return(STATUS_SUCCESS); } @@ -1735,7 +1617,6 @@ NTSTATUS Status; PVOID PAddress; LARGE_INTEGER Offset; - PMM_PAGEOP PageOp; PMM_REGION Region; ULONG_PTR Entry; PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); @@ -1799,46 +1680,29 @@ /* * Get or create a pageop */ - PageOp = MmGetPageOp(MemoryArea, NULL, 0, Segment, Offset.LowPart, - MM_PAGEOP_ACCESSFAULT, FALSE); - if (PageOp == NULL) - { - DPRINT1("MmGetPageOp failed\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - + MmLockSectionSegment(Segment); + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); + /* * Wait for any other operations to complete */ - if (PageOp->Thread != PsGetCurrentThread()) - { - MmUnlockAddressSpace(AddressSpace); - Status = MmspWaitForPageOpCompletionEvent(PageOp); - /* - * Check for various strange conditions - */ - if (Status == STATUS_TIMEOUT) - { - DPRINT1("Failed to wait for page op, status = %x\n", Status); - KeBugCheck(MEMORY_MANAGEMENT); - } - if (PageOp->Status == STATUS_PENDING) - { - DPRINT1("Woke for page op before completion\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - /* - * Restart the operation - */ - MmLockAddressSpace(AddressSpace); - MmspCompleteAndReleasePageOp(PageOp); - DPRINT("Address 0x%.8X\n", Address); - return(STATUS_MM_RESTART_OPERATION); + if (Entry == SWAPENTRY_FROM_SSE(MM_WAIT_ENTRY)) + { + MmUnlockSectionSegment(Segment); + MmUnlockAddressSpace(AddressSpace); + MiWaitForPageEvent(NULL, NULL); + /* + * Restart the operation + */ + MmLockAddressSpace(AddressSpace); + DPRINT("Address 0x%.8X\n", Address); + return(STATUS_MM_RESTART_OPERATION); }
/* * Release locks now we have the pageop */ + MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace);
/* @@ -1890,11 +1754,10 @@ MmDeleteRmap(OldPage, Process, PAddress); MmInsertRmap(NewPage, Process, PAddress); MmLockSectionSegment(Segment); - MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, FALSE, FALSE); + MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, FALSE, FALSE, NULL); MmUnlockSectionSegment(Segment);
- PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); + MiSetPageEvent(Process, Address); DPRINT("Address 0x%.8X\n", Address); return(STATUS_SUCCESS); } @@ -1904,7 +1767,7 @@ { MM_SECTION_PAGEOUT_CONTEXT* PageOutContext; BOOLEAN WasDirty; - PFN_NUMBER Page; + PFN_NUMBER Page = 0;
PageOutContext = (MM_SECTION_PAGEOUT_CONTEXT*)Context; if (Process) @@ -1928,7 +1791,8 @@ PageOutContext->Segment, &PageOutContext->Offset, PageOutContext->WasDirty, - TRUE); + TRUE, + &PageOutContext->SectionEntry); MmUnlockSectionSegment(PageOutContext->Segment); } if (Process) @@ -1940,21 +1804,17 @@ { MmReleasePageMemoryConsumer(MC_USER, Page); } - - DPRINT("PhysicalAddress %x, Address %x\n", Page << PAGE_SHIFT, Address); }
NTSTATUS NTAPI MmPageOutSectionView(PMMSUPPORT AddressSpace, MEMORY_AREA* MemoryArea, - PVOID Address, - PMM_PAGEOP PageOp) + PVOID Address, ULONG_PTR Entry) { PFN_NUMBER Page; MM_SECTION_PAGEOUT_CONTEXT Context; SWAPENTRY SwapEntry; - ULONG_PTR Entry; ULONGLONG FileOffset; NTSTATUS Status; PFILE_OBJECT FileObject; @@ -1973,6 +1833,7 @@ */ Context.Segment = MemoryArea->Data.SectionData.Segment; Context.Section = MemoryArea->Data.SectionData.Section; + Context.SectionEntry = Entry; Context.CallingProcess = Process;
Context.Offset.QuadPart = (ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress @@ -1983,6 +1844,8 @@
FileObject = Context.Section->FileObject; DirectMapped = FALSE; + + MmLockSectionSegment(Context.Segment);
#ifndef NEWCC if (FileObject != NULL && @@ -2035,16 +1898,14 @@ { DPRINT("Cannot page out locked section page: 0x%p (RefCount: %d)\n", Page, MmGetReferenceCountPage(Page)); - PageOp->Status = STATUS_UNSUCCESSFUL; - MmspCompleteAndReleasePageOp(PageOp); + MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); + MmUnlockSectionSegment(Context.Segment); return STATUS_UNSUCCESSFUL; }
/* * Prepare the context structure for the rmap delete call. */ - MmLockSectionSegment(Context.Segment); - Entry = MmGetPageEntrySectionSegment(Context.Segment, &Context.Offset); MmUnlockSectionSegment(Context.Segment); Context.WasDirty = FALSE; if (Context.Segment->Image.Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA || @@ -2077,9 +1938,12 @@ }
MmDeleteAllRmaps(Page, (PVOID)&Context, MmPageOutDeleteMapping); - MmLockSectionSegment(Context.Segment); - Entry = MmGetPageEntrySectionSegment(Context.Segment, &Context.Offset); - MmUnlockSectionSegment(Context.Segment); + + /* Since we passed in a surrogate, we'll get back the page entry + * state in our context. This is intended to make intermediate + * decrements of share count not release the wait entry. + */ + Entry = Context.SectionEntry;
/* * If this wasn't a private page then we should have reduced the entry to @@ -2117,8 +1981,7 @@ MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry)); MmUnlockSectionSegment(Context.Segment); MmReleasePageMemoryConsumer(MC_USER, Page); - PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); + MiSetPageEvent(NULL, NULL); return(STATUS_SUCCESS); } } @@ -2140,8 +2003,7 @@ MmUnlockSectionSegment(Context.Segment); } MmReleasePageMemoryConsumer(MC_USER, Page); - PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); + MiSetPageEvent(NULL, NULL); return(STATUS_SUCCESS); } } @@ -2165,8 +2027,7 @@ KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)Bcb, (ULONG_PTR)FileOffset, (ULONG_PTR)Address); } #endif - PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); + MiSetPageEvent(NULL, NULL); return(STATUS_SUCCESS); } else if (!Context.WasDirty && !DirectMapped && !Context.Private) @@ -2178,13 +2039,12 @@ KeBugCheckEx(MEMORY_MANAGEMENT, SwapEntry, Page, (ULONG_PTR)Process, (ULONG_PTR)Address); } MmReleasePageMemoryConsumer(MC_USER, Page); - PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); + MiSetPageEvent(NULL, NULL); return(STATUS_SUCCESS); } else if (!Context.WasDirty && Context.Private && SwapEntry != 0) { - DPRINT1("Not dirty and private and not swapped (%p:%p)\n", Process, Address); + DPRINT("Not dirty and private and not swapped (%p:%p)\n", Process, Address); MmSetSavedSwapEntryPage(Page, 0); MmLockAddressSpace(AddressSpace); Status = MmCreatePageFileMapping(Process, @@ -2197,8 +2057,7 @@ KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)Process, (ULONG_PTR)Address, SwapEntry); } MmReleasePageMemoryConsumer(MC_USER, Page); - PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); + MiSetPageEvent(NULL, NULL); return(STATUS_SUCCESS); }
@@ -2253,8 +2112,7 @@ MmUnlockSectionSegment(Context.Segment); } MmUnlockAddressSpace(AddressSpace); - PageOp->Status = STATUS_UNSUCCESSFUL; - MmspCompleteAndReleasePageOp(PageOp); + MiSetPageEvent(NULL, NULL); return(STATUS_PAGEFILE_QUOTA); } } @@ -2299,8 +2157,7 @@ MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); } MmUnlockAddressSpace(AddressSpace); - PageOp->Status = STATUS_UNSUCCESSFUL; - MmspCompleteAndReleasePageOp(PageOp); + MiSetPageEvent(NULL, NULL); return(STATUS_UNSUCCESSFUL); }
@@ -2308,12 +2165,13 @@ * Otherwise we have succeeded. */ DPRINT("MM: Wrote section page 0x%.8X to swap!\n", Page << PAGE_SHIFT); - MmLockSectionSegment(Context.Segment); MmSetSavedSwapEntryPage(Page, 0); if (Context.Segment->Flags & MM_PAGEFILE_SEGMENT || Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) { + MmLockSectionSegment(Context.Segment); MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry)); + MmUnlockSectionSegment(Context.Segment); } else { @@ -2323,9 +2181,11 @@ if (Context.Private) { MmLockAddressSpace(AddressSpace); + MmLockSectionSegment(Context.Segment); Status = MmCreatePageFileMapping(Process, Address, SwapEntry); + MmUnlockSectionSegment(Context.Segment); MmUnlockAddressSpace(AddressSpace); if (!NT_SUCCESS(Status)) { @@ -2335,13 +2195,15 @@ } else { + MmLockAddressSpace(AddressSpace); + MmLockSectionSegment(Context.Segment); Entry = MAKE_SWAP_SSE(SwapEntry); MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - } - - MmUnlockSectionSegment(Context.Segment); - PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); + MmUnlockSectionSegment(Context.Segment); + MmUnlockAddressSpace(AddressSpace); + } + + MiSetPageEvent(NULL, NULL); return(STATUS_SUCCESS); }
@@ -2350,7 +2212,7 @@ MmWritePageSectionView(PMMSUPPORT AddressSpace, PMEMORY_AREA MemoryArea, PVOID Address, - PMM_PAGEOP PageOp) + ULONG PageEntry) { LARGE_INTEGER Offset; PROS_SECTION_OBJECT Section; @@ -2447,12 +2309,16 @@ */ if (DirectMapped && !Private) { + LARGE_INTEGER SOffset; ASSERT(SwapEntry == 0); + SOffset.QuadPart = Offset.QuadPart + Segment->Image.FileOffset; #ifndef NEWCC - CcRosMarkDirtyCacheSegment(Bcb, (ULONG)(Offset.QuadPart + Segment->Image.FileOffset)); + CcRosMarkDirtyCacheSegment(Bcb, Offset.LowPart); #endif - PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); + MmLockSectionSegment(Segment); + MmSetPageEntrySectionSegment(Segment, &Offset, PageEntry); + MmUnlockSectionSegment(Segment); + MiSetPageEvent(NULL, NULL); return(STATUS_SUCCESS); }
@@ -2465,8 +2331,7 @@ if (SwapEntry == 0) { MmSetDirtyAllRmaps(Page); - PageOp->Status = STATUS_UNSUCCESSFUL; - MmspCompleteAndReleasePageOp(PageOp); + MiSetPageEvent(NULL, NULL); return(STATUS_PAGEFILE_QUOTA); } MmSetSavedSwapEntryPage(Page, SwapEntry); @@ -2481,8 +2346,7 @@ DPRINT1("MM: Failed to write to swap page (Status was 0x%.8X)\n", Status); MmSetDirtyAllRmaps(Page); - PageOp->Status = STATUS_UNSUCCESSFUL; - MmspCompleteAndReleasePageOp(PageOp); + MiSetPageEvent(NULL, NULL); return(STATUS_UNSUCCESSFUL); }
@@ -2490,8 +2354,7 @@ * Otherwise we have succeeded. */ DPRINT("MM: Wrote section page 0x%.8X to swap!\n", Page << PAGE_SHIFT); - PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); + MiSetPageEvent(NULL, NULL); return(STATUS_SUCCESS); }
@@ -4039,8 +3902,6 @@ PBCB Bcb; LARGE_INTEGER Offset; SWAPENTRY SavedSwapEntry; - PMM_PAGEOP PageOp; - NTSTATUS Status; PROS_SECTION_OBJECT Section; PMM_SECTION_SEGMENT Segment; PMMSUPPORT AddressSpace; @@ -4057,27 +3918,18 @@ Section = MemoryArea->Data.SectionData.Section; Segment = MemoryArea->Data.SectionData.Segment;
- PageOp = MmCheckForPageOp(MemoryArea, NULL, NULL, Segment, Offset.LowPart); - - while (PageOp) + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); + while (Entry && IS_SWAP_FROM_SSE(Entry) && SWAPENTRY_FROM_SSE(Entry) == MM_WAIT_ENTRY) { MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace);
- Status = MmspWaitForPageOpCompletionEvent(PageOp); - if (Status != STATUS_SUCCESS) - { - DPRINT1("Failed to wait for page op, status = %x\n", Status); - KeBugCheck(MEMORY_MANAGEMENT); - } + MiWaitForPageEvent(NULL, NULL);
MmLockAddressSpace(AddressSpace); MmLockSectionSegment(Segment); - MmspCompleteAndReleasePageOp(PageOp); - PageOp = MmCheckForPageOp(MemoryArea, NULL, NULL, Segment, Offset.LowPart); - } - - Entry = MmGetPageEntrySectionSegment(Segment, &Offset); + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); + }
/* * For a dirty, datafile, non-private page mark it as dirty in the @@ -4136,7 +3988,7 @@ else { MmDeleteRmap(Page, Process, Address); - MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, Dirty, FALSE); + MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, Dirty, FALSE, NULL); } } } @@ -4208,9 +4060,7 @@ PMEMORY_AREA MemoryArea; PMMSUPPORT AddressSpace; PROS_SECTION_OBJECT Section; - PMM_PAGEOP PageOp; - ULONG_PTR Offset; - PVOID ImageBaseAddress = 0; + PVOID ImageBaseAddress = 0;
DPRINT("Opening memory area Process %x BaseAddress %x\n", Process, BaseAddress); @@ -4232,40 +4082,6 @@ }
MemoryArea->DeleteInProgress = TRUE; - - while (MemoryArea->PageOpCount) - { - Offset = PAGE_ROUND_UP((ULONG_PTR)MemoryArea->EndingAddress - (ULONG_PTR)MemoryArea->StartingAddress); - - while (Offset) - { - Offset -= PAGE_SIZE; - PageOp = MmCheckForPageOp(MemoryArea, NULL, NULL, - MemoryArea->Data.SectionData.Segment, - Offset + MemoryArea->Data.SectionData.ViewOffset.QuadPart); - if (PageOp) - { - MmUnlockAddressSpace(AddressSpace); - Status = MmspWaitForPageOpCompletionEvent(PageOp); - if (Status != STATUS_SUCCESS) - { - DPRINT1("Failed to wait for page op, status = %x\n", Status); - KeBugCheck(MEMORY_MANAGEMENT); - } - MmLockAddressSpace(AddressSpace); - MmspCompleteAndReleasePageOp(PageOp); - MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, - BaseAddress); - if (MemoryArea == NULL || - MemoryArea->Type != MEMORY_AREA_SECTION_VIEW) - { - MmUnlockAddressSpace(AddressSpace); - return STATUS_NOT_MAPPED_VIEW; - } - break; - } - } - }
Section = MemoryArea->Data.SectionData.Section;