[arty] 44281: Deprecate old separate lazy write thread Fix and enable MPW Fix page op's hash function (really, |?) Simplify section implementation to completely remove use of page ops in all section types. Yields a surprising speed improvement.
Author: arty Date: Tue Nov 24 18:45:24 2009 New Revision: 44281
URL: http://svn.reactos.org/svn/reactos?rev=44281&view=rev Log: Deprecate old separate lazy write thread Fix and enable MPW Fix page op's hash function (really, |?) Simplify section implementation to completely remove use of page ops in all section types. Yields a surprising speed improvement.
Modified: branches/arty-newcc/ntoskrnl/cache/fssup.c branches/arty-newcc/ntoskrnl/cache/lazyrite.c branches/arty-newcc/ntoskrnl/mm/balance.c branches/arty-newcc/ntoskrnl/mm/freelist.c branches/arty-newcc/ntoskrnl/mm/mpw.c branches/arty-newcc/ntoskrnl/mm/pageop.c branches/arty-newcc/ntoskrnl/mm/section/data.c branches/arty-newcc/ntoskrnl/mm/section/image.c branches/arty-newcc/ntoskrnl/mm/section/pagefile.c
Modified: branches/arty-newcc/ntoskrnl/cache/fssup.c URL: http://svn.reactos.org/svn/reactos/branches/arty-newcc/ntoskrnl/cache/fssup.... ============================================================================== --- branches/arty-newcc/ntoskrnl/cache/fssup.c [iso-8859-1] (original) +++ branches/arty-newcc/ntoskrnl/cache/fssup.c [iso-8859-1] Tue Nov 24 18:45:24 2009 @@ -17,12 +17,7 @@
PFSN_PREFETCHER_GLOBALS CcPfGlobals; extern LONG CcOutstandingDeletes; -extern KEVENT CcpLazyWriteEvent; extern KEVENT CcFinalizeEvent; -extern VOID NTAPI CcpUnmapThread(PVOID Unused); -extern VOID NTAPI CcpLazyWriteThread(PVOID Unused); -HANDLE CcUnmapThreadHandle, CcLazyWriteThreadHandle; -CLIENT_ID CcUnmapThreadId, CcLazyWriteThreadId;
typedef struct _NOCC_PRIVATE_CACHE_MAP { @@ -48,7 +43,6 @@
KeInitializeEvent(&CcDeleteEvent, SynchronizationEvent, FALSE); KeInitializeEvent(&CcFinalizeEvent, SynchronizationEvent, FALSE); - KeInitializeEvent(&CcpLazyWriteEvent, SynchronizationEvent, FALSE);
CcCacheBitmap->Buffer = ((PULONG)&CcCacheBitmap[1]); CcCacheBitmap->SizeOfBitMap = ROUND_UP(CACHE_NUM_SECTIONS, 32);
Modified: branches/arty-newcc/ntoskrnl/cache/lazyrite.c URL: http://svn.reactos.org/svn/reactos/branches/arty-newcc/ntoskrnl/cache/lazyri... ============================================================================== --- branches/arty-newcc/ntoskrnl/cache/lazyrite.c [iso-8859-1] (original) +++ branches/arty-newcc/ntoskrnl/cache/lazyrite.c [iso-8859-1] Tue Nov 24 18:45:24 2009 @@ -14,21 +14,15 @@
/* GLOBALS ********************************************************************/
-KEVENT CcpLazyWriteEvent; +extern KEVENT MpwCompleteEvent;
/* FUNCTIONS ******************************************************************/ - -VOID NTAPI -CcpLazyWriteThread(PVOID Unused) -{ - /* Not implemented */ -}
NTSTATUS NTAPI CcWaitForCurrentLazyWriterActivity(VOID) { - //KeWaitForSingleObject(&CcpLazyWriteEvent, Executive, KernelMode, FALSE, NULL); + KeWaitForSingleObject(&MpwCompleteEvent, Executive, KernelMode, FALSE, NULL); return STATUS_SUCCESS; }
Modified: branches/arty-newcc/ntoskrnl/mm/balance.c URL: http://svn.reactos.org/svn/reactos/branches/arty-newcc/ntoskrnl/mm/balance.c... ============================================================================== --- branches/arty-newcc/ntoskrnl/mm/balance.c [iso-8859-1] (original) +++ branches/arty-newcc/ntoskrnl/mm/balance.c [iso-8859-1] Tue Nov 24 18:45:24 2009 @@ -138,10 +138,7 @@ else { KeReleaseSpinLock(&AllocationListLock, OldIrql); - if(Consumer == MC_USER) MmRemoveLRUUserPage(Page); - OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); MmDereferencePage(Page); - KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); }
return(STATUS_SUCCESS);
Modified: branches/arty-newcc/ntoskrnl/mm/freelist.c URL: http://svn.reactos.org/svn/reactos/branches/arty-newcc/ntoskrnl/mm/freelist.... ============================================================================== --- branches/arty-newcc/ntoskrnl/mm/freelist.c [iso-8859-1] (original) +++ branches/arty-newcc/ntoskrnl/mm/freelist.c [iso-8859-1] Tue Nov 24 18:45:24 2009 @@ -99,6 +99,7 @@ return 0; } PageDescriptor = CONTAINING_RECORD(NextListEntry, PHYSICAL_PAGE, ListEntry); + MmReferencePage(PageDescriptor - MmPfnDatabase); ASSERT_PFN(PageDescriptor); KeReleaseQueuedSpinLock(LockQueuePfnLock, oldIrql); return PageDescriptor - MmPfnDatabase; @@ -137,10 +138,13 @@ NextListEntry = (PLIST_ENTRY)Page->ListEntry.Flink; if (NextListEntry == &UserPageListHead) { + MmDereferencePage(PreviousPfn); KeReleaseQueuedSpinLock(LockQueuePfnLock, oldIrql); return 0; } PageDescriptor = CONTAINING_RECORD(NextListEntry, PHYSICAL_PAGE, ListEntry); + MmReferencePage(PageDescriptor - MmPfnDatabase); + MmDereferencePage(PreviousPfn); KeReleaseQueuedSpinLock(LockQueuePfnLock, oldIrql); return PageDescriptor - MmPfnDatabase; } @@ -906,6 +910,7 @@ NTAPI MmDereferencePage(PFN_TYPE Pfn) { + KIRQL oldIrql; PPHYSICAL_PAGE Page;
DPRINT("MmDereferencePage(PhysicalAddress %x)\n", Pfn << PAGE_SHIFT); @@ -928,7 +933,12 @@ if (Page->ReferenceCount == 0) { MmAvailablePages++; - if (Page->Flags.Consumer == MC_USER) RemoveEntryList(&Page->ListEntry); + if (Page->Flags.Consumer == MC_USER) + { + oldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + RemoveEntryList(&Page->ListEntry); + KeReleaseQueuedSpinLock(LockQueuePfnLock, oldIrql); + } if (Page->RmapListHead != (LONG)NULL) { DPRINT1("Freeing page with rmap entries.\n");
Modified: branches/arty-newcc/ntoskrnl/mm/mpw.c URL: http://svn.reactos.org/svn/reactos/branches/arty-newcc/ntoskrnl/mm/mpw.c?rev... ============================================================================== --- branches/arty-newcc/ntoskrnl/mm/mpw.c [iso-8859-1] (original) +++ branches/arty-newcc/ntoskrnl/mm/mpw.c [iso-8859-1] Tue Nov 24 18:45:24 2009 @@ -18,7 +18,7 @@
HANDLE MpwThreadHandle; static CLIENT_ID MpwThreadId; -KEVENT MpwThreadEvent; +KEVENT MpwThreadEvent, MpwCompleteEvent; BOOLEAN MpwThreadShouldTerminate;
/* FUNCTIONS *****************************************************************/ @@ -81,12 +81,11 @@ }
PagesWritten = 0; -#if 0 /* * FIXME: MmWriteDirtyPages doesn't work correctly. */ MmWriteDirtyPages(128, &PagesWritten); -#endif + KeSetEvent(&MpwCompleteEvent, IO_NO_INCREMENT, TRUE); } }
@@ -99,6 +98,7 @@
MpwThreadShouldTerminate = FALSE; KeInitializeEvent(&MpwThreadEvent, SynchronizationEvent, FALSE); + KeInitializeEvent(&MpwCompleteEvent, SynchronizationEvent, FALSE);
Status = PsCreateSystemThread(&MpwThreadHandle, THREAD_ALL_ACCESS,
Modified: branches/arty-newcc/ntoskrnl/mm/pageop.c URL: http://svn.reactos.org/svn/reactos/branches/arty-newcc/ntoskrnl/mm/pageop.c?... ============================================================================== --- branches/arty-newcc/ntoskrnl/mm/pageop.c [iso-8859-1] (original) +++ branches/arty-newcc/ntoskrnl/mm/pageop.c [iso-8859-1] Tue Nov 24 18:45:24 2009 @@ -83,11 +83,11 @@ */ if (MArea->Type == MEMORY_AREA_SECTION_VIEW) { - Hash = (((ULONG_PTR)Segment) | (((ULONG_PTR)Offset) / PAGE_SIZE)); + Hash = (((ULONG_PTR)Segment) ^ (((ULONG_PTR)Offset) / PAGE_SIZE)); } else { - Hash = (((ULONG_PTR)Pid) | (((ULONG_PTR)Address) / PAGE_SIZE)); + Hash = (((ULONG_PTR)Pid) ^ (((ULONG_PTR)Address) / PAGE_SIZE)); } Hash = Hash % PAGEOP_HASH_TABLE_SIZE;
@@ -151,11 +151,11 @@ */ if (MArea->Type == MEMORY_AREA_SECTION_VIEW) { - Hash = (((ULONG_PTR)Segment) | (((ULONG_PTR)Offset) / PAGE_SIZE)); + Hash = (((ULONG_PTR)Segment) ^ (((ULONG_PTR)Offset) / PAGE_SIZE)); } else { - Hash = (((ULONG_PTR)Pid) | (((ULONG_PTR)Address) / PAGE_SIZE)); + Hash = (((ULONG_PTR)Pid) ^ (((ULONG_PTR)Address) / PAGE_SIZE)); } Hash = Hash % PAGEOP_HASH_TABLE_SIZE;
Modified: branches/arty-newcc/ntoskrnl/mm/section/data.c URL: http://svn.reactos.org/svn/reactos/branches/arty-newcc/ntoskrnl/mm/section/d... ============================================================================== --- branches/arty-newcc/ntoskrnl/mm/section/data.c [iso-8859-1] (original) +++ branches/arty-newcc/ntoskrnl/mm/section/data.c [iso-8859-1] Tue Nov 24 18:45:24 2009 @@ -2333,50 +2333,6 @@ return(STATUS_SUCCESS); }
-NTSTATUS -NTAPI -MiAwaitPageOps(PMMSUPPORT AddressSpace, PMEMORY_AREA MemoryArea, PVOID BaseAddress) -{ - NTSTATUS Status; - PMM_PAGEOP PageOp; - ULONG Offset; - - while (MemoryArea->PageOpCount) - { - Offset = PAGE_ROUND_UP((ULONG_PTR)MemoryArea->EndingAddress - (ULONG_PTR)MemoryArea->StartingAddress); - - while (Offset) - { - Offset -= PAGE_SIZE; - PageOp = MmCheckForPageOp(MemoryArea, NULL, NULL, - MemoryArea->Data.SectionData.Segment, - Offset); - if (PageOp) - { - MmUnlockAddressSpace(AddressSpace); - Status = MmspWaitForPageOpCompletionEvent(PageOp); - if (Status != STATUS_SUCCESS) - { - DPRINT1("Failed to wait for page op, status = %x\n", Status); - ASSERT(FALSE); - } - MmLockAddressSpace(AddressSpace); - MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, - BaseAddress); - if (MemoryArea == NULL || - MemoryArea->Type != MEMORY_AREA_SECTION_VIEW) - { - MmUnlockAddressSpace(AddressSpace); - DPRINT("STATUS_NOT_MAPPED_VIEW\n"); - return STATUS_NOT_MAPPED_VIEW; - } - break; - } - } - } - return STATUS_SUCCESS; -} - /* * @implemented */ @@ -2409,18 +2365,14 @@ if (MemoryArea->Type == MEMORY_AREA_IMAGE_SECTION) { MemoryArea->DeleteInProgress = TRUE; - Status = MiAwaitPageOps(AddressSpace, MemoryArea, BaseAddress); - if (NT_SUCCESS(Status)) - Status = MiUnmapImageSection(AddressSpace, MemoryArea, BaseAddress); + Status = MiUnmapImageSection(AddressSpace, MemoryArea, BaseAddress); } else if (MemoryArea->Type == MEMORY_AREA_PHYSICAL_MEMORY_SECTION || MemoryArea->Type == MEMORY_AREA_PAGE_FILE_SECTION || MemoryArea->Type == MEMORY_AREA_SECTION_VIEW) { MemoryArea->DeleteInProgress = TRUE; - Status = MiAwaitPageOps(AddressSpace, MemoryArea, BaseAddress); - if (NT_SUCCESS(Status)) - Status = MmUnmapViewOfSegment(AddressSpace, BaseAddress); + Status = MmUnmapViewOfSegment(AddressSpace, BaseAddress); } else {
Modified: branches/arty-newcc/ntoskrnl/mm/section/image.c URL: http://svn.reactos.org/svn/reactos/branches/arty-newcc/ntoskrnl/mm/section/i... ============================================================================== --- branches/arty-newcc/ntoskrnl/mm/section/image.c [iso-8859-1] (original) +++ branches/arty-newcc/ntoskrnl/mm/section/image.c [iso-8859-1] Tue Nov 24 18:45:24 2009 @@ -1848,8 +1848,6 @@ //DPRINT("MiUnmapImageSection @ %x\n", BaseAddress); MemoryArea->DeleteInProgress = TRUE;
- MiAwaitPageOps(AddressSpace, MemoryArea, BaseAddress); - Section = MemoryArea->Data.SectionData.Section;
if (Section->AllocationAttributes & SEC_IMAGE)
Modified: branches/arty-newcc/ntoskrnl/mm/section/pagefile.c URL: http://svn.reactos.org/svn/reactos/branches/arty-newcc/ntoskrnl/mm/section/p... ============================================================================== --- branches/arty-newcc/ntoskrnl/mm/section/pagefile.c [iso-8859-1] (original) +++ branches/arty-newcc/ntoskrnl/mm/section/pagefile.c [iso-8859-1] Tue Nov 24 18:45:24 2009 @@ -69,9 +69,7 @@ PROS_SECTION_OBJECT Section; PMM_SECTION_SEGMENT Segment; ULONG Entry; - ULONG Entry1; ULONG Attributes; - PMM_PAGEOP PageOp; PMM_REGION Region; BOOLEAN HasSwapEntry; PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); @@ -100,10 +98,14 @@ Region = MmFindRegion(MemoryArea->StartingAddress, &MemoryArea->Data.SectionData.RegionListHead, Address, NULL); + /* * Lock the segment */ MmLockSectionSegment(Segment); + + Entry = MiGetPageEntrySectionSegment(Segment, &Offset); + HasSwapEntry = MmIsPageSwapEntry(Process, (PVOID)PAddress);
/* * Check if this page needs to be mapped COW @@ -119,161 +121,38 @@ Attributes = Region->Protect; }
- /* - * Get or create a page operation descriptor - */ - PageOp = MmGetPageOp(MemoryArea, NULL, 0, Segment, Offset.QuadPart, MM_PAGEOP_PAGEIN, FALSE); - if (PageOp == NULL) - { - DPRINT1("MmGetPageOp failed\n"); - ASSERT(FALSE); - } - - /* - * Check if someone else is already handling this fault, if so wait - * for them - */ - if (PageOp->Thread != PsGetCurrentThread()) - { - MmUnlockSectionSegment(Segment); - MmUnlockAddressSpace(AddressSpace); - Status = MmspWaitForPageOpCompletionEvent(PageOp); - /* - * Check for various strange conditions - */ - if (Status != STATUS_SUCCESS) - { - DPRINT1("Failed to wait for page op, status = %x\n", Status); - ASSERT(FALSE); - } - if (PageOp->Status == STATUS_PENDING) - { - DPRINT1("Woke for page op before completion\n"); - ASSERT(FALSE); - } - MmLockAddressSpace(AddressSpace); - /* - * If this wasn't a pagein then restart the operation - */ - if (PageOp->OpType != MM_PAGEOP_PAGEIN) - { - MmspCompleteAndReleasePageOp(PageOp); - DPRINT("Address 0x%.8X\n", Address); - return(STATUS_MM_RESTART_OPERATION); - } - - /* - * If the thread handling this fault has failed then we don't retry - */ - if (!NT_SUCCESS(PageOp->Status)) - { - Status = PageOp->Status; - MmspCompleteAndReleasePageOp(PageOp); - DPRINT("Address 0x%.8X\n", Address); - return(Status); - } - MmLockSectionSegment(Segment); - /* - * If the completed fault was for another address space then set the - * page in this one. - */ - if (!MmIsPagePresent(Process, Address)) - { - DPRINT("!MmIsPagePresent(%p, %p)\n", Process, Address); - - Entry = MiGetPageEntrySectionSegment(Segment, &Offset); - HasSwapEntry = MmIsPageSwapEntry(Process, (PVOID)PAddress); - - if (PAGE_FROM_SSE(Entry) == 0 || HasSwapEntry) - { - /* - * The page was a private page in another or in our address space - */ - MmUnlockSectionSegment(Segment); - MmspCompleteAndReleasePageOp(PageOp); - return(STATUS_MM_RESTART_OPERATION); - } - - Page = PFN_FROM_SSE(Entry); - - MmSharePageEntrySectionSegment(Segment, &Offset); - - /* FIXME: Should we call MmCreateVirtualMappingUnsafe if - * (Section->AllocationAttributes & SEC_PHYSICALMEMORY) is true? - */ - Status = MmCreateVirtualMapping(Process, - Address, - Attributes, - &Page, - 1); - if (!NT_SUCCESS(Status)) - { - DPRINT1("Unable to create virtual mapping\n"); - ASSERT(FALSE); - } - MmInsertRmap(Page, Process, (PVOID)PAddress); - } - if (Locked) - { - MmLockPage(Page); - } - MmUnlockSectionSegment(Segment); - PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); - DPRINT("Address 0x%.8X\n", Address); - return(STATUS_SUCCESS); - } - - /* - * Get the entry corresponding to the offset within the section - */ - Entry = MiGetPageEntrySectionSegment(Segment, &Offset); - if (Entry == 0) { - /* - * If the entry is zero (and it can't change because we have - * locked the segment) then we need to load the page. - */ - - /* - * Release all our locks and read in the page from disk - */ - MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); - Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &Page); if (!NT_SUCCESS(Status)) { DPRINT1("MmRequestPageMemoryConsumer failed (Status %x)\n", Status); + MmLockAddressSpace(AddressSpace); + MmUnlockSectionSegment(Segment); + return Status; } + + DPRINT("Allocated page %x\n", Page); Status = MmCreateVirtualMapping(Process, - Address, + PAddress, Attributes, &Page, 1); if (!NT_SUCCESS(Status)) { DPRINT1("Unable to create virtual mapping\n"); - ASSERT(FALSE); + MmReleasePageMemoryConsumer(MC_USER, Page); + DPRINT("Release page %x\n", Page); + MmLockAddressSpace(AddressSpace); + MmUnlockSectionSegment(Segment); + return Status; }
/* * Relock the address space and segment */ MmLockAddressSpace(AddressSpace); - MmLockSectionSegment(Segment); - - /* - * Check the entry. No one should change the status of a page - * that has a pending page-in. - */ - Entry1 = MiGetPageEntrySectionSegment(Segment, &Offset); - if (Entry != Entry1) - { - DPRINT1("Someone changed ppte entry while we slept\n"); - ASSERT(FALSE); - }
/* * Mark the offset within the section as having valid, in-memory @@ -281,16 +160,14 @@ */ Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); MiSetPageEntrySectionSegment(Segment, &Offset, Entry); - MmUnlockSectionSegment(Segment); - MmInsertRmap(Page, Process, (PVOID)PAddress);
if (Locked) { MmLockPage(Page); } - PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); + + MmUnlockSectionSegment(Segment); DPRINT("Address 0x%.8X\n", Address); return(STATUS_SUCCESS); } @@ -302,8 +179,6 @@ { MmLockPage(Page); } - PageOp->Status = Status; - MmspCompleteAndReleasePageOp(PageOp); return Status; } else @@ -333,8 +208,6 @@ { MmLockPage(Page); } - PageOp->Status = STATUS_SUCCESS; - MmspCompleteAndReleasePageOp(PageOp); DPRINT("Address 0x%.8X\n", Address); return(STATUS_SUCCESS); } @@ -699,8 +572,6 @@ PFN_TYPE Page, SWAPENTRY SwapEntry, BOOLEAN Dirty) { ULONG Entry; - PMM_PAGEOP PageOp; - NTSTATUS Status; LARGE_INTEGER Offset; PROS_SECTION_OBJECT Section; PMM_SECTION_SEGMENT Segment; @@ -717,26 +588,6 @@
Section = MemoryArea->Data.SectionData.Section; Segment = MemoryArea->Data.SectionData.Segment; - - PageOp = MmCheckForPageOp(MemoryArea, NULL, NULL, Segment, Offset.QuadPart); - - while (PageOp) - { - MmUnlockSectionSegment(Segment); - MmUnlockAddressSpace(AddressSpace); - - Status = MmspWaitForPageOpCompletionEvent(PageOp); - if (Status != STATUS_SUCCESS) - { - DPRINT1("Failed to wait for page op, status = %x\n", Status); - ASSERT(FALSE); - } - - MmLockAddressSpace(AddressSpace); - MmLockSectionSegment(Segment); - MmspCompleteAndReleasePageOp(PageOp); - PageOp = MmCheckForPageOp(MemoryArea, NULL, NULL, Segment, Offset.QuadPart); - }
DPRINT("MmFreeSectionPage -> MmGetPageEntrySectionSegment(%p, %x)\n", Segment, Offset.u.LowPart); Entry = MiGetPageEntrySectionSegment(Segment, &Offset);
-
arty＠svn.reactos.org