[jgardou] 64228: Try to repair catastrophic merge attempt from r64223, part 1 of 2 [NTOS/MM] Commit what I got so far, including: - A very (very) basic balancer for the paged pool, simply sweeping ...
Author: jgardou Date: Mon Sep 22 17:20:38 2014 New Revision: 64228
URL: http://svn.reactos.org/svn/reactos?rev=64228&view=rev Log: Try to repair catastrophic merge attempt from r64223, part 1 of 2 [NTOS/MM] Commit what I got so far, including: - A very (very) basic balancer for the paged pool, simply sweeping all the pages of it and ageing the pages as they're run over - A page writer thread writing unreferenced pages to the pagefile. - Support for reading from the pagefile in the ARM3 page fault handler - Popping a page from the standby list when running out of free pages - Various fixes here and there Enjoy!
Modified: branches/TransitionPte/ntoskrnl/include/internal/mm.h branches/TransitionPte/ntoskrnl/mm/ARM3/miarm.h branches/TransitionPte/ntoskrnl/mm/ARM3/pagfault.c branches/TransitionPte/ntoskrnl/mm/ARM3/pfnlist.c branches/TransitionPte/ntoskrnl/mm/ARM3/section.c branches/TransitionPte/ntoskrnl/mm/ARM3/virtual.c branches/TransitionPte/ntoskrnl/mm/balance.c branches/TransitionPte/ntoskrnl/mm/mminit.c branches/TransitionPte/ntoskrnl/mm/pagefile.c
Modified: branches/TransitionPte/ntoskrnl/include/internal/mm.h URL: http://svn.reactos.org/svn/reactos/branches/TransitionPte/ntoskrnl/include/i... ============================================================================== --- branches/TransitionPte/ntoskrnl/include/internal/mm.h [iso-8859-1] (original) +++ branches/TransitionPte/ntoskrnl/include/internal/mm.h [iso-8859-1] Mon Sep 22 17:20:38 2014 @@ -373,6 +373,7 @@ extern MMPFNLIST MmFreePageListHead; extern MMPFNLIST MmStandbyPageListHead; extern MMPFNLIST MmModifiedPageListHead; +extern MMPFNLIST MmModifiedPageListByColor[1]; extern MMPFNLIST MmModifiedNoWritePageListHead;
typedef struct _MM_MEMORY_CONSUMER @@ -652,8 +653,27 @@ NTAPI MiReadPageFile( _In_ PFN_NUMBER Page, - _In_ ULONG PageFileIndex, - _In_ ULONG_PTR PageFileOffset); + _In_ const MMPTE* PointerPte +); + +NTSTATUS +NTAPI +MiWritePageFile( + _In_ PFN_NUMBER Page, + _In_ const MMPTE* PointerPte +); + +VOID +NTAPI +MiFreePageFileEntry( + _In_ PMMPTE PointerPte +); + +NTSTATUS +NTAPI +MiReservePageFileEntry( + _Out_ PMMPTE PointerPte +);
/* process.c ****************************************************************/
Modified: branches/TransitionPte/ntoskrnl/mm/ARM3/miarm.h URL: http://svn.reactos.org/svn/reactos/branches/TransitionPte/ntoskrnl/mm/ARM3/m... ============================================================================== --- branches/TransitionPte/ntoskrnl/mm/ARM3/miarm.h [iso-8859-1] (original) +++ branches/TransitionPte/ntoskrnl/mm/ARM3/miarm.h [iso-8859-1] Mon Sep 22 17:20:38 2014 @@ -687,6 +687,7 @@ extern PKEVENT MiHighPagedPoolEvent; extern PKEVENT MiLowNonPagedPoolEvent; extern PKEVENT MiHighNonPagedPoolEvent; +extern KEVENT MpwThreadEvent; extern PFN_NUMBER MmLowMemoryThreshold; extern PFN_NUMBER MmHighMemoryThreshold; extern PFN_NUMBER MiLowPagedPoolThreshold; @@ -695,6 +696,7 @@ extern PFN_NUMBER MiHighNonPagedPoolThreshold; extern PFN_NUMBER MmMinimumFreePages; extern PFN_NUMBER MmPlentyFreePages; +extern ULONG_PTR MmTotalPagesForPagingFile; extern SIZE_T MmMinimumStackCommitInBytes; extern PFN_COUNT MiExpansionPoolPagesInitialCharge; extern PFN_NUMBER MmResidentAvailablePages;
Modified: branches/TransitionPte/ntoskrnl/mm/ARM3/pagfault.c URL: http://svn.reactos.org/svn/reactos/branches/TransitionPte/ntoskrnl/mm/ARM3/p... ============================================================================== --- branches/TransitionPte/ntoskrnl/mm/ARM3/pagfault.c [iso-8859-1] (original) +++ branches/TransitionPte/ntoskrnl/mm/ARM3/pagfault.c [iso-8859-1] Mon Sep 22 17:20:38 2014 @@ -583,7 +583,7 @@ ASSERT(PointerPte->u.Hard.Valid == 0);
/* Assert we have enough pages */ - ASSERT(MmAvailablePages >= 32); + ASSERT(MmAvailablePages != 0);
#if MI_TRACE_PFNS if (UserPdeFault) MI_SET_USAGE(MI_USAGE_PAGE_TABLE); @@ -688,7 +688,8 @@ IN PMMPTE PointerPte, IN PMMPTE PointerProtoPte, IN KIRQL OldIrql, - IN PMMPFN* LockedProtoPfn) + IN PMMPFN* LockedProtoPfn, + PVOID InPageBlock) { MMPTE TempPte; PMMPTE OriginalPte, PageTablePte; @@ -799,6 +800,10 @@
/* Reset the protection if needed */ if (OriginalProtection) Protection = MM_ZERO_ACCESS; + + /* Let waiters go ahead */ + if (InPageBlock) + KeSetEvent(InPageBlock, IO_NO_INCREMENT, FALSE);
/* Return success */ ASSERT(PointerPte == MiAddressToPte(Address)); @@ -816,26 +821,30 @@ ULONG Color; PFN_NUMBER Page; NTSTATUS Status; - MMPTE TempPte = *PointerPte; - KEVENT Event; + MMPTE PageFilePte = *PointerPte, TempPte; PMMPFN Pfn1; - ULONG PageFileIndex = TempPte.u.Soft.PageFileLow; - ULONG_PTR PageFileOffset = TempPte.u.Soft.PageFileHigh; + + DPRINT1("Serving page fault for PTE %p (%x).\n", PointerPte, PageFilePte.u.Long);
/* Things we don't support yet */ - ASSERT(CurrentProcess > HYDRA_PROCESS); ASSERT(*OldIrql != MM_NOIRQL);
/* We must hold the PFN lock */ ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
/* Some sanity checks */ - ASSERT(TempPte.u.Hard.Valid == 0); - ASSERT(TempPte.u.Soft.PageFileHigh != 0); - ASSERT(TempPte.u.Soft.PageFileHigh != MI_PTE_LOOKUP_NEEDED); + ASSERT(PageFilePte.u.Hard.Valid == 0); + ASSERT(PageFilePte.u.Soft.Transition == 0); + ASSERT(PageFilePte.u.Soft.Prototype == 0); + ASSERT(PageFilePte.u.Soft.PageFileHigh != 0); + ASSERT(PageFilePte.u.Soft.PageFileHigh != MI_PTE_LOOKUP_NEEDED);
/* Get any page, it will be overwritten */ - Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess); + if (CurrentProcess > HYDRA_PROCESS) + Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess); + else + Color = MI_GET_NEXT_COLOR(); + Page = MiRemoveAnyPage(Color);
/* Initialize this PFN */ @@ -847,14 +856,13 @@ ASSERT(Pfn1->u3.e1.ReadInProgress == 0); ASSERT(Pfn1->u3.e1.WriteInProgress == 0);
- KeInitializeEvent(&Event, NotificationEvent, FALSE); - Pfn1->u1.Event = &Event; + /* Set NULL event */ + Pfn1->u1.Event = NULL; Pfn1->u3.e1.ReadInProgress = 1;
/* We must write the PTE now as the PFN lock will be released while performing the IO operation */ + TempPte.u.Long = 0; TempPte.u.Soft.Transition = 1; - TempPte.u.Soft.PageFileLow = 0; - TempPte.u.Soft.Prototype = 0; TempPte.u.Trans.PageFrameNumber = Page;
MI_WRITE_INVALID_PTE(PointerPte, TempPte); @@ -863,15 +871,25 @@ KeReleaseQueuedSpinLock(LockQueuePfnLock, *OldIrql);
/* Do the paging IO */ - Status = MiReadPageFile(Page, PageFileIndex, PageFileOffset); + Status = MiReadPageFile(Page, &PageFilePte);
/* Lock the PFN database again */ *OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
/* Nobody should have changed that while we were not looking */ - ASSERT(Pfn1->u1.Event == &Event); ASSERT(Pfn1->u3.e1.ReadInProgress == 1); ASSERT(Pfn1->u3.e1.WriteInProgress == 0); + + /* But maybe someone began waiting on us */ + if (Pfn1->u1.Event != NULL) + { + DPRINT1("The page got a waiter!\n"); + /* Waiters gonna wait */ + KeSetEvent(Pfn1->u1.Event, IO_NO_INCREMENT, FALSE); + Pfn1->u1.Event = NULL; + } + + DPRINT1("Pagefile read finished. Status: 0x%08x.\n", Status);
if (!NT_SUCCESS(Status)) { @@ -879,38 +897,60 @@ ASSERT(FALSE); Pfn1->u4.InPageError = 1; Pfn1->u1.ReadStatus = Status; + Status = STATUS_IN_PAGE_ERROR; + } + else + { + /* This is a success status */ + Status = STATUS_PAGE_FAULT_PAGING_FILE; }
/* This is now a nice and normal PFN */ - Pfn1->u1.Event = NULL; Pfn1->u3.e1.ReadInProgress = 0;
- /* And the PTE can finally be valid */ - MI_MAKE_HARDWARE_PTE(&TempPte, PointerPte, TempPte.u.Trans.Protection, Page); + /* Check if this is a kernel or user address */ + if (PointerPte <= MiHighestUserPte) + { + /* Build the user PTE */ + MI_MAKE_HARDWARE_PTE_USER(&TempPte, + PointerPte, + PageFilePte.u.Soft.Protection, + Page); + } + else + { + /* Build the kernel PTE */ + MI_MAKE_HARDWARE_PTE(&TempPte, + PointerPte, + PageFilePte.u.Soft.Protection, + Page); + } + + /* Write it */ MI_WRITE_VALID_PTE(PointerPte, TempPte); - - /* Waiters gonna wait */ - KeSetEvent(&Event, IO_NO_INCREMENT, FALSE);
return Status; }
NTSTATUS NTAPI -MiResolveTransitionFault(IN PVOID FaultingAddress, - IN PMMPTE PointerPte, - IN PEPROCESS CurrentProcess, - IN KIRQL OldIrql, - OUT PVOID *InPageBlock) +MiResolveTransitionFault( + _In_ BOOLEAN StoreInstruction, + _In_ PVOID FaultingAddress, + _In_ PMMPTE PointerPte, + _In_ PEPROCESS CurrentProcess, + _In_ KIRQL OldIrql, + _In_ PVOID *InPageBlock) { PFN_NUMBER PageFrameIndex; PMMPFN Pfn1; MMPTE TempPte; PMMPTE PointerToPteForProtoPage; - DPRINT1("Transition fault on 0x%p with PTE 0x%p in process %s\n", - FaultingAddress, PointerPte, CurrentProcess->ImageFileName); - - /* Windowss does this check */ + + DPRINT("Transition fault on 0x%p with PTE 0x%p in process %s\n", + FaultingAddress, PointerPte, CurrentProcess ? CurrentProcess->ImageFileName : "Kernel"); + + /* Windows does this check */ ASSERT(*InPageBlock == NULL);
/* ARM3 doesn't support this path */ @@ -934,25 +974,52 @@ ASSERT(Pfn1->u4.InPageError == 0);
/* See if we should wait before terminating the fault */ - if (Pfn1->u3.e1.ReadInProgress == 1) - { - DPRINT1("The page is currently being read!\n"); - ASSERT(Pfn1->u1.Event != NULL); - *InPageBlock = Pfn1->u1.Event; - if (PointerPte == Pfn1->PteAddress) - { - DPRINT1("And this if for this particular PTE.\n"); - /* The PTE will be made valid by the thread serving the fault */ - return STATUS_SUCCESS; // FIXME: Maybe something more descriptive - } + if ((Pfn1->u3.e1.WriteInProgress == 1) || (Pfn1->u3.e1.ReadInProgress == 1)) + { + KEVENT IoEvent; + DPRINT1("The page is currently being written to or read!\n"); + + /* Both of them, it's not possible */ + ASSERT(((Pfn1->u3.e1.ReadInProgress == 0) && (Pfn1->u3.e1.WriteInProgress == 1)) || + ((Pfn1->u3.e1.ReadInProgress == 1) && (Pfn1->u3.e1.WriteInProgress == 0))); + + KeInitializeEvent(&IoEvent, NotificationEvent, FALSE); + *InPageBlock = Pfn1->u1.Event; + Pfn1->u1.Event = &IoEvent; + + /* + * Mark it as unmodified. + * The page-out thread will put it back in the standby list + * and we will be able to start anew for this PTE + */ + if ((Pfn1->u3.e1.WriteInProgress) && (!StoreInstruction)) + Pfn1->u3.e1.Modified = 0; + + /* Release the lock so the writer can finish */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + + /* Wait for it to really finish */ + KeWaitForSingleObject(&IoEvent, WrPageIn, KernelMode, FALSE, NULL); + + /* Get it again */ + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + + /* Read the PTE again */ + TempPte = *PointerPte; + if (TempPte.u.Hard.Valid == 1) + { + /* Someone did it while we were not looking. */ + return STATUS_PAGE_FAULT_TRANSITION; + } + /* Otherwise the state of this PTE must still be the same */ + ASSERT(TempPte.u.Soft.Transition == 1); + ASSERT(TempPte.u.Soft.Prototype == 0); + ASSERT(TempPte.u.Trans.PageFrameNumber == PageFrameIndex); }
/* Windows checks there's some free pages and this isn't an in-page error */ ASSERT(MmAvailablePages > 0); ASSERT(Pfn1->u4.InPageError == 0); - - /* ReactOS checks for this */ - ASSERT(MmAvailablePages > 32);
/* Was this a transition page in the valid list, or free/zero list? */ if (Pfn1->u3.e1.PageLocation == ActiveAndValid) @@ -1066,7 +1133,8 @@ PointerPte, PointerProtoPte, OldIrql, - OutPfn); + OutPfn, + NULL); }
/* Make sure there's some protection mask */ @@ -1125,18 +1193,25 @@ { /* Resolve the transition fault */ ASSERT(OldIrql != MM_NOIRQL); - Status = MiResolveTransitionFault(Address, + Status = MiResolveTransitionFault(StoreInstruction, + Address, PointerProtoPte, Process, OldIrql, &InPageBlock); ASSERT(NT_SUCCESS(Status)); } + else if (TempPte.u.Soft.PageFileHigh != 0) + { + Status = MiResolvePageFileFault(StoreInstruction, + Address, + PointerProtoPte, + Process, + &OldIrql); + ASSERT(NT_SUCCESS(Status)); + } else { - /* We also don't support paged out pages */ - ASSERT(TempPte.u.Soft.PageFileHigh == 0); - /* Resolve the demand zero fault */ Status = MiResolveDemandZeroFault(Address, PointerProtoPte, @@ -1152,7 +1227,8 @@ PointerPte, PointerProtoPte, OldIrql, - OutPfn); + OutPfn, + InPageBlock); }
NTSTATUS @@ -1173,9 +1249,8 @@ PMMPFN Pfn1, OutPfn = NULL; PFN_NUMBER PageFrameIndex; PFN_COUNT PteCount, ProcessedPtes; - DPRINT("ARM3 Page Fault Dispatcher for address: %p in process: %p\n", - Address, - Process); + DPRINT("ARM3 Page Fault Dispatcher for address: %p (PTE %p) in process: %p\n", + Address, PointerPte, Process);
/* Make sure the addresses are ok */ ASSERT(PointerPte == MiAddressToPte(Address)); @@ -1283,8 +1358,49 @@ ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
/* Should not yet happen in ReactOS */ - ASSERT(Pfn1->u3.e1.ReadInProgress == 0); ASSERT(Pfn1->u4.InPageError == 0); + + if ((Pfn1->u3.e1.ReadInProgress == 1) || (Pfn1->u3.e1.WriteInProgress == 1)) + { + KEVENT IoEvent; + PKEVENT PageEvent; + DPRINT1("The page is currently being written to or read!\n"); + + /* Both of them, it's not possible */ + ASSERT(((Pfn1->u3.e1.ReadInProgress == 0) && (Pfn1->u3.e1.WriteInProgress == 1)) || + ((Pfn1->u3.e1.ReadInProgress == 1) && (Pfn1->u3.e1.WriteInProgress == 0))); + + KeInitializeEvent(&IoEvent, NotificationEvent, FALSE); + PageEvent = Pfn1->u1.Event; + Pfn1->u1.Event = &IoEvent; + + /* + * Mark it as unmodified. + * The page-out thread will put it back in the standby list + * and we will be able to start anew for this PTE + */ + if (Pfn1->u3.e1.WriteInProgress) + Pfn1->u3.e1.Modified = 0; + + /* Release the lock so the writer can finish */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, LockIrql); + + /* Wait for it to really finish */ + KeWaitForSingleObject(&IoEvent, WrPageIn, KernelMode, FALSE, NULL); + + /* Get it again */ + LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + + /* Read the PTE again */ + TempPte = *PointerProtoPte; + + /* Maybe someone else was also waiting */ + if (PageEvent) + KeSetEvent(PageEvent, IO_NO_INCREMENT, FALSE); + + /* And re-loop */ + continue; + }
/* Get the page */ MiUnlinkPageFromList(Pfn1); @@ -1337,7 +1453,8 @@ PointerPte, PointerProtoPte, LockIrql, - &OutPfn); + &OutPfn, + NULL);
/* THIS RELEASES THE PFN LOCK! */ break; @@ -1390,14 +1507,14 @@ { /* We had a locked PFN, so acquire the PFN lock to dereference it */ ASSERT(PointerProtoPte != NULL); - OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
/* Dereference the locked PFN */ MiDereferencePfnAndDropLockCount(OutPfn); ASSERT(OutPfn->u3.e2.ReferenceCount >= 1);
/* And now release the lock */ - KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + KeReleaseQueuedSpinLock(LockQueuePfnLock, LockIrql); }
/* Complete this as a transition fault */ @@ -1416,7 +1533,12 @@ LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
/* Resolve */ - Status = MiResolveTransitionFault(Address, PointerPte, Process, LockIrql, &InPageBlock); + Status = MiResolveTransitionFault(StoreInstruction, + Address, + PointerPte, + Process, + LockIrql, + &InPageBlock);
NT_ASSERT(NT_SUCCESS(Status));
@@ -1425,8 +1547,8 @@
if (InPageBlock != NULL) { - /* The page is being paged in by another process */ - KeWaitForSingleObject(InPageBlock, WrPageIn, KernelMode, FALSE, NULL); + /* We must wake the blocked threads */ + KeSetEvent(InPageBlock, IO_NO_INCREMENT, FALSE); }
ASSERT(OldIrql == KeGetCurrentIrql()); @@ -1821,9 +1943,6 @@ } else { - /* We don't implement transition PTEs */ - ASSERT(TempPte.u.Soft.Transition == 0); - /* Check for no-access PTE */ if (TempPte.u.Soft.Protection == MM_NOACCESS) { @@ -2090,7 +2209,7 @@ OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
/* Make sure we have enough pages */ - ASSERT(MmAvailablePages >= 32); + ASSERT(MmAvailablePages != 0);
/* Try to get a zero page */ MI_SET_USAGE(MI_USAGE_PEB_TEB);
Modified: branches/TransitionPte/ntoskrnl/mm/ARM3/pfnlist.c URL: http://svn.reactos.org/svn/reactos/branches/TransitionPte/ntoskrnl/mm/ARM3/p... ============================================================================== --- branches/TransitionPte/ntoskrnl/mm/ARM3/pfnlist.c [iso-8859-1] (original) +++ branches/TransitionPte/ntoskrnl/mm/ARM3/pfnlist.c [iso-8859-1] Mon Sep 22 17:20:38 2014 @@ -36,7 +36,7 @@ ULONG MmSystemPageColor;
ULONG MmTransitionSharedPages; -ULONG MmTotalPagesForPagingFile; +ULONG_PTR MmTotalPagesForPagingFile;
MMPFNLIST MmZeroedPageListHead = {0, ZeroedPageList, LIST_HEAD, LIST_HEAD}; MMPFNLIST MmFreePageListHead = {0, FreePageList, LIST_HEAD, LIST_HEAD}; @@ -266,7 +266,6 @@ }
/* Decrease transition page counter */ - ASSERT(Pfn->u3.e1.PrototypePte == 1); /* Only supported ARM3 case */ MmTransitionSharedPages--;
/* One less page */ @@ -287,6 +286,9 @@ /* Decrement the counters */ ListHead->Total--; MmTotalPagesForPagingFile--; + + if (MmTotalPagesForPagingFile == 0) + KeClearEvent(&MpwThreadEvent);
/* Pick the correct colored list */ ListHead = &MmModifiedPageListByColor[0]; @@ -477,12 +479,89 @@ return PageIndex; }
+static +PFN_NUMBER +NTAPI +MiPopPageFromStandbyList(void) +{ + unsigned i; + PFN_NUMBER PageFrameIndex; + PMMPFN Pfn1; + PMMPTE PointerPte, SysPte, PteTable; + MMPTE PdePte; + + /* Make sure PFN lock is held and we have pages */ + ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); + ASSERT(MmAvailablePages != 0); + + /* This should be called in last resort if there are no more free pages */ + ASSERT(MmZeroedPageListHead.Total == 0); + ASSERT(MmFreePageListHead.Total == 0); + + /* Go by priority */ + for (i = 0; i < 8; i++) + { + /* Get the oldest one */ + PageFrameIndex = MmStandbyPageListByPriority[i].Blink; + if (PageFrameIndex != LIST_HEAD) + break; + } + + /* There are no free pages but still available ones, so this must be here */ + ASSERT(PageFrameIndex != LIST_HEAD); + + /* Get the PFN */ + Pfn1 = MI_PFN_ELEMENT(PageFrameIndex); + + DPRINT1("Popping page %x from standbylist (PTE %p).\n", + PageFrameIndex, Pfn1->PteAddress); + + /* Sanity checks */ + ASSERT(Pfn1->u3.e1.PageLocation == StandbyPageList); + ASSERT(Pfn1->u3.e1.Modified == 0); + + /* Get a PTE to map the page directory */ + SysPte = MiReserveSystemPtes(1, SystemPteSpace); + ASSERT(SysPte != NULL); + + /* Build it */ + MI_MAKE_HARDWARE_PTE_KERNEL(&PdePte, + SysPte, + MM_READWRITE, + Pfn1->u4.PteFrame); + /* And map it */ + MI_WRITE_VALID_PTE(SysPte, PdePte); + PteTable = MiPteToAddress(SysPte); + + /* Finally get a pointer to the PTE this page represents */ + PointerPte = &PteTable[MiAddressToPteOffset(MiPteToAddress(Pfn1->PteAddress))]; + + /* Other sanity checks */ + ASSERT(PointerPte->u.Hard.Valid == 0); + ASSERT(PointerPte->u.Soft.Transition == 1); + + /* Restore the PTE to its original value */ + *PointerPte = Pfn1->OriginalPte; + + /* We can get rid of this */ + MiReleaseSystemPtes(SysPte, 1, SystemPteSpace); + + /* And dereference the page table frame */ + MiDecrementShareCount(MI_PFN_ELEMENT(Pfn1->u4.PteFrame), Pfn1->u4.PteFrame); + + /* Unlink */ + MiUnlinkPageFromList(Pfn1); + + return PageFrameIndex; +} + PFN_NUMBER NTAPI MiRemoveAnyPage(IN ULONG Color) { PFN_NUMBER PageIndex; PMMPFN Pfn1; + BOOLEAN FromStandbyList = FALSE;
/* Make sure PFN lock is held and we have pages */ ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); @@ -507,23 +586,27 @@ ASSERT_LIST_INVARIANT(&MmZeroedPageListHead); PageIndex = MmZeroedPageListHead.Flink; Color = PageIndex & MmSecondaryColorMask; - ASSERT(PageIndex != LIST_HEAD); if (PageIndex == LIST_HEAD) { - /* FIXME: Should check the standby list */ ASSERT(MmZeroedPageListHead.Total == 0); + /* Pop one page from the standby list and try again */ + PageIndex = MiPopPageFromStandbyList(); + FromStandbyList = TRUE; + ASSERT(PageIndex != LIST_HEAD); } } } }
/* Remove the page from its list */ - PageIndex = MiRemovePageByColor(PageIndex, Color); + if (!FromStandbyList) + PageIndex = MiRemovePageByColor(PageIndex, Color);
/* Sanity checks */ Pfn1 = MI_PFN_ELEMENT(PageIndex); ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) || - (Pfn1->u3.e1.PageLocation == ZeroedPageList)); + (Pfn1->u3.e1.PageLocation == ZeroedPageList) || + (Pfn1->u3.e1.PageLocation == StandbyPageList)); ASSERT(Pfn1->u3.e2.ReferenceCount == 0); ASSERT(Pfn1->u2.ShareCount == 0); ASSERT_LIST_INVARIANT(&MmFreePageListHead); @@ -540,6 +623,7 @@ PFN_NUMBER PageIndex; PMMPFN Pfn1; BOOLEAN Zero = FALSE; + BOOLEAN FromStandbyList = FALSE;
/* Make sure PFN lock is held and we have pages */ ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); @@ -567,11 +651,13 @@ ASSERT_LIST_INVARIANT(&MmFreePageListHead); PageIndex = MmFreePageListHead.Flink; Color = PageIndex & MmSecondaryColorMask; - ASSERT(PageIndex != LIST_HEAD); if (PageIndex == LIST_HEAD) { - /* FIXME: Should check the standby list */ ASSERT(MmZeroedPageListHead.Total == 0); + /* Pop one page from the standby list and try again */ + PageIndex = MiPopPageFromStandbyList(); + FromStandbyList = TRUE; + ASSERT(PageIndex != LIST_HEAD); } } } @@ -584,10 +670,12 @@ /* Sanity checks */ Pfn1 = MI_PFN_ELEMENT(PageIndex); ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) || - (Pfn1->u3.e1.PageLocation == ZeroedPageList)); + (Pfn1->u3.e1.PageLocation == ZeroedPageList)|| + (Pfn1->u3.e1.PageLocation == StandbyPageList));
/* Remove the page from its list */ - PageIndex = MiRemovePageByColor(PageIndex, Color); + if (!FromStandbyList) + PageIndex = MiRemovePageByColor(PageIndex, Color); ASSERT(Pfn1 == MI_PFN_ELEMENT(PageIndex));
/* Zero it, if needed */ @@ -744,7 +832,6 @@ Pfn1 = MI_PFN_ELEMENT(PageFrameIndex); ASSERT(Pfn1->u4.MustBeCached == 0); ASSERT(Pfn1->u3.e2.ReferenceCount == 0); - ASSERT(Pfn1->u3.e1.PrototypePte == 1); ASSERT(Pfn1->u3.e1.Rom != 1);
/* One more transition page on a list */ @@ -979,9 +1066,8 @@ } else if (ListName == ModifiedPageList) { - /* In ARM3, page must be destined for page file, and not yet written out */ + /* In ARM3, page must be destined for page file */ ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0); - ASSERT(Pfn1->OriginalPte.u.Soft.PageFileHigh == 0);
/* One more transition page */ MmTransitionSharedPages++; @@ -989,7 +1075,8 @@ /* Increment the number of per-process modified pages */ PsGetCurrentProcess()->ModifiedPageCount++;
- /* FIXME: Wake up modified page writer if there are not enough free pages */ + /* Wake the page writer thread */ + KeSetEvent(&MpwThreadEvent, IO_NO_INCREMENT, FALSE); } else if (ListName == ModifiedNoWritePageList) { @@ -1059,6 +1146,9 @@ PageFrameIndex = PFN_FROM_PTE(PointerPtePte); ASSERT(PageFrameIndex != 0); Pfn1->u4.PteFrame = PageFrameIndex; + + /* HACK until we get working sets rolling */ + Pfn1->Wsle.u1.e1.Age = 0;
/* Increase its share count so we don't get rid of it */ Pfn1 = MI_PFN_ELEMENT(PageFrameIndex); @@ -1270,32 +1360,8 @@ /* PFN lock must be held */ ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
- if (Pfn1->u3.e2.ReferenceCount == 1) - { - /* Is there still a PFN for this page? */ - if (MI_IS_PFN_DELETED(Pfn1) == TRUE) - { - /* Clear the last reference */ - Pfn1->u3.e2.ReferenceCount = 0; - ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0); - - /* Mark the page temporarily as valid, we're going to make it free soon */ - Pfn1->u3.e1.PageLocation = ActiveAndValid; - - /* Bring it back into the free list */ - MiInsertPageInFreeList(PageFrameIndex); - } - else - { - /* PFN not yet deleted, drop a ref count */ - MiDecrementReferenceCount(Pfn1, PageFrameIndex); - } - } - else - { - /* Otherwise, just drop the reference count */ - InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount); - } + /* Drop a ref count */ + MiDecrementReferenceCount(Pfn1, PageFrameIndex); } }
@@ -1338,6 +1404,13 @@ /* Did someone set the delete flag? */ if (MI_IS_PFN_DELETED(Pfn1)) { + /* Did we have a Pagefile entry for it */ + if ((Pfn1->OriginalPte.u.Soft.Prototype == 0) + && (Pfn1->OriginalPte.u.Soft.PageFileHigh != 0)) + { + /* Yes. Set it free */ + MiFreePageFileEntry(&Pfn1->OriginalPte); + } /* Insert it into the free list, there's nothing left to do */ MiInsertPageInFreeList(PageFrameIndex); return;
Modified: branches/TransitionPte/ntoskrnl/mm/ARM3/section.c URL: http://svn.reactos.org/svn/reactos/branches/TransitionPte/ntoskrnl/mm/ARM3/s... ============================================================================== --- branches/TransitionPte/ntoskrnl/mm/ARM3/section.c [iso-8859-1] (original) +++ branches/TransitionPte/ntoskrnl/mm/ARM3/section.c [iso-8859-1] Mon Sep 22 17:20:38 2014 @@ -680,25 +680,55 @@ PageFrameIndex = PFN_FROM_PTE(&TempPte); Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
- /* As this is a paged-backed section, nobody should reference it anymore (no cache or whatever) */ - ASSERT(Pfn1->u3.ReferenceCount == 0); - - /* And it should be in standby or modified list */ - ASSERT((Pfn1->u3.e1.PageLocation == ModifiedPageList) || (Pfn1->u3.e1.PageLocation == StandbyPageList)); - - /* Unlink it and put it back in free list */ - MiUnlinkPageFromList(Pfn1); - - /* Temporarily mark this as active and make it free again */ - Pfn1->u3.e1.PageLocation = ActiveAndValid; - MI_SET_PFN_DELETED(Pfn1); - - MiInsertPageInFreeList(PageFrameIndex); + if (Pfn1->u3.e1.WriteInProgress == 1) + { + KEVENT WriteEvent; + + DPRINT1("Setting the page free while it is being paged out!\n"); + + /* Only the page writer thread should hold a reference to it */ + ASSERT(Pfn1->u3.ReferenceCount == 1); + + /* Set the page event */ + KeInitializeEvent(&WriteEvent, NotificationEvent, FALSE); + /* We must be the only ones waiting */ + ASSERT(Pfn1->u1.Event == NULL); + Pfn1->u1.Event = &WriteEvent; + + /* Tell the page-out thread to abort */ + Pfn1->u3.e1.WriteInProgress = 0; + /* We canceled the write operation, so the page-write-thread will simply dereference + * the Pfn. It has no way to know whether the page will be re-used again (page fault case) + * or if it should be made free. So mark it as deleted here */ + MI_SET_PFN_DELETED(Pfn1); + + /* Let the writer thread finish and go along */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + KeWaitForSingleObject(&WriteEvent, FreePage, KernelMode, FALSE, NULL); + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + } + else + { + /* As this is a paged-backed section, nobody should reference it anymore (no cache or whatever) */ + ASSERT(Pfn1->u3.ReferenceCount == 0); + + /* And it should be in standby or modified list */ + ASSERT((Pfn1->u3.e1.PageLocation == ModifiedPageList) || + (Pfn1->u3.e1.PageLocation == StandbyPageList)); + + /* Unlink it and temporarily take a reference */ + MiUnlinkPageFromList(Pfn1); + Pfn1->u3.e2.ReferenceCount++; + + /* This will put it back in free list and clean up properly */ + MI_SET_PFN_DELETED(Pfn1); + MiDecrementReferenceCount(Pfn1, PageFrameIndex); + } } else if (TempPte.u.Soft.PageFileHigh != 0) { - /* Should not happen for now */ - ASSERT(FALSE); + /* There isn't much to do */ + MiFreePageFileEntry(&TempPte); } } else
Modified: branches/TransitionPte/ntoskrnl/mm/ARM3/virtual.c URL: http://svn.reactos.org/svn/reactos/branches/TransitionPte/ntoskrnl/mm/ARM3/v... ============================================================================== --- branches/TransitionPte/ntoskrnl/mm/ARM3/virtual.c [iso-8859-1] (original) +++ branches/TransitionPte/ntoskrnl/mm/ARM3/virtual.c [iso-8859-1] Mon Sep 22 17:20:38 2014 @@ -317,7 +317,6 @@ { /* As always, only handle current ARM3 scenarios */ ASSERT(PointerPte->u.Soft.Prototype == 0); - ASSERT(PointerPte->u.Soft.Transition == 0);
/* Normally this is one possibility -- freeing a valid page */ if (PointerPte->u.Hard.Valid) @@ -345,6 +344,57 @@
/* Decrement the page table too */ MiDecrementShareCount(Pfn2, PageTableIndex); + + /* Release the PFN database */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + + /* Destroy the PTE */ + MI_ERASE_PTE(PointerPte); + } + else if (PointerPte->u.Soft.Transition == 1) + { + /* Get the page PFN */ + PageFrameIndex = PFN_FROM_PTE(PointerPte); + Pfn1 = MiGetPfnEntry(PageFrameIndex); + + /* The page should be out of any working set */ + ASSERT(Pfn1->u1.WsIndex == 0); + + /* Get the page table entry */ + PageTableIndex = Pfn1->u4.PteFrame; + Pfn2 = MiGetPfnEntry(PageTableIndex); + + /* Lock the PFN database */ + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + + /* Delete the page */ + MI_SET_PFN_DELETED(Pfn1); + + /* Decrement the page table */ + MiDecrementShareCount(Pfn2, PageTableIndex); + + ASSERT(Pfn1->u3.e2.ReferenceCount == 0); + /* Unlink it and temporarily take a reference */ + MiUnlinkPageFromList(Pfn1); + Pfn1->u3.e2.ReferenceCount++; + + /* This will put it back in free list and clean properly up */ + MI_SET_PFN_DELETED(Pfn1); + MiDecrementReferenceCount(Pfn1, PageFrameIndex); + + /* Release the PFN database */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + + /* Destroy the PTE */ + MI_ERASE_PTE(PointerPte); + } + else if (PointerPte->u.Soft.PageFileHigh != 0) + { + /* The PFN lock also protects the page files */ + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + + /* Pretty easy: free the page file slot and go look elsewhere */ + MiFreePageFileEntry(PointerPte);
/* Release the PFN database */ KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); @@ -358,10 +408,7 @@ * The only other ARM3 possibility is a demand zero page, which would * mean freeing some of the paged pool pages that haven't even been * touched yet, as part of a larger allocation. - * - * Right now, we shouldn't expect any page file information in the PTE */ - ASSERT(PointerPte->u.Soft.PageFileHigh == 0);
/* Destroy the PTE */ MI_ERASE_PTE(PointerPte); @@ -407,9 +454,8 @@ /* See if the PTE is valid */ if (TempPte.u.Hard.Valid == 0) { - /* Prototype and paged out PTEs not supported yet */ + /* Prototype PTEs not supported yet */ ASSERT(TempPte.u.Soft.Prototype == 0); - ASSERT((TempPte.u.Soft.PageFileHigh == 0) || (TempPte.u.Soft.Transition == 1));
if (TempPte.u.Soft.Transition) { @@ -435,15 +481,24 @@ /* And it should be in standby or modified list */ ASSERT((Pfn1->u3.e1.PageLocation == ModifiedPageList) || (Pfn1->u3.e1.PageLocation == StandbyPageList));
- /* Unlink it and temporarily mark it as active */ + /* Unlink it and temporarily take a reference */ MiUnlinkPageFromList(Pfn1); Pfn1->u3.e2.ReferenceCount++; - Pfn1->u3.e1.PageLocation = ActiveAndValid;
/* This will put it back in free list and clean properly up */ MI_SET_PFN_DELETED(Pfn1); MiDecrementReferenceCount(Pfn1, PageFrameIndex); } + return; + } + + if (TempPte.u.Soft.PageFileHigh != 0) + { + /* Release the page file entry */ + MiFreePageFileEntry(&TempPte); + + /* And that's pretty much it */ + MI_ERASE_PTE(PointerPte); return; } } @@ -475,7 +530,6 @@ } #endif /* Drop the share count on the page table */ - PointerPde = MiPteToPde(PointerPte); MiDecrementShareCount(MiGetPfnEntry(PointerPde->u.Hard.PageFrameNumber), PointerPde->u.Hard.PageFrameNumber);
Modified: branches/TransitionPte/ntoskrnl/mm/balance.c URL: http://svn.reactos.org/svn/reactos/branches/TransitionPte/ntoskrnl/mm/balanc... ============================================================================== --- branches/TransitionPte/ntoskrnl/mm/balance.c [iso-8859-1] (original) +++ branches/TransitionPte/ntoskrnl/mm/balance.c [iso-8859-1] Mon Sep 22 17:20:38 2014 @@ -22,22 +22,14 @@ #pragma alloc_text(INIT, MiInitBalancerThread) #endif
- -/* TYPES ********************************************************************/ -typedef struct _MM_ALLOCATION_REQUEST -{ - PFN_NUMBER Page; - LIST_ENTRY ListEntry; - KEVENT Event; -} -MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST; +#define TEST_PAGING + + /* GLOBALS ******************************************************************/
MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM]; static ULONG MiMinimumAvailablePages; static ULONG MiNrTotalPages; -static LIST_ENTRY AllocationListHead; -static KSPIN_LOCK AllocationListLock; static ULONG MiMinimumPagesPerRun;
static CLIENT_ID MiBalancerThreadId; @@ -53,8 +45,6 @@ MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages) { memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers)); - InitializeListHead(&AllocationListHead); - KeInitializeSpinLock(&AllocationListLock);
MiNrTotalPages = NrAvailablePages;
@@ -86,18 +76,10 @@ MiMemoryConsumers[Consumer].Trim = Trim; }
-VOID -NTAPI -MiZeroPhysicalPage( - IN PFN_NUMBER PageFrameIndex -); - NTSTATUS NTAPI MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page) { - PMM_ALLOCATION_REQUEST Request; - PLIST_ENTRY Entry; KIRQL OldIrql;
if (Page == 0) @@ -110,26 +92,11 @@ { if(Consumer == MC_USER) MmRemoveLRUUserPage(Page); (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed); - if ((Entry = ExInterlockedRemoveHeadList(&AllocationListHead, &AllocationListLock)) == NULL) - { - OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); - MmDereferencePage(Page); - KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); - } - else - { - Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry); - MiZeroPhysicalPage(Page); - Request->Page = Page; - KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE); - } } - else - { - OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); - MmDereferencePage(Page); - KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); - } + + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + MmDereferencePage(Page); + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
return(STATUS_SUCCESS); } @@ -149,10 +116,13 @@ return InitialTarget; }
- if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget) - { - /* Consumer page limit exceeded */ - Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget); + // if (MiFreeSwapPages == 0) + { + if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget) + { + /* Consumer page limit exceeded */ + Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget); + } } if (MmAvailablePages < MiMinimumAvailablePages) { @@ -162,13 +132,6 @@
if (Target) { - if (!InitialTarget) - { - /* If there was no initial target, - * swap at least MiMinimumPagesPerRun */ - Target = max(Target, MiMinimumPagesPerRun); - } - /* Now swap the pages out */ Status = MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
@@ -265,8 +228,9 @@ * Make sure we don't exceed our individual target. */ PagesUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed); - if (PagesUsed > MiMemoryConsumers[Consumer].PagesTarget && - !MiIsBalancerThread()) + if ((PagesUsed > MiMemoryConsumers[Consumer].PagesTarget) && + !MiIsBalancerThread() && + (MmNumberOfPagingFiles == 0)) { MmRebalanceMemoryConsumers(); } @@ -285,52 +249,6 @@ } if (Consumer == MC_USER) MmInsertLRULastUserPage(Page); *AllocatedPage = Page; - if (MmAvailablePages < MiMinimumAvailablePages) - MmRebalanceMemoryConsumers(); - return(STATUS_SUCCESS); - } - - /* - * Make sure we don't exceed global targets. - */ - if (MmAvailablePages < MiMinimumAvailablePages) - { - MM_ALLOCATION_REQUEST Request; - - if (!CanWait) - { - (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed); - MmRebalanceMemoryConsumers(); - return(STATUS_NO_MEMORY); - } - - /* Insert an allocation request. */ - Request.Page = 0; - KeInitializeEvent(&Request.Event, NotificationEvent, FALSE); - - ExInterlockedInsertTailList(&AllocationListHead, &Request.ListEntry, &AllocationListLock); - MmRebalanceMemoryConsumers(); - - KeWaitForSingleObject(&Request.Event, - 0, - KernelMode, - FALSE, - NULL); - - Page = Request.Page; - if (Page == 0) - { - KeBugCheck(NO_PAGES_AVAILABLE); - } - - if(Consumer == MC_USER) MmInsertLRULastUserPage(Page); - *AllocatedPage = Page; - - if (MmAvailablePages < MiMinimumAvailablePages) - { - MmRebalanceMemoryConsumers(); - } - return(STATUS_SUCCESS); }
@@ -347,91 +265,246 @@ if(Consumer == MC_USER) MmInsertLRULastUserPage(Page); *AllocatedPage = Page;
- if (MmAvailablePages < MiMinimumAvailablePages) - { - MmRebalanceMemoryConsumers(); - } - return(STATUS_SUCCESS); }
+/* Old implementation of the balancer thread */ +static +VOID +NTAPI +MiRosBalancerThread(VOID) +{ + ULONG InitialTarget = 0; + + DPRINT1("Legacy MM balancer in action!\n"); + +#if (_MI_PAGING_LEVELS == 2) + if (!MiIsBalancerThread()) + { + /* Clean up the unused PDEs */ + ULONG_PTR Address; + PEPROCESS Process = PsGetCurrentProcess(); + + /* Acquire PFN lock */ + KIRQL OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + PMMPDE pointerPde; + for (Address = (ULONG_PTR) MI_LOWEST_VAD_ADDRESS ; + Address < (ULONG_PTR) MM_HIGHEST_VAD_ADDRESS; Address += (PAGE_SIZE * PTE_COUNT)) + { + if (MiQueryPageTableReferences((PVOID) Address) == 0) + { + pointerPde = MiAddressToPde(Address); + if (pointerPde->u.Hard.Valid) + MiDeletePte(pointerPde, MiPdeToPte(pointerPde), Process, NULL); + ASSERT(pointerPde->u.Hard.Valid == 0); + } + } + /* Release lock */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + } +#endif + do + { + ULONG OldTarget = InitialTarget; + ULONG i; + + /* Trim each consumer */ + for (i = 0; i < MC_MAXIMUM; i++) + { + InitialTarget = MiTrimMemoryConsumer(i, InitialTarget); + } + + /* No pages left to swap! */ + if (InitialTarget != 0 && InitialTarget == OldTarget) + { + /* Game over */ + KeBugCheck(NO_PAGES_AVAILABLE); + } + } while (InitialTarget != 0); +}
VOID NTAPI MiBalancerThread(PVOID Unused) { - PVOID WaitObjects[2]; - NTSTATUS Status; - ULONG i; - - WaitObjects[0] = &MiBalancerEvent; - WaitObjects[1] = &MiBalancerTimer; - - while (1) - { - Status = KeWaitForMultipleObjects(2, - WaitObjects, - WaitAny, - Executive, - KernelMode, - FALSE, - NULL, - NULL); - - if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1) - { - ULONG InitialTarget = 0; - -#if (_MI_PAGING_LEVELS == 2) - if (!MiIsBalancerThread()) - { - /* Clean up the unused PDEs */ - ULONG_PTR Address; - PEPROCESS Process = PsGetCurrentProcess(); - - /* Acquire PFN lock */ - KIRQL OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); - PMMPDE pointerPde; - for (Address = (ULONG_PTR)MI_LOWEST_VAD_ADDRESS; - Address < (ULONG_PTR)MM_HIGHEST_VAD_ADDRESS; - Address += (PAGE_SIZE * PTE_COUNT)) - { - if (MiQueryPageTableReferences((PVOID)Address) == 0) + PVOID WaitObjects[2]; + NTSTATUS Status; + ULONG i; + PVOID LastPagedPoolAddress = MmPagedPoolStart; +#ifdef TEST_PAGING + PULONG PagingTestBuffer; + BOOLEAN TestPage = FALSE; +#endif + + WaitObjects[0] = &MiBalancerTimer; + WaitObjects[1] = &MiBalancerEvent; + +#ifdef TEST_PAGING + PagingTestBuffer = ExAllocatePoolWithTag(PagedPool, PAGE_SIZE, 'tseT'); + DPRINT1("PagingTestBuffer: %p, PTE %p\n", PagingTestBuffer, MiAddressToPte(PagingTestBuffer)); + for (i = 0; i < (PAGE_SIZE / sizeof(ULONG)); i++) + PagingTestBuffer[i] = i; +#endif + + while (1) + { + /* For now, we only age the paged pool, with 100 pages per run. */ + LONG Count = 100; + PVOID Address = LastPagedPoolAddress; + KIRQL OldIrql; + PMMPTE PointerPte; + PMMPDE PointerPde; + ULONG PdeIndex; + MMPTE TempPte; + PMMPFN Pfn1; + BOOLEAN Flush; + + Status = KeWaitForMultipleObjects( + 2, + WaitObjects, + WaitAny, + Executive, + KernelMode, + FALSE, + NULL, + NULL); + if (!NT_SUCCESS(Status)) + { + DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status); + KeBugCheck(MEMORY_MANAGEMENT); + } + + DPRINT("MM Balancer: Starting to loop.\n"); + + while (Count-- > 0) + { + Address = (PVOID)((ULONG_PTR) Address + PAGE_SIZE); + + /* Acquire PFN lock while we are cooking */ + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + + if (Address > MmPagedPoolEnd) + { + Address = MmPagedPoolStart; + } + + PointerPde = MiAddressToPde(Address); + PointerPte = MiAddressToPte(Address); + + PdeIndex = ((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE); + + if (MmSystemPagePtes[PdeIndex].u.Hard.Valid == 0) + TempPte.u.Long = 0; + else + TempPte = *PointerPte; + + if (TempPte.u.Hard.Valid == 0) + { + /* Bad luck, not a paged-in address */ +#ifdef TEST_PAGING + if (PointerPte == MiAddressToPte(PagingTestBuffer)) { - pointerPde = MiAddressToPde(Address); - if (pointerPde->u.Hard.Valid) - MiDeletePte(pointerPde, MiPdeToPte(pointerPde), Process, NULL); - ASSERT(pointerPde->u.Hard.Valid == 0); + /* Of course it should not magically become a prototype pte */ + ASSERT(TempPte.u.Soft.Prototype == 0); + if (TempPte.u.Soft.Transition == 0) + { + /* It was paged out! */ + ASSERT(TempPte.u.Soft.PageFileHigh != 0); + TestPage = TRUE; + } } - } - /* Release lock */ +#endif + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + continue; + } + + Flush = FALSE; + + /* Get the Pfn */ + Pfn1 = MI_PFN_ELEMENT(PFN_FROM_PTE(&TempPte)); + ASSERT(Pfn1->PteAddress == PointerPte); + ASSERT(Pfn1->u3.e1.PrototypePte == 0); + + /* First check if it was written to */ + if (TempPte.u.Hard.Dirty) + Pfn1->u3.e1.Modified = 1; + + /* See if it was accessed since the last time we looked */ + if (TempPte.u.Hard.Accessed) + { + /* Yes! Mark the page as young and fresh again */ + Pfn1->Wsle.u1.e1.Age = 0; + /* Tell the CPU we want to know for the next time */ + TempPte.u.Hard.Accessed = 0; + MI_UPDATE_VALID_PTE(PointerPte, TempPte); + Flush = TRUE; + } + else if (Pfn1->Wsle.u1.e1.Age == 3) + { + /* Page is getting old: Mark the PTE as transition */ + DPRINT1("MM Balancer: putting %p (page %x) as transition.\n", PointerPte, + PFN_FROM_PTE(&TempPte)); + TempPte.u.Hard.Valid = 0; + TempPte.u.Soft.Transition = 1; + TempPte.u.Trans.Protection = MM_READWRITE; + MiDecrementShareCount(Pfn1, PFN_FROM_PTE(&TempPte)); + MI_WRITE_INVALID_PTE(PointerPte, TempPte); + Flush = TRUE; + } + else + { + /* The page is just getting older */ + Pfn1->Wsle.u1.e1.Age++; + } + + /* Flush the TLB entry if we have to */ + if (Flush) + { +#ifdef CONFIG_SMP + // FIXME: Should invalidate entry in every CPU TLB + ASSERT(FALSE); +#endif + KeInvalidateTlbEntry(Address); + } + + /* Unlock, we're done for this address */ KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } -#endif - do - { - ULONG OldTarget = InitialTarget; - - /* Trim each consumer */ - for (i = 0; i < MC_MAXIMUM; i++) - { - InitialTarget = MiTrimMemoryConsumer(i, InitialTarget); - } - - /* No pages left to swap! */ - if (InitialTarget != 0 && - InitialTarget == OldTarget) - { - /* Game over */ - KeBugCheck(NO_PAGES_AVAILABLE); - } - } while (InitialTarget != 0); - } - else - { - DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status); - KeBugCheck(MEMORY_MANAGEMENT); - } - } + + DPRINT("MM Balancer: End of the loop.\n"); + +#ifdef TEST_PAGING + if (TestPage) + { + BOOLEAN Passed = TRUE; + DPRINT1("Verifying data integrity of paged out data!\n"); + for (i = 0; i < (PAGE_SIZE / sizeof(ULONG)); i++) + { + if (PagingTestBuffer[i] != i) + Passed = FALSE; + } + if (Passed) + DPRINT1("PASSED! \o/\n"); + else + DPRINT1("FAILED /o\\n"); + TestPage = FALSE; + } +#endif + /* Remember this for the next run */ + LastPagedPoolAddress = Address; + + if (Status == STATUS_WAIT_1) + { + /* Balancer thread was woken up by legacy MM + * This means we are really starving. Let's see if this still a problem */ + if ((MmAvailablePages >= MmMinimumFreePages) && (MmNumberOfPagingFiles != 0)) + { + /* Yay! No need to trust the old balancer */ + continue; + } + + MiRosBalancerThread(); + } + } }
VOID
Modified: branches/TransitionPte/ntoskrnl/mm/mminit.c URL: http://svn.reactos.org/svn/reactos/branches/TransitionPte/ntoskrnl/mm/mminit... ============================================================================== --- branches/TransitionPte/ntoskrnl/mm/mminit.c [iso-8859-1] (original) +++ branches/TransitionPte/ntoskrnl/mm/mminit.c [iso-8859-1] Mon Sep 22 17:20:38 2014 @@ -299,36 +299,179 @@ NTAPI MmMpwThreadMain(PVOID Parameter) { - NTSTATUS Status; - ULONG PagesWritten; - LARGE_INTEGER Timeout; - - UNREFERENCED_PARAMETER(Parameter); - - Timeout.QuadPart = -50000000; - - for(;;) - { - Status = KeWaitForSingleObject(&MpwThreadEvent, - 0, - KernelMode, - FALSE, - &Timeout); - if (!NT_SUCCESS(Status)) - { - DbgPrint("MpwThread: Wait failed\n"); - KeBugCheck(MEMORY_MANAGEMENT); - return; - } - - PagesWritten = 0; - -#ifndef NEWCC - // XXX arty -- we flush when evicting pages or destorying cache - // sections. - CcRosFlushDirtyPages(128, &PagesWritten, FALSE); -#endif - } + NTSTATUS Status; + PMMPTE SysPte, PteTable; + + UNREFERENCED_PARAMETER(Parameter); + + /* Reserve a PTE for the page table */ + SysPte = MiReserveSystemPtes(1, SystemPteSpace); + ASSERT(SysPte != NULL); + PteTable = MiPteToAddress(SysPte); + + for(;;) + { + KIRQL OldIrql; + PFN_NUMBER PageFrameIndex; + PMMPFN Pfn; + MMPTE TempPte, PdePte; + PMMPTE PointerPte; + BOOLEAN PageFileEntryFromPage = FALSE; + /* + * To start working, we wait for two conditions: + * - there are pages to be paged out. + * - We are in a low memory situation. + */ + Status = KeWaitForSingleObject( + &MpwThreadEvent, + WrPageOut, + KernelMode, + FALSE, + NULL); + + DPRINT("THE KRAKEN WAS RELEASED AND WILL PAGE YOUR ASS OUT!\n"); + + if (!NT_SUCCESS(Status)) + { + DPRINT1("MpwThread: Wait failed\n"); + KeBugCheck(MEMORY_MANAGEMENT); + return; + } + + /* Lock the PFN database */ + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + + /* The true main loop */ + while ((MmTotalPagesForPagingFile != 0) && (MiFreeSwapPages != 0)) + { + /* Get the first page from the list */ + PageFrameIndex = MmModifiedPageListByColor[0].Flink; + + /* Get The Pfn */ + Pfn = MI_PFN_ELEMENT(PageFrameIndex); + + /* Some things which must always hold */ + ASSERT(Pfn->OriginalPte.u.Soft.Transition == 0); + ASSERT(Pfn->u3.ReferenceCount == 0); + + /* And some that are not yet supported */ + ASSERT(Pfn->OriginalPte.u.Soft.Prototype == 0); + + /* Maybe this is not the first time */ + if ((Pfn->OriginalPte.u.Soft.Prototype == 0) && + (Pfn->OriginalPte.u.Soft.PageFileHigh != 0)) + { + /* Use that */ + TempPte = Pfn->OriginalPte; + PageFileEntryFromPage = TRUE; + } + else + { + MiReservePageFileEntry(&TempPte); + } + + /* Get it out of the list and reference it */ + MiUnlinkPageFromList(Pfn); + MiReferenceUnusedPageAndBumpLockCount(Pfn); + + ASSERT(Pfn->u3.e1.PageLocation == ModifiedPageList); + + /* Mark it as write in progress */ + Pfn->u3.e1.WriteInProgress = 1; + Pfn->u1.Event = NULL; + + /* Release the PFN lock while we are writing */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + + /* Do the actual write */ + Status = MiWritePageFile(PageFrameIndex, &TempPte); + + /* Get the lock again */ + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + + /* Get a mapping to the page table */ + MI_MAKE_HARDWARE_PTE_KERNEL(&PdePte, + SysPte, + MM_READWRITE, + Pfn->u4.PteFrame); + MI_WRITE_VALID_PTE(SysPte, PdePte); + + /* Finally get a pointer to the PTE this page represents */ + PointerPte = &PteTable[MiAddressToPteOffset(MiPteToAddress(Pfn->PteAddress))]; + + /* Get relevant values from the original PTE */ + TempPte.u.Soft.Protection = PointerPte->u.Soft.Protection; + + /* Maybe someone aborted the operation */ + if (Pfn->u3.e1.WriteInProgress == 0) + { + DPRINT1("Someone aborted the page-out operation!\n"); + /* Just set the event and let the waiter go along */ + ASSERT(Pfn->u1.Event != NULL); + KeSetEvent(Pfn->u1.Event, IO_NO_INCREMENT, FALSE); + Pfn->u1.Event = NULL; + /* This is now useless */ + if (!PageFileEntryFromPage) + MiFreePageFileEntry(&TempPte); + } + /* Maybe someone tried to access the page while we were not looking */ + else if (Pfn->u1.Event != NULL) + { + DPRINT1("Page fault occured while we were paging out!\n"); + + /* So the page fault handler marked the page as unmodified */ + ASSERT(Pfn->u3.e1.Modified == 0); + + /* Save the pagefile entry for this page */ + Pfn->OriginalPte = TempPte; + KeSetEvent(Pfn->u1.Event, IO_NO_INCREMENT, FALSE); + Pfn->u1.Event = NULL; + } + else if (!NT_SUCCESS(Status)) + { + DPRINT1("Failed to write page to pagefile!\n"); + /* This is now useless */ + if (!PageFileEntryFromPage) + MiFreePageFileEntry(&TempPte); + /* MiDereferencePfnAndDropLockCount will put it back on the tail of the list */ + } + else + { + DPRINT1("Page %x successfully paged out. PTE pointer %p (-> %x)\n", + PageFrameIndex, Pfn->PteAddress, TempPte.u.Long); + + /* Of course it must already be invalid */ + ASSERT(PointerPte->u.Hard.Valid == 0); + + /* And be in transition */ + ASSERT(PointerPte->u.Soft.Transition == 1); + + /* So the PTE is now officially paged out */ + MI_WRITE_INVALID_PTE(PointerPte, TempPte); + + /* And the pagefile entry belongs to the PTE, not to the page! */ + Pfn->OriginalPte.u.Long = 0; + + /* And dereference the page table frame */ + MiDecrementShareCount(MI_PFN_ELEMENT(Pfn->u4.PteFrame), Pfn->u4.PteFrame); + /* We can finally make it available for real */ + MI_SET_PFN_DELETED(Pfn); + } + + /* We're done with this */ + Pfn->u3.e1.WriteInProgress = 0; + + /* This will put it back in the free list */ + MiDereferencePfnAndDropLockCount(Pfn); + + /* Unmap the Page Table */ + MI_ERASE_PTE(SysPte); + KeInvalidateTlbEntry(PteTable); + } + + /* We're done for this run */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + } }
NTSTATUS
Modified: branches/TransitionPte/ntoskrnl/mm/pagefile.c URL: http://svn.reactos.org/svn/reactos/branches/TransitionPte/ntoskrnl/mm/pagefi... ============================================================================== --- branches/TransitionPte/ntoskrnl/mm/pagefile.c [iso-8859-1] (original) +++ branches/TransitionPte/ntoskrnl/mm/pagefile.c [iso-8859-1] Mon Sep 22 17:20:38 2014 @@ -46,13 +46,14 @@ { LIST_ENTRY PagingFileListEntry; PFILE_OBJECT FileObject; - LARGE_INTEGER MaximumSize; - LARGE_INTEGER CurrentSize; + ULONG_PTR MaximumSize; + ULONG_PTR CurrentSize; PFN_NUMBER FreePages; PFN_NUMBER UsedPages; - PULONG AllocMap; + PVOID AllocMap; + RTL_BITMAP AllocBitMap; + ULONG_PTR AllocBitMapHint; KSPIN_LOCK AllocMapLock; - ULONG AllocMapSize; PRETRIEVAL_POINTERS_BUFFER RetrievalPointers; } PAGINGFILE, *PPAGINGFILE; @@ -72,9 +73,6 @@
/* List of paging files, both used and free */ static PPAGINGFILE PagingFileList[MAX_PAGING_FILES]; - -/* Lock for examining the list of paging files */ -static KSPIN_LOCK PagingFileListLock;
/* Number of paging files */ ULONG MmNumberOfPagingFiles; @@ -221,30 +219,62 @@ NTAPI MmWriteToSwapPage(SWAPENTRY SwapEntry, PFN_NUMBER Page) { - ULONG i; - ULONG_PTR offset; - LARGE_INTEGER file_offset; + MMPTE TempPte; + + TempPte.u.Long = 0; + TempPte.u.Soft.PageFileHigh = OFFSET_FROM_ENTRY(SwapEntry); + TempPte.u.Soft.PageFileLow = FILE_FROM_ENTRY(SwapEntry); + return MiWritePageFile(Page, &TempPte); +} + + +NTSTATUS +NTAPI +MmReadFromSwapPage(SWAPENTRY SwapEntry, PFN_NUMBER Page) +{ + MMPTE TempPte; + + TempPte.u.Long = 0; + TempPte.u.Soft.PageFileHigh = OFFSET_FROM_ENTRY(SwapEntry); + TempPte.u.Soft.PageFileLow = FILE_FROM_ENTRY(SwapEntry); + return MiReadPageFile(Page, &TempPte); +} + +NTSTATUS +NTAPI +MiReadPageFile( + _In_ PFN_NUMBER Page, + _In_ const MMPTE* PointerPte +) +{ + LARGE_INTEGER FileOffset; IO_STATUS_BLOCK Iosb; + ULONG PageFileIndex = PointerPte->u.Soft.PageFileLow; + ULONG_PTR PageFileOffset = PointerPte->u.Soft.PageFileHigh; NTSTATUS Status; KEVENT Event; - UCHAR MdlBase[sizeof(MDL) + sizeof(ULONG)]; + UCHAR MdlBase[sizeof(MDL) + sizeof(PFN_NUMBER)]; PMDL Mdl = (PMDL)MdlBase; - - DPRINT("MmWriteToSwapPage\n"); - - if (SwapEntry == 0) + PPAGINGFILE PagingFile; + + DPRINT("MiReadPageFile\n"); + + if (PageFileOffset == 0) { KeBugCheck(MEMORY_MANAGEMENT); return(STATUS_UNSUCCESSFUL); }
- i = FILE_FROM_ENTRY(SwapEntry); - offset = OFFSET_FROM_ENTRY(SwapEntry); - - if (PagingFileList[i]->FileObject == NULL || - PagingFileList[i]->FileObject->DeviceObject == NULL) - { - DPRINT1("Bad paging file 0x%.8X\n", SwapEntry); + /* Normalize it */ + PageFileOffset--; + + ASSERT(PageFileIndex < MAX_PAGING_FILES); + + PagingFile = PagingFileList[PageFileIndex]; + + if (PagingFile->FileObject == NULL || PagingFile->FileObject->DeviceObject == NULL) + { + DPRINT1("Bad paging file %u\n", PageFileIndex); KeBugCheck(MEMORY_MANAGEMENT); }
@@ -252,80 +282,13 @@ MmBuildMdlFromPages(Mdl, &Page); Mdl->MdlFlags |= MDL_PAGES_LOCKED;
- file_offset.QuadPart = offset * PAGE_SIZE; - file_offset = MmGetOffsetPageFile(PagingFileList[i]->RetrievalPointers, file_offset); - - KeInitializeEvent(&Event, NotificationEvent, FALSE); - Status = IoSynchronousPageWrite(PagingFileList[i]->FileObject, - Mdl, - &file_offset, - &Event, - &Iosb); - if (Status == STATUS_PENDING) - { - KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); - Status = Iosb.Status; - } - - if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) - { - MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl); - } - return(Status); -} - - -NTSTATUS -NTAPI -MmReadFromSwapPage(SWAPENTRY SwapEntry, PFN_NUMBER Page) -{ - return MiReadPageFile(Page, FILE_FROM_ENTRY(SwapEntry), OFFSET_FROM_ENTRY(SwapEntry)); -} - -NTSTATUS -NTAPI -MiReadPageFile( - _In_ PFN_NUMBER Page, - _In_ ULONG PageFileIndex, - _In_ ULONG_PTR PageFileOffset) -{ - LARGE_INTEGER file_offset; - IO_STATUS_BLOCK Iosb; - NTSTATUS Status; - KEVENT Event; - UCHAR MdlBase[sizeof(MDL) + sizeof(ULONG)]; - PMDL Mdl = (PMDL)MdlBase; - PPAGINGFILE PagingFile; - - DPRINT("MiReadSwapFile\n"); - - if (PageFileOffset == 0) - { - KeBugCheck(MEMORY_MANAGEMENT); - return(STATUS_UNSUCCESSFUL); - } - - ASSERT(PageFileIndex < MAX_PAGING_FILES); - - PagingFile = PagingFileList[PageFileIndex]; - - if (PagingFile->FileObject == NULL || PagingFile->FileObject->DeviceObject == NULL) - { - DPRINT1("Bad paging file %u\n", PageFileIndex); - KeBugCheck(MEMORY_MANAGEMENT); - } - - MmInitializeMdl(Mdl, NULL, PAGE_SIZE); - MmBuildMdlFromPages(Mdl, &Page); - Mdl->MdlFlags |= MDL_PAGES_LOCKED; - - file_offset.QuadPart = PageFileOffset * PAGE_SIZE; - file_offset = MmGetOffsetPageFile(PagingFile->RetrievalPointers, file_offset); + FileOffset.QuadPart = PageFileOffset * PAGE_SIZE; + FileOffset = MmGetOffsetPageFile(PagingFile->RetrievalPointers, FileOffset);
KeInitializeEvent(&Event, NotificationEvent, FALSE); Status = IoPageRead(PagingFile->FileObject, Mdl, - &file_offset, + &FileOffset, &Event, &Iosb); if (Status == STATUS_PENDING) @@ -340,14 +303,182 @@ return(Status); }
+NTSTATUS +NTAPI +MiWritePageFile( + _In_ PFN_NUMBER Page, + _In_ const MMPTE* PointerPte +) +{ + ULONG PageFileIndex = PointerPte->u.Soft.PageFileLow; + ULONG_PTR PageFileOffset = PointerPte->u.Soft.PageFileHigh; + LARGE_INTEGER FileOffset; + IO_STATUS_BLOCK Iosb; + NTSTATUS Status; + KEVENT Event; + UCHAR MdlBase[sizeof(MDL) + sizeof(PFN_NUMBER)]; + PMDL Mdl = (PMDL) MdlBase; + PPAGINGFILE PagingFile; + + /* The PTE must already be setup to point to a pagefile entry */ + ASSERT(PointerPte->u.Hard.Valid == 0); + ASSERT(PointerPte->u.Soft.Transition == 0); + ASSERT(PointerPte->u.Soft.Prototype == 0); + + DPRINT("MiWriteSwapFile\n"); + + if (PageFileOffset == 0) + { + KeBugCheck(MEMORY_MANAGEMENT); + return (STATUS_UNSUCCESSFUL); + } + + /* Normalize it */ + PageFileOffset--; + + ASSERT(PageFileIndex < MAX_PAGING_FILES); + + PagingFile = PagingFileList[PageFileIndex]; + + if (PagingFile->FileObject == NULL || PagingFile->FileObject->DeviceObject == NULL) + { + DPRINT1("Bad paging file %u\n", PageFileIndex); + KeBugCheck(MEMORY_MANAGEMENT); + } + + MmInitializeMdl(Mdl, NULL, PAGE_SIZE); + MmBuildMdlFromPages(Mdl, &Page); + Mdl->MdlFlags |= MDL_PAGES_LOCKED; + + FileOffset.QuadPart = PageFileOffset * PAGE_SIZE; + FileOffset = MmGetOffsetPageFile(PagingFile->RetrievalPointers, FileOffset); + + KeInitializeEvent(&Event, NotificationEvent, FALSE); + Status = IoSynchronousPageWrite(PagingFile->FileObject, + Mdl, + &FileOffset, + &Event, + &Iosb); + if (Status == STATUS_PENDING) + { + KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); + Status = Iosb.Status; + } + if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) + { + MmUnmapLockedPages(Mdl->MappedSystemVa, Mdl); + } + return Status; +} + +VOID +NTAPI +MiFreePageFileEntry( + _In_ PMMPTE PointerPte +) +{ + ULONG PageFileIndex; + ULONG_PTR PageFileOffset; + PPAGINGFILE PageFile; + + /* Some sanity checks */ + ASSERT(PointerPte->u.Hard.Valid == 0); + ASSERT(PointerPte->u.Soft.Transition == 0); + ASSERT(PointerPte->u.Soft.Prototype == 0); + + /* We must hold the PFN lock */ + ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); + + PageFileIndex = PointerPte->u.Soft.PageFileLow; + PageFileOffset = PointerPte->u.Soft.PageFileHigh; + + /* Fix up the value */ + ASSERT(PageFileOffset != 0); + PageFileOffset--; + + DPRINT1("Releasing page file entry %u, %lu\n", PageFileIndex, PageFileOffset); + + ASSERT(PageFileIndex < MAX_PAGING_FILES); + + PageFile = PagingFileList[PageFileIndex]; + + /* And also lock this pagefile */ + KeAcquireSpinLockAtDpcLevel(&PageFile->AllocMapLock); + + /* It must be already in use */ + ASSERT(RtlTestBit(&PageFile->AllocBitMap, PageFileOffset)); + RtlClearBit(&PageFile->AllocBitMap, PageFileOffset); + + /* One more */ + PageFile->FreePages++; + PageFile->UsedPages--; + + MiFreeSwapPages++; + MiUsedSwapPages--; + + /* Done */ + KeReleaseSpinLockFromDpcLevel(&PageFile->AllocMapLock); +} + +NTSTATUS +NTAPI +MiReservePageFileEntry( + _Out_ PMMPTE PointerPte +) +{ + ULONG PageFileIndex; + ULONG_PTR PageFileOffset; + PPAGINGFILE PageFile; + + /* We must hold the PFN lock */ + ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); + + for (PageFileIndex = 0; PageFileIndex < MAX_PAGING_FILES; PageFileIndex++) + { + if (PagingFileList[PageFileIndex] == NULL) + continue; + if (PagingFileList[PageFileIndex]->FreePages != 0) + break; + } + + if (PageFileIndex == MAX_PAGING_FILES) + { + DPRINT1("Pagefiles are full!\n"); + return STATUS_PAGEFILE_QUOTA; + } + + PointerPte->u.Long = 0; + + PageFile = PagingFileList[PageFileIndex]; + + /* And also lock this page file */ + KeAcquireSpinLockAtDpcLevel(&PageFile->AllocMapLock); + + /* Find a free entry in this page file */ + PageFileOffset = RtlFindClearBitsAndSet(&PageFile->AllocBitMap, 1, PageFile->AllocBitMapHint); + + /* We must have found something as the file was marked as having free entries */ + ASSERT(PageFileOffset != ~((ULONG_PTR)0)); + PageFile->AllocBitMapHint = PageFileOffset & ~((PageFile->MaximumSize / PAGE_SIZE) - 1); + + DPRINT1("Reserving page file entry %lu, %Iu\n", PageFileIndex, PageFileOffset); + + /* Done */ + KeReleaseSpinLockFromDpcLevel(&PageFile->AllocMapLock); + + /* We rely on PageFileHigh being != 0 */ + PointerPte->u.Soft.PageFileHigh = PageFileOffset + 1; + PointerPte->u.Soft.PageFileLow = PageFileIndex; + + return STATUS_SUCCESS; +} + VOID INIT_FUNCTION NTAPI MmInitPagingFile(VOID) { ULONG i; - - KeInitializeSpinLock(&PagingFileListLock);
MiFreeSwapPages = 0; MiUsedSwapPages = 0; @@ -360,104 +491,41 @@ MmNumberOfPagingFiles = 0; }
-static ULONG -MiAllocPageFromPagingFile(PPAGINGFILE PagingFile) -{ - KIRQL oldIrql; - ULONG i, j; - - KeAcquireSpinLock(&PagingFile->AllocMapLock, &oldIrql); - - for (i = 0; i < PagingFile->AllocMapSize; i++) - { - for (j = 0; j < 32; j++) - { - if (!(PagingFile->AllocMap[i] & (1 << j))) - { - PagingFile->AllocMap[i] |= (1 << j); - PagingFile->UsedPages++; - PagingFile->FreePages--; - KeReleaseSpinLock(&PagingFile->AllocMapLock, oldIrql); - return((i * 32) + j); - } - } - } - - KeReleaseSpinLock(&PagingFile->AllocMapLock, oldIrql); - return(0xFFFFFFFF); -} - VOID NTAPI MmFreeSwapPage(SWAPENTRY Entry) { - ULONG i; - ULONG_PTR off; - KIRQL oldIrql; - - i = FILE_FROM_ENTRY(Entry); - off = OFFSET_FROM_ENTRY(Entry); - - KeAcquireSpinLock(&PagingFileListLock, &oldIrql); - if (PagingFileList[i] == NULL) - { - KeBugCheck(MEMORY_MANAGEMENT); - } - KeAcquireSpinLockAtDpcLevel(&PagingFileList[i]->AllocMapLock); - - PagingFileList[i]->AllocMap[off >> 5] &= (~(1 << (off % 32))); - - PagingFileList[i]->FreePages++; - PagingFileList[i]->UsedPages--; - - MiFreeSwapPages++; - MiUsedSwapPages--; - - KeReleaseSpinLockFromDpcLevel(&PagingFileList[i]->AllocMapLock); - KeReleaseSpinLock(&PagingFileListLock, oldIrql); + MMPTE TempPte; + KIRQL OldIrql; + + TempPte.u.Long = 0; + TempPte.u.Soft.PageFileLow = FILE_FROM_ENTRY(Entry); + TempPte.u.Soft.PageFileHigh = OFFSET_FROM_ENTRY(Entry); + + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + MiFreePageFileEntry(&TempPte); + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); }
SWAPENTRY NTAPI MmAllocSwapPage(VOID) { - KIRQL oldIrql; - ULONG i; - ULONG off; - SWAPENTRY entry; - - KeAcquireSpinLock(&PagingFileListLock, &oldIrql); - - if (MiFreeSwapPages == 0) - { - KeReleaseSpinLock(&PagingFileListLock, oldIrql); - return(0); - } - - for (i = 0; i < MAX_PAGING_FILES; i++) - { - if (PagingFileList[i] != NULL && - PagingFileList[i]->FreePages >= 1) - { - off = MiAllocPageFromPagingFile(PagingFileList[i]); - if (off == 0xFFFFFFFF) - { - KeBugCheck(MEMORY_MANAGEMENT); - KeReleaseSpinLock(&PagingFileListLock, oldIrql); - return(STATUS_UNSUCCESSFUL); - } - MiUsedSwapPages++; - MiFreeSwapPages--; - KeReleaseSpinLock(&PagingFileListLock, oldIrql); - - entry = ENTRY_FROM_FILE_OFFSET(i, off); - return(entry); - } - } - - KeReleaseSpinLock(&PagingFileListLock, oldIrql); - KeBugCheck(MEMORY_MANAGEMENT); - return(0); + MMPTE TempPte; + SWAPENTRY entry; + NTSTATUS Status; + KIRQL OldIrql; + + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + Status = MiReservePageFileEntry(&TempPte); + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + + if (!NT_SUCCESS(Status)) + return 0; + + entry = ENTRY_FROM_FILE_OFFSET(TempPte.u.Soft.PageFileLow, TempPte.u.Soft.PageFileHigh); + + return entry; }
static PRETRIEVEL_DESCRIPTOR_LIST FASTCALL @@ -488,7 +556,7 @@ IO_STATUS_BLOCK IoStatus; PFILE_OBJECT FileObject; PPAGINGFILE PagingFile; - KIRQL oldIrql; + KIRQL OldIrql; ULONG AllocMapSize; FILE_FS_SIZE_INFORMATION FsSizeInformation; PRETRIEVEL_DESCRIPTOR_LIST RetDescList; @@ -646,7 +714,6 @@ #if defined(__GNUC__) Vcn.QuadPart = 0LL; #else - Vcn.QuadPart = 0; #endif
@@ -718,16 +785,15 @@ RtlZeroMemory(PagingFile, sizeof(*PagingFile));
PagingFile->FileObject = FileObject; - PagingFile->MaximumSize.QuadPart = SafeMaximumSize.QuadPart; - PagingFile->CurrentSize.QuadPart = SafeInitialSize.QuadPart; - PagingFile->FreePages = (ULONG)(SafeInitialSize.QuadPart / PAGE_SIZE); + PagingFile->MaximumSize = SafeMaximumSize.LowPart; + PagingFile->CurrentSize = SafeInitialSize.LowPart; + PagingFile->FreePages = (ULONG)(SafeMaximumSize.LowPart / PAGE_SIZE); PagingFile->UsedPages = 0; KeInitializeSpinLock(&PagingFile->AllocMapLock);
AllocMapSize = (PagingFile->FreePages / 32) + 1; PagingFile->AllocMap = ExAllocatePool(NonPagedPool, AllocMapSize * sizeof(ULONG)); - PagingFile->AllocMapSize = AllocMapSize;
if (PagingFile->AllocMap == NULL) { @@ -760,8 +826,10 @@ return(STATUS_NO_MEMORY); }
- RtlZeroMemory(PagingFile->AllocMap, AllocMapSize * sizeof(ULONG)); RtlZeroMemory(PagingFile->RetrievalPointers, Size); + + RtlInitializeBitMap(&PagingFile->AllocBitMap, PagingFile->AllocMap, AllocMapSize * 32); + RtlClearAllBits(&PagingFile->AllocBitMap);
Count = 0; PagingFile->RetrievalPointers->ExtentCount = ExtentCount; @@ -799,7 +867,7 @@ PagingFile->RetrievalPointers->Extents[i].NextVcn.QuadPart *= BytesPerAllocationUnit; }
- KeAcquireSpinLock(&PagingFileListLock, &oldIrql); + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); for (i = 0; i < MAX_PAGING_FILES; i++) { if (PagingFileList[i] == NULL) @@ -810,7 +878,7 @@ } MiFreeSwapPages = MiFreeSwapPages + PagingFile->FreePages; MmNumberOfPagingFiles++; - KeReleaseSpinLock(&PagingFileListLock, oldIrql); + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
ZwClose(FileHandle);
-
jgardou＠svn.reactos.org