Ros-diffs February 2021

ros-diffs@reactos.org

30 participants
434 discussions

by Jérôme Gardou

https://git.reactos.org/?p=reactos.git;a=commitdiff;h=774a4c703fdf241333906… commit 774a4c703fdf2413339064422f39ef2dbfd88fcc Author: Jérôme Gardou <jerome.gardou(a)reactos.org> AuthorDate: Tue Nov 24 14:07:06 2020 +0100 Commit: Jérôme Gardou <jerome.gardou(a)reactos.org> CommitDate: Wed Feb 3 09:41:22 2021 +0100 [NTOS/MM] Misc fixes Purge data section object before creating an image mapping Zero-out the tail of the page after reading from file Properly map page as read-only when paging-in a COW memory map. --- ntoskrnl/mm/section.c | 92 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 68 insertions(+), 24 deletions(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index f108533ed2e..8df9b99cfb8 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -1214,6 +1214,18 @@ MiReadPage(PMEMORY_AREA MemoryArea, Status = STATUS_SUCCESS; } + if ((SegOffset + PAGE_SIZE) > MemoryArea->SectionData.Segment->RawLength.QuadPart) + { + KIRQL OldIrql; + PUCHAR PageMap; + + /* Zero out the end of it */ + PageMap = MiMapPageInHyperSpace(PsGetCurrentProcess(), *Page, &OldIrql); + RtlZeroMemory(PageMap + MemoryArea->SectionData.Segment->RawLength.QuadPart - SegOffset, + PAGE_SIZE - (MemoryArea->SectionData.Segment->RawLength.QuadPart - SegOffset)); + MiUnmapPageInHyperSpace(PsGetCurrentProcess(), PageMap, OldIrql); + } + return Status; } @@ -1448,32 +1460,24 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, { SWAPENTRY DummyEntry; - /* - * Is it a wait entry? - */ - if (HasSwapEntry) + MmGetPageFileMapping(Process, Address, &SwapEntry); + if (SwapEntry == MM_WAIT_ENTRY) { - MmGetPageFileMapping(Process, Address, &SwapEntry); - - if (SwapEntry == MM_WAIT_ENTRY) - { - MmUnlockSectionSegment(Segment); - MmUnlockAddressSpace(AddressSpace); - MiWaitForPageEvent(NULL, NULL); - MmLockAddressSpace(AddressSpace); - return STATUS_MM_RESTART_OPERATION; - } - - /* - * Must be private page we have swapped out. - */ - - /* - * Sanity check - */ - MmDeletePageFileMapping(Process, Address, &SwapEntry); + MmUnlockSectionSegment(Segment); + MmUnlockAddressSpace(AddressSpace); + MiWaitForPageEvent(NULL, NULL); + MmLockAddressSpace(AddressSpace); + return STATUS_MM_RESTART_OPERATION; } + /* + * Must be private page we have swapped out. + */ + + /* + * Sanity check + */ + MmDeletePageFileMapping(Process, Address, &SwapEntry); MmUnlockSectionSegment(Segment); /* Tell everyone else we are serving the fault. */ @@ -1709,7 +1713,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, /* Map the page into the process address space */ Status = MmCreateVirtualMapping(Process, PAddress, - Region->Protect, + Attributes, &Page, 1); if (!NT_SUCCESS(Status)) @@ -3096,6 +3100,7 @@ MmCreateImageSection(PSECTION *SectionObject, if (ImageSectionObject == NULL) { NTSTATUS StatusExeFmt; + PMM_SECTION_SEGMENT DataSectionObject; ImageSectionObject = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_IMAGE_SECTION_OBJECT), TAG_MM_SECTION_SEGMENT); if (ImageSectionObject == NULL) @@ -3112,8 +3117,47 @@ MmCreateImageSection(PSECTION *SectionObject, ImageSectionObject->RefCount = 1; FileObject->SectionObjectPointer->ImageSectionObject = ImageSectionObject; + /* Get a ref on the data section object */ + DataSectionObject = FileObject->SectionObjectPointer->DataSectionObject; + while (DataSectionObject && (DataSectionObject->SegFlags & (MM_SEGMENT_INDELETE | MM_SEGMENT_INCREATE))) + { + LARGE_INTEGER ShortTime; + + MiReleasePfnLock(OldIrql); + + ShortTime.QuadPart = - 10 * 100 * 1000; + KeDelayExecutionThread(KernelMode, FALSE, &ShortTime); + + OldIrql = MiAcquirePfnLock(); + DataSectionObject = FileObject->SectionObjectPointer->DataSectionObject; + ASSERT(DataSectionObject->SegFlags & MM_DATAFILE_SEGMENT); + } + + /* Get a ref on it. */ + if (DataSectionObject) + InterlockedIncrementUL(&DataSectionObject->RefCount); + MiReleasePfnLock(OldIrql); + if (DataSectionObject) + { + if ((DataSectionObject->SectionCount - (FileObject->SectionObjectPointer->SharedCacheMap != NULL)) > 0) + { + /* Someone's got a section opened. Deny creation */ + DPRINT1("Denying image creation for %wZ: Sections opened: %lu.\n", + &FileObject->FileName, DataSectionObject->SectionCount); + InterlockedExchangePointer(&FileObject->SectionObjectPointer->ImageSectionObject, NULL); + ExFreePoolWithTag(ImageSectionObject, TAG_MM_SECTION_SEGMENT); + MmDereferenceSegment(DataSectionObject); + ObDereferenceObject(FileObject); + ObDereferenceObject(Section); + return STATUS_ACCESS_DENIED; + } + + /* Purge the cache. */ + CcPurgeCacheSection(FileObject->SectionObjectPointer, NULL, 0, FALSE); + } + StatusExeFmt = ExeFmtpCreateImageSection(FileObject, ImageSectionObject); if (!NT_SUCCESS(StatusExeFmt))

3 years, 10 months

[reactos] 26/100: [NTOS/MM] Do not dirtify section pages for system-space maps

by Jérôme Gardou

https://git.reactos.org/?p=reactos.git;a=commitdiff;h=093333740468ad0af6828… commit 093333740468ad0af68280d53c928ed4bacfeea1 Author: Jérôme Gardou <jerome.gardou(a)reactos.org> AuthorDate: Wed Nov 18 17:45:47 2020 +0100 Commit: Jérôme Gardou <jerome.gardou(a)reactos.org> CommitDate: Wed Feb 3 09:41:22 2021 +0100 [NTOS/MM] Do not dirtify section pages for system-space maps Cc does it for itself. --- ntoskrnl/mm/section.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 46bf5cb571d..f108533ed2e 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -3343,8 +3343,12 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, else { if (Process) + { MmDeleteRmap(Page, Process, Address); - MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, FALSE, NULL); + } + + /* We don't dirtify for System Space Maps. We let Cc manage that */ + MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Process ? Dirty : FALSE, FALSE, NULL); } } }

3 years, 10 months

[reactos] 25/100: [NTOS/MM] Do not error when FS returns STATUS_END_OF_FILE for paged read.

by Jérôme Gardou

https://git.reactos.org/?p=reactos.git;a=commitdiff;h=d440434a8e120067abba5… commit d440434a8e120067abba5630214dd804ba06449f Author: Jérôme Gardou <jerome.gardou(a)reactos.org> AuthorDate: Wed Nov 18 12:14:41 2020 +0100 Commit: Jérôme Gardou <jerome.gardou(a)reactos.org> CommitDate: Wed Feb 3 09:41:22 2021 +0100 [NTOS/MM] Do not error when FS returns STATUS_END_OF_FILE for paged read. Also remove a seemingly invalid ASSERT --- ntoskrnl/mm/section.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 6c521522f4f..46bf5cb571d 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -1209,6 +1209,11 @@ MiReadPage(PMEMORY_AREA MemoryArea, MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl); } + if (Status == STATUS_END_OF_FILE) + { + Status = STATUS_SUCCESS; + } + return Status; } @@ -2211,7 +2216,7 @@ MmCreateDataFileSection(PSECTION *SectionObject, if (!GotFileHandle) { ASSERT(UMaximumSize != NULL); - ASSERT(UMaximumSize->QuadPart != 0); + // ASSERT(UMaximumSize->QuadPart != 0); MaximumSize = *UMaximumSize; } else

3 years, 10 months

[reactos] 24/100: [NTOS/MM] DO not assert when being passed invalid parameters in MmCreateSection

by Jérôme Gardou

https://git.reactos.org/?p=reactos.git;a=commitdiff;h=b46173edaff285e3c4140… commit b46173edaff285e3c414070eb1d1655f2873d998 Author: Jérôme Gardou <jerome.gardou(a)reactos.org> AuthorDate: Wed Nov 18 10:29:17 2020 +0100 Commit: Jérôme Gardou <jerome.gardou(a)reactos.org> CommitDate: Wed Feb 3 09:41:22 2021 +0100 [NTOS/MM] DO not assert when being passed invalid parameters in MmCreateSection --- ntoskrnl/mm/section.c | 1 - 1 file changed, 1 deletion(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 39ae8bf75f5..6c521522f4f 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -4358,7 +4358,6 @@ MmCreateSection (OUT PVOID * Section, else { /* All cases should be handled above */ - ASSERT(FALSE); Status = STATUS_INVALID_PARAMETER; if (FileObject) ObDereferenceObject(FileObject);

3 years, 10 months

[reactos] 23/100: [NTOS/MM] Assert there are no RMAP left before freeing the page

by Jérôme Gardou

https://git.reactos.org/?p=reactos.git;a=commitdiff;h=ba49c3904124501572753… commit ba49c390412450157275330268bcaab096aa6149 Author: Jérôme Gardou <jerome.gardou(a)reactos.org> AuthorDate: Tue Nov 17 17:38:53 2020 +0100 Commit: Jérôme Gardou <jerome.gardou(a)reactos.org> CommitDate: Wed Feb 3 09:41:22 2021 +0100 [NTOS/MM] Assert there are no RMAP left before freeing the page --- ntoskrnl/mm/rmap.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index 21b5b246457..80576f4cc59 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -225,9 +225,8 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) /* We can finally let this page go */ MmDeleteRmap(Page, Process, Address); - MmReleasePageMemoryConsumer(MC_USER, Page); - ASSERT(MmGetRmapListHeadPage(Page) == NULL); + MmReleasePageMemoryConsumer(MC_USER, Page); if (Address < MmSystemRangeStart) {

3 years, 10 months

[reactos] 22/100: [NTOS:MM] Fix locking when paging out

by Jérôme Gardou

https://git.reactos.org/?p=reactos.git;a=commitdiff;h=d932bdb9505e270830e0e… commit d932bdb9505e270830e0eb177b523f39d32d1f10 Author: Jérôme Gardou <jerome.gardou(a)reactos.org> AuthorDate: Fri Jan 22 09:30:55 2021 +0100 Commit: Jérôme Gardou <jerome.gardou(a)reactos.org> CommitDate: Wed Feb 3 09:41:22 2021 +0100 [NTOS:MM] Fix locking when paging out --- ntoskrnl/mm/rmap.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index d1cfb584dce..21b5b246457 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -70,7 +70,7 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) if (entry == NULL) { MiReleasePfnLock(OldIrql); - return STATUS_UNSUCCESSFUL; + goto WriteSegment; } Process = entry->Process; @@ -176,6 +176,12 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) MmSetDirtyPage(Process, Address); MmUnlockAddressSpace(AddressSpace); + if (Address < MmSystemRangeStart) + { + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + } + return STATUS_UNSUCCESSFUL; } } @@ -199,6 +205,11 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) MmSetDirtyPage(Process, Address); MmUnlockAddressSpace(AddressSpace); + if (Address < MmSystemRangeStart) + { + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + } return STATUS_UNSUCCESSFUL; } } @@ -256,12 +267,7 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) KeBugCheck(MEMORY_MANAGEMENT); } - if (Address < MmSystemRangeStart) - { - ExReleaseRundownProtection(&Process->RundownProtect); - ObDereferenceObject(Process); - } - +WriteSegment: /* Now write this page to file, if needed */ Segment = MmGetSectionAssociation(Page, &SegmentOffset); if (Segment)

3 years, 10 months

[reactos] 21/100: [NTOSKRNL] Overhaul Cc and Mm relationship

by Jérôme Gardou

https://git.reactos.org/?p=reactos.git;a=commitdiff;h=d8cdb89fb03006595dc40… commit d8cdb89fb03006595dc40ac23db5267b8d9d9c09 Author: Jérôme Gardou <jerome.gardou(a)reactos.org> AuthorDate: Fri Nov 6 09:39:31 2020 +0100 Commit: Jérôme Gardou <jerome.gardou(a)reactos.org> CommitDate: Wed Feb 3 09:41:22 2021 +0100 [NTOSKRNL] Overhaul Cc and Mm relationship Previously, when creating a file section, Mm requested Cc to cache the file, then Cc would request pages from Mm, then Mm would request them back to serve its file-mapping role Now, Mm does it all by itself. If file cahcing is requested by the FS driver, then Cc creates a file mapping and uses that to serve its purpose. This is a rewrite of Cc --- ntoskrnl/cache/section/sptab.c | 3 - ntoskrnl/cc/copy.c | 557 ++++------- ntoskrnl/cc/fs.c | 58 +- ntoskrnl/cc/pin.c | 234 ++--- ntoskrnl/cc/view.c | 495 ++++------ ntoskrnl/ex/init.c | 2 - ntoskrnl/ex/sysinfo.c | 12 +- ntoskrnl/include/internal/cc.h | 30 +- ntoskrnl/include/internal/mm.h | 82 +- ntoskrnl/mm/ARM3/section.c | 32 +- ntoskrnl/mm/ARM3/virtual.c | 8 +- ntoskrnl/mm/ARM3/zeropage.c | 2 +- ntoskrnl/mm/balance.c | 12 - ntoskrnl/mm/pagefile.c | 5 +- ntoskrnl/mm/rmap.c | 315 ++++--- ntoskrnl/mm/section.c | 2032 ++++++++++++++++------------------------ ntoskrnl/po/power.c | 4 +- 17 files changed, 1518 insertions(+), 2365 deletions(-) diff --git a/ntoskrnl/cache/section/sptab.c b/ntoskrnl/cache/section/sptab.c index 454fb25af42..7a2481a4752 100644 --- a/ntoskrnl/cache/section/sptab.c +++ b/ntoskrnl/cache/section/sptab.c @@ -187,9 +187,6 @@ _MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, ASSERT(Segment->Locked); ASSERT(!IS_SWAP_FROM_SSE(Entry) || !IS_DIRTY_SSE(Entry)); - if (Entry && !IS_SWAP_FROM_SSE(Entry)) - MmGetRmapListHeadPage(PFN_FROM_SSE(Entry)); - PageTable = MiSectionPageTableGetOrAllocate(&Segment->PageTable, Offset); if (!PageTable) return STATUS_NO_MEMORY; diff --git a/ntoskrnl/cc/copy.c b/ntoskrnl/cc/copy.c index 1f731716b1f..5bb14cc210a 100644 --- a/ntoskrnl/cc/copy.c +++ b/ntoskrnl/cc/copy.c @@ -20,13 +20,6 @@ static PFN_NUMBER CcZeroPage = 0; #define MAX_ZERO_LENGTH (256 * 1024) -typedef enum _CC_COPY_OPERATION -{ - CcOperationRead, - CcOperationWrite, - CcOperationZero -} CC_COPY_OPERATION; - typedef enum _CC_CAN_WRITE_RETRY { FirstTry = 0, @@ -35,7 +28,7 @@ typedef enum _CC_CAN_WRITE_RETRY RetryMasterLocked = 255, } CC_CAN_WRITE_RETRY; -ULONG CcRosTraceLevel = 0; +ULONG CcRosTraceLevel = CC_API_DEBUG; ULONG CcFastMdlReadWait; ULONG CcFastMdlReadNotPossible; ULONG CcFastReadNotPossible; @@ -76,338 +69,6 @@ CcInitCacheZeroPage ( MiZeroPhysicalPage(CcZeroPage); } -NTSTATUS -NTAPI -CcReadVirtualAddress ( - PROS_VACB Vacb) -{ - ULONG Size; - PMDL Mdl; - NTSTATUS Status; - IO_STATUS_BLOCK IoStatus; - KEVENT Event; - ULARGE_INTEGER LargeSize; - - LargeSize.QuadPart = Vacb->SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart; - if (LargeSize.QuadPart > VACB_MAPPING_GRANULARITY) - { - LargeSize.QuadPart = VACB_MAPPING_GRANULARITY; - } - Size = LargeSize.LowPart; - - Size = ROUND_TO_PAGES(Size); - ASSERT(Size <= VACB_MAPPING_GRANULARITY); - ASSERT(Size > 0); - - Mdl = IoAllocateMdl(Vacb->BaseAddress, Size, FALSE, FALSE, NULL); - if (!Mdl) - { - return STATUS_INSUFFICIENT_RESOURCES; - } - - Status = STATUS_SUCCESS; - _SEH2_TRY - { - MmProbeAndLockPages(Mdl, KernelMode, IoWriteAccess); - } - _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) - { - Status = _SEH2_GetExceptionCode(); - DPRINT1("MmProbeAndLockPages failed with: %lx for %p (%p, %p)\n", Status, Mdl, Vacb, Vacb->BaseAddress); - KeBugCheck(CACHE_MANAGER); - } _SEH2_END; - - if (NT_SUCCESS(Status)) - { - Mdl->MdlFlags |= MDL_IO_PAGE_READ; - KeInitializeEvent(&Event, NotificationEvent, FALSE); - Status = IoPageRead(Vacb->SharedCacheMap->FileObject, Mdl, &Vacb->FileOffset, &Event, &IoStatus); - if (Status == STATUS_PENDING) - { - KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); - Status = IoStatus.Status; - } - - MmUnlockPages(Mdl); - } - - IoFreeMdl(Mdl); - - if (!NT_SUCCESS(Status) && (Status != STATUS_END_OF_FILE)) - { - DPRINT1("IoPageRead failed, Status %x\n", Status); - return Status; - } - - if (Size < VACB_MAPPING_GRANULARITY) - { - RtlZeroMemory((char*)Vacb->BaseAddress + Size, - VACB_MAPPING_GRANULARITY - Size); - } - - return STATUS_SUCCESS; -} - -NTSTATUS -NTAPI -CcWriteVirtualAddress ( - PROS_VACB Vacb) -{ - ULONG Size; - PMDL Mdl; - NTSTATUS Status; - IO_STATUS_BLOCK IoStatus; - KEVENT Event; - ULARGE_INTEGER LargeSize; - - LargeSize.QuadPart = Vacb->SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart; - if (LargeSize.QuadPart > VACB_MAPPING_GRANULARITY) - { - LargeSize.QuadPart = VACB_MAPPING_GRANULARITY; - } - Size = LargeSize.LowPart; - // - // Nonpaged pool PDEs in ReactOS must actually be synchronized between the - // MmGlobalPageDirectory and the real system PDE directory. What a mess... - // - { - ULONG i = 0; - do - { - MmGetPfnForProcess(NULL, (PVOID)((ULONG_PTR)Vacb->BaseAddress + (i << PAGE_SHIFT))); - } while (++i < (Size >> PAGE_SHIFT)); - } - - ASSERT(Size <= VACB_MAPPING_GRANULARITY); - ASSERT(Size > 0); - - Mdl = IoAllocateMdl(Vacb->BaseAddress, Size, FALSE, FALSE, NULL); - if (!Mdl) - { - return STATUS_INSUFFICIENT_RESOURCES; - } - - Status = STATUS_SUCCESS; - _SEH2_TRY - { - MmProbeAndLockPages(Mdl, KernelMode, IoReadAccess); - } - _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) - { - Status = _SEH2_GetExceptionCode(); - DPRINT1("MmProbeAndLockPages failed with: %lx for %p (%p, %p)\n", Status, Mdl, Vacb, Vacb->BaseAddress); - KeBugCheck(CACHE_MANAGER); - } _SEH2_END; - - if (NT_SUCCESS(Status)) - { - KeInitializeEvent(&Event, NotificationEvent, FALSE); - Status = IoSynchronousPageWrite(Vacb->SharedCacheMap->FileObject, Mdl, &Vacb->FileOffset, &Event, &IoStatus); - if (Status == STATUS_PENDING) - { - KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); - Status = IoStatus.Status; - } - - MmUnlockPages(Mdl); - } - IoFreeMdl(Mdl); - if (!NT_SUCCESS(Status) && (Status != STATUS_END_OF_FILE)) - { - DPRINT1("IoPageWrite failed, Status %x\n", Status); - return Status; - } - - return STATUS_SUCCESS; -} - -NTSTATUS -ReadWriteOrZero( - _Inout_ PVOID BaseAddress, - _Inout_opt_ PVOID Buffer, - _In_ ULONG Length, - _In_ CC_COPY_OPERATION Operation) -{ - NTSTATUS Status = STATUS_SUCCESS; - - if (Operation == CcOperationZero) - { - /* Zero */ - RtlZeroMemory(BaseAddress, Length); - } - else - { - _SEH2_TRY - { - if (Operation == CcOperationWrite) - RtlCopyMemory(BaseAddress, Buffer, Length); - else - RtlCopyMemory(Buffer, BaseAddress, Length); - } - _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) - { - Status = _SEH2_GetExceptionCode(); - } - _SEH2_END; - } - return Status; -} - -BOOLEAN -CcCopyData ( - _In_ PFILE_OBJECT FileObject, - _In_ LONGLONG FileOffset, - _Inout_ PVOID Buffer, - _In_ LONGLONG Length, - _In_ CC_COPY_OPERATION Operation, - _In_ BOOLEAN Wait, - _Out_ PIO_STATUS_BLOCK IoStatus) -{ - NTSTATUS Status; - LONGLONG CurrentOffset; - ULONG BytesCopied; - KIRQL OldIrql; - PROS_SHARED_CACHE_MAP SharedCacheMap; - PLIST_ENTRY ListEntry; - PROS_VACB Vacb; - ULONG PartialLength; - PVOID BaseAddress; - BOOLEAN Valid; - PPRIVATE_CACHE_MAP PrivateCacheMap; - - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - PrivateCacheMap = FileObject->PrivateCacheMap; - CurrentOffset = FileOffset; - BytesCopied = 0; - - if (!Wait) - { - /* test if the requested data is available */ - KeAcquireSpinLock(&SharedCacheMap->CacheMapLock, &OldIrql); - /* FIXME: this loop doesn't take into account areas that don't have - * a VACB in the list yet */ - ListEntry = SharedCacheMap->CacheMapVacbListHead.Flink; - while (ListEntry != &SharedCacheMap->CacheMapVacbListHead) - { - Vacb = CONTAINING_RECORD(ListEntry, - ROS_VACB, - CacheMapVacbListEntry); - ListEntry = ListEntry->Flink; - if (!Vacb->Valid && - DoRangesIntersect(Vacb->FileOffset.QuadPart, - VACB_MAPPING_GRANULARITY, - CurrentOffset, Length)) - { - KeReleaseSpinLock(&SharedCacheMap->CacheMapLock, OldIrql); - /* data not available */ - return FALSE; - } - if (Vacb->FileOffset.QuadPart >= CurrentOffset + Length) - break; - } - KeReleaseSpinLock(&SharedCacheMap->CacheMapLock, OldIrql); - } - - PartialLength = CurrentOffset % VACB_MAPPING_GRANULARITY; - if (PartialLength != 0) - { - PartialLength = min(Length, VACB_MAPPING_GRANULARITY - PartialLength); - Status = CcRosRequestVacb(SharedCacheMap, - ROUND_DOWN(CurrentOffset, - VACB_MAPPING_GRANULARITY), - &BaseAddress, - &Valid, - &Vacb); - if (!NT_SUCCESS(Status)) - ExRaiseStatus(Status); - if (!Valid) - { - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - ExRaiseStatus(Status); - } - } - Status = ReadWriteOrZero((PUCHAR)BaseAddress + CurrentOffset % VACB_MAPPING_GRANULARITY, - Buffer, - PartialLength, - Operation); - - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, Operation != CcOperationRead, FALSE); - - if (!NT_SUCCESS(Status)) - ExRaiseStatus(STATUS_INVALID_USER_BUFFER); - - Length -= PartialLength; - CurrentOffset += PartialLength; - BytesCopied += PartialLength; - - if (Operation != CcOperationZero) - Buffer = (PVOID)((ULONG_PTR)Buffer + PartialLength); - } - - while (Length > 0) - { - ASSERT(CurrentOffset % VACB_MAPPING_GRANULARITY == 0); - PartialLength = min(VACB_MAPPING_GRANULARITY, Length); - Status = CcRosRequestVacb(SharedCacheMap, - CurrentOffset, - &BaseAddress, - &Valid, - &Vacb); - if (!NT_SUCCESS(Status)) - ExRaiseStatus(Status); - if (!Valid && - (Operation == CcOperationRead || - PartialLength < VACB_MAPPING_GRANULARITY)) - { - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - ExRaiseStatus(Status); - } - } - Status = ReadWriteOrZero(BaseAddress, Buffer, PartialLength, Operation); - - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, Operation != CcOperationRead, FALSE); - - if (!NT_SUCCESS(Status)) - ExRaiseStatus(STATUS_INVALID_USER_BUFFER); - - Length -= PartialLength; - CurrentOffset += PartialLength; - BytesCopied += PartialLength; - - if (Operation != CcOperationZero) - Buffer = (PVOID)((ULONG_PTR)Buffer + PartialLength); - } - - /* If that was a successful sync read operation, let's handle read ahead */ - if (Operation == CcOperationRead && Length == 0 && Wait) - { - /* If file isn't random access and next read may get us cross VACB boundary, - * schedule next read - */ - if (!BooleanFlagOn(FileObject->Flags, FO_RANDOM_ACCESS) && - (CurrentOffset - 1) / VACB_MAPPING_GRANULARITY != (CurrentOffset + BytesCopied - 1) / VACB_MAPPING_GRANULARITY) - { - CcScheduleReadAhead(FileObject, (PLARGE_INTEGER)&FileOffset, BytesCopied); - } - - /* And update read history in private cache map */ - PrivateCacheMap->FileOffset1.QuadPart = PrivateCacheMap->FileOffset2.QuadPart; - PrivateCacheMap->BeyondLastByte1.QuadPart = PrivateCacheMap->BeyondLastByte2.QuadPart; - PrivateCacheMap->FileOffset2.QuadPart = FileOffset; - PrivateCacheMap->BeyondLastByte2.QuadPart = FileOffset + BytesCopied; - } - - IoStatus->Status = STATUS_SUCCESS; - IoStatus->Information = BytesCopied; - return TRUE; -} - VOID CcPostDeferredWrites(VOID) { @@ -492,8 +153,6 @@ CcPerformReadAhead( PROS_SHARED_CACHE_MAP SharedCacheMap; PROS_VACB Vacb; ULONG PartialLength; - PVOID BaseAddress; - BOOLEAN Valid; ULONG Length; PPRIVATE_CACHE_MAP PrivateCacheMap; BOOLEAN Locked; @@ -556,10 +215,7 @@ CcPerformReadAhead( { PartialLength = min(Length, VACB_MAPPING_GRANULARITY - PartialLength); Status = CcRosRequestVacb(SharedCacheMap, - ROUND_DOWN(CurrentOffset, - VACB_MAPPING_GRANULARITY), - &BaseAddress, - &Valid, + ROUND_DOWN(CurrentOffset, VACB_MAPPING_GRANULARITY), &Vacb); if (!NT_SUCCESS(Status)) { @@ -567,15 +223,13 @@ CcPerformReadAhead( goto Clear; } - if (!Valid) + Status = CcRosEnsureVacbResident(Vacb, TRUE, FALSE, + CurrentOffset % VACB_MAPPING_GRANULARITY, PartialLength); + if (!NT_SUCCESS(Status)) { - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - DPRINT1("Failed to read data: %lx!\n", Status); - goto Clear; - } + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); + DPRINT1("Failed to read data: %lx!\n", Status); + goto Clear; } CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); @@ -590,8 +244,6 @@ CcPerformReadAhead( PartialLength = min(VACB_MAPPING_GRANULARITY, Length); Status = CcRosRequestVacb(SharedCacheMap, CurrentOffset, - &BaseAddress, - &Valid, &Vacb); if (!NT_SUCCESS(Status)) { @@ -599,15 +251,12 @@ CcPerformReadAhead( goto Clear; } - if (!Valid) + Status = CcRosEnsureVacbResident(Vacb, TRUE, FALSE, 0, PartialLength); + if (!NT_SUCCESS(Status)) { - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - DPRINT1("Failed to read data: %lx!\n", Status); - goto Clear; - } + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); + DPRINT1("Failed to read data: %lx!\n", Status); + goto Clear; } CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); @@ -811,6 +460,12 @@ CcCopyRead ( OUT PVOID Buffer, OUT PIO_STATUS_BLOCK IoStatus) { + PROS_VACB Vacb; + PROS_SHARED_CACHE_MAP SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; + NTSTATUS Status; + LONGLONG CurrentOffset; + LONGLONG ReadEnd = FileOffset->QuadPart + Length; + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%I64d Length=%lu Wait=%d\n", FileObject, FileOffset->QuadPart, Length, Wait); @@ -819,13 +474,58 @@ CcCopyRead ( FileObject, FileOffset->QuadPart, Length, Wait, Buffer, IoStatus); - return CcCopyData(FileObject, - FileOffset->QuadPart, - Buffer, - Length, - CcOperationRead, - Wait, - IoStatus); + if (!SharedCacheMap) + return FALSE; + + /* Documented to ASSERT, but KMTests test this case... */ + // ASSERT((FileOffset->QuadPart + Length) <= SharedCacheMap->FileSize.QuadPart); + + IoStatus->Status = STATUS_SUCCESS; + IoStatus->Information = 0; + + CurrentOffset = FileOffset->QuadPart; + while(CurrentOffset < ReadEnd) + { + Status = CcRosGetVacb(SharedCacheMap, CurrentOffset, &Vacb); + if (!NT_SUCCESS(Status)) + { + ExRaiseStatus(Status); + return FALSE; + } + + _SEH2_TRY + { + ULONG VacbOffset = CurrentOffset % VACB_MAPPING_GRANULARITY; + ULONG VacbLength = min(Length, VACB_MAPPING_GRANULARITY - VacbOffset); + SIZE_T CopyLength = VacbLength; + + if (!CcRosEnsureVacbResident(Vacb, Wait, FALSE, VacbOffset, VacbLength)) + return FALSE; + + /* Do not copy past the section */ + if (CurrentOffset + VacbLength > SharedCacheMap->SectionSize.QuadPart) + CopyLength = SharedCacheMap->SectionSize.QuadPart - CurrentOffset; + if (CopyLength != 0) + RtlCopyMemory(Buffer, (PUCHAR)Vacb->BaseAddress + VacbOffset, CopyLength); + + /* Zero-out the buffer tail if needed */ + if (CopyLength < VacbLength) + RtlZeroMemory((PUCHAR)Buffer + CopyLength, VacbLength - CopyLength); + + IoStatus->Information += VacbLength; + + Buffer = (PVOID)((ULONG_PTR)Buffer + VacbLength); + CurrentOffset += VacbLength; + Length -= VacbLength; + } + _SEH2_FINALLY + { + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + } + _SEH2_END; + } + + return TRUE; } /* @@ -840,7 +540,11 @@ CcCopyWrite ( IN BOOLEAN Wait, IN PVOID Buffer) { - IO_STATUS_BLOCK IoStatus; + PROS_VACB Vacb; + PROS_SHARED_CACHE_MAP SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; + NTSTATUS Status; + LONGLONG CurrentOffset; + LONGLONG WriteEnd = FileOffset->QuadPart + Length; CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%I64d Length=%lu Wait=%d Buffer=%p\n", FileObject, FileOffset->QuadPart, Length, Wait, Buffer); @@ -849,13 +553,48 @@ CcCopyWrite ( "Length %lu, Wait %u, Buffer 0x%p)\n", FileObject, FileOffset->QuadPart, Length, Wait, Buffer); - return CcCopyData(FileObject, - FileOffset->QuadPart, - Buffer, - Length, - CcOperationWrite, - Wait, - &IoStatus); + if (!SharedCacheMap) + return FALSE; + + /* FIXME: Honor FileObject FO_WRITE_THROUGH flag */ + + ASSERT((FileOffset->QuadPart + Length) <= SharedCacheMap->FileSize.QuadPart); + + CurrentOffset = FileOffset->QuadPart; + while(CurrentOffset < WriteEnd) + { + ULONG VacbOffset = CurrentOffset % VACB_MAPPING_GRANULARITY; + ULONG VacbLength = min(Length, VACB_MAPPING_GRANULARITY - VacbOffset); + + Status = CcRosGetVacb(SharedCacheMap, CurrentOffset, &Vacb); + if (!NT_SUCCESS(Status)) + { + ExRaiseStatus(Status); + return FALSE; + } + + _SEH2_TRY + { + if (!CcRosEnsureVacbResident(Vacb, Wait, FALSE, VacbOffset, VacbLength)) + { + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + return FALSE; + } + + RtlCopyMemory((PVOID)((ULONG_PTR)Vacb->BaseAddress + VacbOffset), Buffer, VacbLength); + + Buffer = (PVOID)((ULONG_PTR)Buffer + VacbLength); + CurrentOffset += VacbLength; + Length -= VacbLength; + } + _SEH2_FINALLY + { + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, TRUE, FALSE); + } + _SEH2_END; + } + + return TRUE; } /* @@ -999,11 +738,8 @@ CcZeroData ( NTSTATUS Status; LARGE_INTEGER WriteOffset; LONGLONG Length; - ULONG CurrentLength; - PMDL Mdl; - ULONG i; - IO_STATUS_BLOCK Iosb; - KEVENT Event; + PROS_VACB Vacb; + PROS_SHARED_CACHE_MAP SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; CCTRACE(CC_API_DEBUG, "FileObject=%p StartOffset=%I64u EndOffset=%I64u Wait=%d\n", FileObject, StartOffset->QuadPart, EndOffset->QuadPart, Wait); @@ -1015,9 +751,14 @@ CcZeroData ( Length = EndOffset->QuadPart - StartOffset->QuadPart; WriteOffset.QuadPart = StartOffset->QuadPart; - if (FileObject->SectionObjectPointer->SharedCacheMap == NULL) + if (!SharedCacheMap || (FileObject->Flags & FO_WRITE_THROUGH)) { - /* File is not cached */ + /* Make this a non-cached write */ + IO_STATUS_BLOCK Iosb; + KEVENT Event; + PMDL Mdl; + ULONG i; + ULONG CurrentLength; Mdl = _alloca(MmSizeOfMdl(NULL, MAX_ZERO_LENGTH)); @@ -1032,7 +773,7 @@ CcZeroData ( CurrentLength = Length; } MmInitializeMdl(Mdl, (PVOID)(ULONG_PTR)WriteOffset.QuadPart, CurrentLength); - Mdl->MdlFlags |= (MDL_PAGES_LOCKED | MDL_IO_PAGE_READ); + Mdl->MdlFlags |= MDL_PAGES_LOCKED; for (i = 0; i < ((Mdl->Size - sizeof(MDL)) / sizeof(ULONG)); i++) { ((PPFN_NUMBER)(Mdl + 1))[i] = CcZeroPage; @@ -1055,18 +796,42 @@ CcZeroData ( WriteOffset.QuadPart += CurrentLength; Length -= CurrentLength; } + + return TRUE; } - else + + ASSERT(EndOffset->QuadPart <= SharedCacheMap->FileSize.QuadPart); + + while(WriteOffset.QuadPart < EndOffset->QuadPart) { - IO_STATUS_BLOCK IoStatus; + ULONG VacbOffset = WriteOffset.QuadPart % VACB_MAPPING_GRANULARITY; + ULONG VacbLength = min(Length, VACB_MAPPING_GRANULARITY - VacbOffset); - return CcCopyData(FileObject, - WriteOffset.QuadPart, - NULL, - Length, - CcOperationZero, - Wait, - &IoStatus); + Status = CcRosGetVacb(SharedCacheMap, WriteOffset.QuadPart, &Vacb); + if (!NT_SUCCESS(Status)) + { + ExRaiseStatus(Status); + return FALSE; + } + + _SEH2_TRY + { + if (!CcRosEnsureVacbResident(Vacb, Wait, FALSE, VacbOffset, VacbLength)) + { + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + return FALSE; + } + + RtlZeroMemory((PVOID)((ULONG_PTR)Vacb->BaseAddress + VacbOffset), VacbLength); + + WriteOffset.QuadPart += VacbLength; + Length -= VacbLength; + } + _SEH2_FINALLY + { + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, TRUE, FALSE); + } + _SEH2_END; } return TRUE; diff --git a/ntoskrnl/cc/fs.c b/ntoskrnl/cc/fs.c index 5798f56a91a..5a74ecee48c 100644 --- a/ntoskrnl/cc/fs.c +++ b/ntoskrnl/cc/fs.c @@ -10,13 +10,10 @@ /* INCLUDES ******************************************************************/ #include <ntoskrnl.h> + #define NDEBUG #include <debug.h> -/* GLOBALS *****************************************************************/ - -NTSTATUS CcRosInternalFreeVacb(PROS_VACB Vacb); - /* FUNCTIONS *****************************************************************/ /* @@ -272,8 +269,9 @@ CcSetFileSizes ( IN PFILE_OBJECT FileObject, IN PCC_FILE_SIZES FileSizes) { - KIRQL oldirql; + KIRQL OldIrql; PROS_SHARED_CACHE_MAP SharedCacheMap; + LARGE_INTEGER OldSectionSize; CCTRACE(CC_API_DEBUG, "FileObject=%p FileSizes=%p\n", FileObject, FileSizes); @@ -294,7 +292,14 @@ CcSetFileSizes ( if (SharedCacheMap == NULL) return; - if (FileSizes->AllocationSize.QuadPart < SharedCacheMap->SectionSize.QuadPart) + /* Update the relevant fields */ + KeAcquireSpinLock(&SharedCacheMap->CacheMapLock, &OldIrql); + OldSectionSize = SharedCacheMap->SectionSize; + SharedCacheMap->SectionSize = FileSizes->AllocationSize; + SharedCacheMap->FileSize = FileSizes->FileSize; + KeReleaseSpinLock(&SharedCacheMap->CacheMapLock, OldIrql); + + if (FileSizes->AllocationSize.QuadPart < OldSectionSize.QuadPart) { CcPurgeCacheSection(FileObject->SectionObjectPointer, &FileSizes->AllocationSize, @@ -303,46 +308,9 @@ CcSetFileSizes ( } else { - PROS_VACB LastVacb; - - /* - * If file (allocation) size has increased, then we need to check whether - * it just grows in a single VACB (the last one). - * If so, we must mark the VACB as invalid to trigger a read to the - * FSD at the next VACB usage, and thus avoid returning garbage - */ - - /* Check for allocation size and the last VACB */ - if (SharedCacheMap->SectionSize.QuadPart < FileSizes->AllocationSize.QuadPart && - SharedCacheMap->SectionSize.QuadPart % VACB_MAPPING_GRANULARITY) - { - LastVacb = CcRosLookupVacb(SharedCacheMap, - SharedCacheMap->SectionSize.QuadPart); - if (LastVacb != NULL) - { - /* Mark it as invalid */ - CcRosReleaseVacb(SharedCacheMap, LastVacb, LastVacb->Dirty ? LastVacb->Valid : FALSE, FALSE, FALSE); - } - } - - /* Check for file size and the last VACB */ - if (SharedCacheMap->FileSize.QuadPart < FileSizes->FileSize.QuadPart && - SharedCacheMap->FileSize.QuadPart % VACB_MAPPING_GRANULARITY) - { - LastVacb = CcRosLookupVacb(SharedCacheMap, - SharedCacheMap->FileSize.QuadPart); - if (LastVacb != NULL) - { - /* Mark it as invalid */ - CcRosReleaseVacb(SharedCacheMap, LastVacb, LastVacb->Dirty ? LastVacb->Valid : FALSE, FALSE, FALSE); - } - } + /* Extend our section object */ + MmExtendSection(SharedCacheMap->Section, &SharedCacheMap->SectionSize); } - - KeAcquireSpinLock(&SharedCacheMap->CacheMapLock, &oldirql); - SharedCacheMap->SectionSize = FileSizes->AllocationSize; - SharedCacheMap->FileSize = FileSizes->FileSize; - KeReleaseSpinLock(&SharedCacheMap->CacheMapLock, oldirql); } /* diff --git a/ntoskrnl/cc/pin.c b/ntoskrnl/cc/pin.c index b839e71dafd..189ba480305 100644 --- a/ntoskrnl/cc/pin.c +++ b/ntoskrnl/cc/pin.c @@ -67,91 +67,6 @@ CcpFindBcb( return (Found ? Bcb : NULL); } -static -BOOLEAN -NTAPI -CcpMapData( - IN PROS_SHARED_CACHE_MAP SharedCacheMap, - IN PLARGE_INTEGER FileOffset, - IN ULONG Length, - IN ULONG Flags, - OUT PROS_VACB *pVacb, - OUT PVOID *pBuffer) -{ - LONGLONG ReadOffset, BaseOffset; - BOOLEAN Valid; - PROS_VACB Vacb; - NTSTATUS Status; - LONGLONG ROffset; - - ReadOffset = FileOffset->QuadPart; - - DPRINT("SectionSize %I64x, FileSize %I64x\n", - SharedCacheMap->SectionSize.QuadPart, - SharedCacheMap->FileSize.QuadPart); - - if (ReadOffset % VACB_MAPPING_GRANULARITY + Length > VACB_MAPPING_GRANULARITY) - { - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", - SharedCacheMap->FileObject, FileOffset, Length, Flags); - return FALSE; - } - - if (!BooleanFlagOn(Flags, MAP_NO_READ)) - { - static int Warned = 0; - - SetFlag(Flags, MAP_NO_READ); - if (!Warned) - { - DPRINT1("Mapping/pinning with no read not implemented. Forcing read, might fail if wait not allowed\n"); - Warned++; - } - } - - /* Properly round offset and call internal helper for getting a VACB */ - ROffset = ROUND_DOWN(ReadOffset, VACB_MAPPING_GRANULARITY); - Status = CcRosGetVacb(SharedCacheMap, - ROffset, - &BaseOffset, - pBuffer, - &Valid, - &Vacb); - if (!NT_SUCCESS(Status)) - { - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", - SharedCacheMap->FileObject, FileOffset, Length, Flags); - ExRaiseStatus(Status); - return FALSE; - } - - if (!Valid && BooleanFlagOn(Flags, MAP_NO_READ)) - { - if (!BooleanFlagOn(Flags, MAP_WAIT)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", - SharedCacheMap->FileObject, FileOffset, Length, Flags); - return FALSE; - } - - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", - SharedCacheMap->FileObject, FileOffset, Length, Flags); - ExRaiseStatus(Status); - return FALSE; - } - } - - *pBuffer = (PUCHAR)*pBuffer + ReadOffset % VACB_MAPPING_GRANULARITY; - *pVacb = Vacb; - - return TRUE; -} - static VOID CcpDereferenceBcb( @@ -304,44 +219,44 @@ CcpPinData( OUT PVOID * Buffer) { PINTERNAL_BCB NewBcb; - BOOLEAN Result; - PROS_VACB Vacb; KIRQL OldIrql; - ULONG MapFlags; + ULONG VacbOffset; + NTSTATUS Status; + BOOLEAN Result; + + VacbOffset = (ULONG)(FileOffset->QuadPart % VACB_MAPPING_GRANULARITY); + /* This seems to be valid, according to KMTests */ + if ((VacbOffset + Length) > VACB_MAPPING_GRANULARITY) + Length = VACB_MAPPING_GRANULARITY - VacbOffset; KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql); NewBcb = CcpFindBcb(SharedCacheMap, FileOffset, Length, TRUE); if (NewBcb != NULL) { + BOOLEAN Result; + ++NewBcb->RefCount; KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql); if (BooleanFlagOn(Flags, PIN_EXCLUSIVE)) - { Result = ExAcquireResourceExclusiveLite(&NewBcb->Lock, BooleanFlagOn(Flags, PIN_WAIT)); - } else - { Result = ExAcquireSharedStarveExclusive(&NewBcb->Lock, BooleanFlagOn(Flags, PIN_WAIT)); - } if (!Result) { CcpDereferenceBcb(SharedCacheMap, NewBcb); - NewBcb = NULL; - } - else - { - NewBcb->PinCount++; - *Bcb = NewBcb; - *Buffer = (PUCHAR)NewBcb->Vacb->BaseAddress + FileOffset->QuadPart % VACB_MAPPING_GRANULARITY; + return FALSE; } - return Result; + NewBcb->PinCount++; } else { + LONGLONG ROffset; + PROS_VACB Vacb; + KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql); if (BooleanFlagOn(Flags, PIN_IF_BCB)) @@ -349,29 +264,49 @@ CcpPinData( return FALSE; } - MapFlags = Flags & PIN_WAIT; - if (BooleanFlagOn(Flags, PIN_NO_READ)) + /* Properly round offset and call internal helper for getting a VACB */ + ROffset = ROUND_DOWN(FileOffset->QuadPart, VACB_MAPPING_GRANULARITY); + Status = CcRosGetVacb(SharedCacheMap, ROffset, &Vacb); + if (!NT_SUCCESS(Status)) { - SetFlag(MapFlags, MAP_NO_READ); + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", + SharedCacheMap->FileObject, FileOffset, Length, Flags); + ExRaiseStatus(Status); + return FALSE; } - Result = CcpMapData(SharedCacheMap, FileOffset, Length, MapFlags, &Vacb, Buffer); - if (Result) + NewBcb = CcpGetAppropriateBcb(SharedCacheMap, Vacb, FileOffset, Length, Flags, TRUE); + if (NewBcb == NULL) { - NewBcb = CcpGetAppropriateBcb(SharedCacheMap, Vacb, FileOffset, Length, Flags, TRUE); - if (NewBcb == NULL) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); - Result = FALSE; - } - else - { - *Bcb = NewBcb; - } + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + return FALSE; } } - return Result; + Result = FALSE; + _SEH2_TRY + { + /* Ensure the pages are resident */ + Result = CcRosEnsureVacbResident(NewBcb->Vacb, + BooleanFlagOn(Flags, PIN_WAIT), + BooleanFlagOn(Flags, PIN_NO_READ), + VacbOffset, Length); + } + _SEH2_FINALLY + { + if (!Result) + { + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", + SharedCacheMap->FileObject, FileOffset, Length, Flags); + CcUnpinData(NewBcb); + return FALSE; + } + } + _SEH2_END; + + *Bcb = NewBcb; + *Buffer = (PVOID)((ULONG_PTR)NewBcb->Vacb->BaseAddress + VacbOffset); + return TRUE; } /* @@ -387,13 +322,15 @@ CcMapData ( OUT PVOID *pBcb, OUT PVOID *pBuffer) { - BOOLEAN Ret; KIRQL OldIrql; PINTERNAL_BCB iBcb; PROS_VACB Vacb; PROS_SHARED_CACHE_MAP SharedCacheMap; + ULONG VacbOffset; + NTSTATUS Status; + BOOLEAN Result; - DPRINT("CcMapData(FileObject 0x%p, FileOffset %I64x, Length %lu, Flags 0x%lx," + CCTRACE(CC_API_DEBUG, "CcMapData(FileObject 0x%p, FileOffset 0x%I64x, Length %lu, Flags 0x%lx," " pBcb 0x%p, pBuffer 0x%p)\n", FileObject, FileOffset->QuadPart, Length, Flags, pBcb, pBuffer); @@ -413,6 +350,11 @@ CcMapData ( ++CcMapDataNoWait; } + VacbOffset = (ULONG)(FileOffset->QuadPart % VACB_MAPPING_GRANULARITY); + /* KMTests seem to show that it is allowed to call accross mapping granularity */ + if ((VacbOffset + Length) > VACB_MAPPING_GRANULARITY) + Length = VACB_MAPPING_GRANULARITY - VacbOffset; + KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql); iBcb = CcpFindBcb(SharedCacheMap, FileOffset, Length, FALSE); @@ -420,34 +362,54 @@ CcMapData ( { KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql); - Ret = CcpMapData(SharedCacheMap, FileOffset, Length, Flags, &Vacb, pBuffer); - if (Ret) + /* Call internal helper for getting a VACB */ + Status = CcRosGetVacb(SharedCacheMap, FileOffset->QuadPart, &Vacb); + if (!NT_SUCCESS(Status)) { - iBcb = CcpGetAppropriateBcb(SharedCacheMap, Vacb, FileOffset, Length, 0, FALSE); - if (iBcb == NULL) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); - Ret = FALSE; - } - else - { - *pBcb = iBcb; - } + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", + SharedCacheMap->FileObject, FileOffset, Length, Flags); + ExRaiseStatus(Status); + return FALSE; + } + + iBcb = CcpGetAppropriateBcb(SharedCacheMap, Vacb, FileOffset, Length, 0, FALSE); + if (iBcb == NULL) + { + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", + SharedCacheMap->FileObject, FileOffset, Length, Flags); + return FALSE; } } else { ++iBcb->RefCount; KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql); + } - *pBcb = iBcb; - *pBuffer = (PUCHAR)iBcb->Vacb->BaseAddress + FileOffset->QuadPart % VACB_MAPPING_GRANULARITY; - Ret = TRUE; + _SEH2_TRY + { + Result = FALSE; + /* Ensure the pages are resident */ + Result = CcRosEnsureVacbResident(iBcb->Vacb, BooleanFlagOn(Flags, MAP_WAIT), + BooleanFlagOn(Flags, MAP_NO_READ), VacbOffset, Length); + } + _SEH2_FINALLY + { + if (!Result) + { + CcpDereferenceBcb(SharedCacheMap, iBcb); + return FALSE; + } } + _SEH2_END; + + *pBcb = iBcb; + *pBuffer = (PVOID)((ULONG_PTR)iBcb->Vacb->BaseAddress + VacbOffset); - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> %d Bcb=%p\n", - FileObject, FileOffset, Length, Flags, Ret, *pBcb); - return Ret; + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> TRUE Bcb=%p, Buffer %p\n", + FileObject, FileOffset, Length, Flags, *pBcb, *pBuffer); + return Result; } /* diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index adb28bb84ab..ddbe77e22af 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -156,20 +156,29 @@ CcRosTraceCacheMap ( #endif } +NTSTATUS +NTAPI +MmFlushVirtualMemory(IN PEPROCESS Process, + IN OUT PVOID *BaseAddress, + IN OUT PSIZE_T RegionSize, + OUT PIO_STATUS_BLOCK IoStatusBlock); + NTSTATUS NTAPI CcRosFlushVacb ( PROS_VACB Vacb) { + IO_STATUS_BLOCK Iosb; + SIZE_T FlushSize = min(VACB_MAPPING_GRANULARITY, + Vacb->SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart); NTSTATUS Status; CcRosUnmarkDirtyVacb(Vacb, TRUE); - Status = CcWriteVirtualAddress(Vacb); + Status = MmFlushVirtualMemory(NULL, &Vacb->BaseAddress, &FlushSize, &Iosb); + if (!NT_SUCCESS(Status)) - { CcRosMarkDirtyVacb(Vacb); - } return Status; } @@ -234,6 +243,8 @@ CcRosFlushDirtyPages ( current->SharedCacheMap->LazyWriteContext, Wait); if (!Locked) { + DPRINT("Not locked!"); + ASSERT(!Wait); OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); CcRosVacbDecRefCount(current); continue; @@ -264,15 +275,18 @@ CcRosFlushDirtyPages ( PagesFreed = VACB_MAPPING_GRANULARITY / PAGE_SIZE; (*Count) += PagesFreed; - /* Make sure we don't overflow target! */ - if (Target < PagesFreed) + if (!Wait) { - /* If we would have, jump to zero directly */ - Target = 0; - } - else - { - Target -= PagesFreed; + /* Make sure we don't overflow target! */ + if (Target < PagesFreed) + { + /* If we would have, jump to zero directly */ + Target = 0; + } + else + { + Target -= PagesFreed; + } } } @@ -286,136 +300,6 @@ CcRosFlushDirtyPages ( return STATUS_SUCCESS; } -NTSTATUS -CcRosTrimCache ( - ULONG Target, - ULONG Priority, - PULONG NrFreed) -/* - * FUNCTION: Try to free some memory from the file cache. - * ARGUMENTS: - * Target - The number of pages to be freed. - * Priority - The priority of free (currently unused). - * NrFreed - Points to a variable where the number of pages - * actually freed is returned. - */ -{ - PLIST_ENTRY current_entry; - PROS_VACB current; - ULONG PagesFreed; - KIRQL oldIrql; - LIST_ENTRY FreeList; - PFN_NUMBER Page; - ULONG i; - BOOLEAN FlushedPages = FALSE; - - DPRINT("CcRosTrimCache(Target %lu)\n", Target); - - InitializeListHead(&FreeList); - - *NrFreed = 0; - -retry: - oldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - - current_entry = VacbLruListHead.Flink; - while (current_entry != &VacbLruListHead) - { - ULONG Refs; - - current = CONTAINING_RECORD(current_entry, - ROS_VACB, - VacbLruListEntry); - current_entry = current_entry->Flink; - - KeAcquireSpinLockAtDpcLevel(&current->SharedCacheMap->CacheMapLock); - - /* Reference the VACB */ - CcRosVacbIncRefCount(current); - - /* Check if it's mapped and not dirty */ - if (InterlockedCompareExchange((PLONG)&current->MappedCount, 0, 0) > 0 && !current->Dirty) - { - /* We have to break these locks because Cc sucks */ - KeReleaseSpinLockFromDpcLevel(&current->SharedCacheMap->CacheMapLock); - KeReleaseQueuedSpinLock(LockQueueMasterLock, oldIrql); - - /* Page out the VACB */ - for (i = 0; i < VACB_MAPPING_GRANULARITY / PAGE_SIZE; i++) - { - Page = (PFN_NUMBER)(MmGetPhysicalAddress((PUCHAR)current->BaseAddress + (i * PAGE_SIZE)).QuadPart >> PAGE_SHIFT); - - MmPageOutPhysicalAddress(Page); - } - - /* Reacquire the locks */ - oldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - KeAcquireSpinLockAtDpcLevel(&current->SharedCacheMap->CacheMapLock); - } - - /* Dereference the VACB */ - Refs = CcRosVacbDecRefCount(current); - - /* Check if we can free this entry now */ - if (Refs < 2) - { - ASSERT(!current->Dirty); - ASSERT(!current->MappedCount); - ASSERT(Refs == 1); - - RemoveEntryList(&current->CacheMapVacbListEntry); - RemoveEntryList(&current->VacbLruListEntry); - InitializeListHead(&current->VacbLruListEntry); - InsertHeadList(&FreeList, &current->CacheMapVacbListEntry); - - /* Calculate how many pages we freed for Mm */ - PagesFreed = min(VACB_MAPPING_GRANULARITY / PAGE_SIZE, Target); - Target -= PagesFreed; - (*NrFreed) += PagesFreed; - } - - KeReleaseSpinLockFromDpcLevel(&current->SharedCacheMap->CacheMapLock); - } - - KeReleaseQueuedSpinLock(LockQueueMasterLock, oldIrql); - - /* Try flushing pages if we haven't met our target */ - if ((Target > 0) && !FlushedPages) - { - /* Flush dirty pages to disk */ - CcRosFlushDirtyPages(Target, &PagesFreed, FALSE, FALSE); - FlushedPages = TRUE; - - /* We can only swap as many pages as we flushed */ - if (PagesFreed < Target) Target = PagesFreed; - - /* Check if we flushed anything */ - if (PagesFreed != 0) - { - /* Try again after flushing dirty pages */ - DPRINT("Flushed %lu dirty cache pages to disk\n", PagesFreed); - goto retry; - } - } - - while (!IsListEmpty(&FreeList)) - { - ULONG Refs; - - current_entry = RemoveHeadList(&FreeList); - current = CONTAINING_RECORD(current_entry, - ROS_VACB, - CacheMapVacbListEntry); - InitializeListHead(&current->CacheMapVacbListEntry); - Refs = CcRosVacbDecRefCount(current); - ASSERT(Refs == 0); - } - - DPRINT("Evicted %lu cache pages\n", (*NrFreed)); - - return STATUS_SUCCESS; -} - NTSTATUS NTAPI CcRosReleaseVacb ( @@ -504,6 +388,7 @@ CcRosMarkDirtyVacb ( { KIRQL oldIrql; PROS_SHARED_CACHE_MAP SharedCacheMap; + ULONG Length = VACB_MAPPING_GRANULARITY; SharedCacheMap = Vacb->SharedCacheMap; @@ -513,8 +398,12 @@ CcRosMarkDirtyVacb ( ASSERT(!Vacb->Dirty); InsertTailList(&DirtyVacbListHead, &Vacb->DirtyVacbListEntry); - CcTotalDirtyPages += VACB_MAPPING_GRANULARITY / PAGE_SIZE; - Vacb->SharedCacheMap->DirtyPages += VACB_MAPPING_GRANULARITY / PAGE_SIZE; +#if 0 + if (Vacb->FileOffset.QuadPart + Length > SharedCacheMap->SectionSize.QuadPart) + Length = SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart; +#endif + CcTotalDirtyPages += PAGE_ROUND_UP(Length) / PAGE_SIZE; + Vacb->SharedCacheMap->DirtyPages += PAGE_ROUND_UP(Length) / PAGE_SIZE; CcRosVacbIncRefCount(Vacb); /* Move to the tail of the LRU list */ @@ -531,6 +420,9 @@ CcRosMarkDirtyVacb ( CcScheduleLazyWriteScan(FALSE); } KeReleaseQueuedSpinLock(LockQueueMasterLock, oldIrql); + + /* Tell Mm */ + MmMakePagesDirty(NULL, Vacb->BaseAddress, Length); } VOID @@ -541,6 +433,7 @@ CcRosUnmarkDirtyVacb ( { KIRQL oldIrql; PROS_SHARED_CACHE_MAP SharedCacheMap; + ULONG Length = VACB_MAPPING_GRANULARITY; SharedCacheMap = Vacb->SharedCacheMap; @@ -556,8 +449,14 @@ CcRosUnmarkDirtyVacb ( RemoveEntryList(&Vacb->DirtyVacbListEntry); InitializeListHead(&Vacb->DirtyVacbListEntry); - CcTotalDirtyPages -= VACB_MAPPING_GRANULARITY / PAGE_SIZE; - Vacb->SharedCacheMap->DirtyPages -= VACB_MAPPING_GRANULARITY / PAGE_SIZE; + +#if 0 + if (Vacb->FileOffset.QuadPart + Length > SharedCacheMap->SectionSize.QuadPart) + Length = SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart; +#endif + CcTotalDirtyPages -= PAGE_ROUND_UP(Length) / PAGE_SIZE; + Vacb->SharedCacheMap->DirtyPages -= PAGE_ROUND_UP(Length) / PAGE_SIZE; + CcRosVacbDecRefCount(Vacb); if (LockViews) @@ -626,73 +525,6 @@ CcRosUnmapVacb ( return STATUS_SUCCESS; } -static -NTSTATUS -CcRosMapVacbInKernelSpace( - PROS_VACB Vacb) -{ - ULONG i; - NTSTATUS Status; - ULONG_PTR NumberOfPages; - PVOID BaseAddress = NULL; - - /* Create a memory area. */ - MmLockAddressSpace(MmGetKernelAddressSpace()); - Status = MmCreateMemoryArea(MmGetKernelAddressSpace(), - 0, // nothing checks for VACB mareas, so set to 0 - &BaseAddress, - VACB_MAPPING_GRANULARITY, - PAGE_READWRITE, - (PMEMORY_AREA*)&Vacb->MemoryArea, - 0, - PAGE_SIZE); - ASSERT(Vacb->BaseAddress == NULL); - Vacb->BaseAddress = BaseAddress; - MmUnlockAddressSpace(MmGetKernelAddressSpace()); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MmCreateMemoryArea failed with %lx for VACB %p\n", Status, Vacb); - return Status; - } - - ASSERT(((ULONG_PTR)Vacb->BaseAddress % PAGE_SIZE) == 0); - ASSERT((ULONG_PTR)Vacb->BaseAddress > (ULONG_PTR)MmSystemRangeStart); - ASSERT((ULONG_PTR)Vacb->BaseAddress + VACB_MAPPING_GRANULARITY - 1 > (ULONG_PTR)MmSystemRangeStart); - - /* Create a virtual mapping for this memory area */ - NumberOfPages = BYTES_TO_PAGES(VACB_MAPPING_GRANULARITY); - for (i = 0; i < NumberOfPages; i++) - { - PFN_NUMBER PageFrameNumber; - - MI_SET_USAGE(MI_USAGE_CACHE); - Status = MmRequestPageMemoryConsumer(MC_CACHE, TRUE, &PageFrameNumber); - if (PageFrameNumber == 0) - { - DPRINT1("Unable to allocate page\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - - ASSERT(BaseAddress == Vacb->BaseAddress); - ASSERT(i * PAGE_SIZE < VACB_MAPPING_GRANULARITY); - ASSERT((ULONG_PTR)Vacb->BaseAddress + (i * PAGE_SIZE) >= (ULONG_PTR)BaseAddress); - ASSERT((ULONG_PTR)Vacb->BaseAddress + (i * PAGE_SIZE) > (ULONG_PTR)MmSystemRangeStart); - - Status = MmCreateVirtualMapping(NULL, - (PVOID)((ULONG_PTR)Vacb->BaseAddress + (i * PAGE_SIZE)), - PAGE_READWRITE, - &PageFrameNumber, - 1); - if (!NT_SUCCESS(Status)) - { - DPRINT1("Unable to create virtual mapping\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - } - - return STATUS_SUCCESS; -} - static BOOLEAN CcRosFreeUnusedVacb ( @@ -789,6 +621,7 @@ CcRosCreateVacb ( KIRQL oldIrql; ULONG Refs; BOOLEAN Retried; + SIZE_T ViewSize = VACB_MAPPING_GRANULARITY; ASSERT(SharedCacheMap); @@ -823,8 +656,9 @@ CcRosCreateVacb ( Retried = FALSE; Retry: - /* Map VACB in kernel space */ - Status = CcRosMapVacbInKernelSpace(current); + /* Map VACB in system space */ + Status = MmMapViewInSystemSpaceEx(SharedCacheMap->Section, &current->BaseAddress, &ViewSize, &current->FileOffset); + if (!NT_SUCCESS(Status)) { ULONG Freed; @@ -932,14 +766,50 @@ Retry: return Status; } +BOOLEAN +NTAPI +CcRosEnsureVacbResident( + _In_ PROS_VACB Vacb, + _In_ BOOLEAN Wait, + _In_ BOOLEAN NoRead, + _In_ ULONG Offset, + _In_ ULONG Length +) +{ + PVOID BaseAddress; + + ASSERT((Offset + Length) <= VACB_MAPPING_GRANULARITY); + + if ((Vacb->FileOffset.QuadPart + Offset) > Vacb->SharedCacheMap->FileSize.QuadPart) + return FALSE; + + BaseAddress = (PVOID)((ULONG_PTR)Vacb->BaseAddress + Offset); + + /* Check if the pages are resident */ + if (!MmArePagesResident(NULL, BaseAddress, Length)) + { + if (!Wait) + { + return FALSE; + } + + if (!NoRead) + { + NTSTATUS Status = MmMakePagesResident(NULL, BaseAddress, Length); + if (!NT_SUCCESS(Status)) + ExRaiseStatus(Status); + } + } + + return TRUE; +} + + NTSTATUS NTAPI CcRosGetVacb ( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, - PLONGLONG BaseOffset, - PVOID* BaseAddress, - PBOOLEAN UptoDate, PROS_VACB *Vacb) { PROS_VACB current; @@ -978,13 +848,9 @@ CcRosGetVacb ( KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); /* - * Return information about the VACB to the caller. + * Return the VACB to the caller. */ - *UptoDate = current->Valid; - *BaseAddress = current->BaseAddress; - DPRINT("*BaseAddress %p\n", *BaseAddress); *Vacb = current; - *BaseOffset = current->FileOffset.QuadPart; ASSERT(Refs > 1); @@ -996,14 +862,11 @@ NTAPI CcRosRequestVacb ( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, - PVOID* BaseAddress, - PBOOLEAN UptoDate, PROS_VACB *Vacb) /* * FUNCTION: Request a page mapping for a shared cache map */ { - LONGLONG BaseOffset; ASSERT(SharedCacheMap); @@ -1016,30 +879,9 @@ CcRosRequestVacb ( return CcRosGetVacb(SharedCacheMap, FileOffset, - &BaseOffset, - BaseAddress, - UptoDate, Vacb); } -static -VOID -CcFreeCachePage ( - PVOID Context, - MEMORY_AREA* MemoryArea, - PVOID Address, - PFN_NUMBER Page, - SWAPENTRY SwapEntry, - BOOLEAN Dirty) -{ - ASSERT(SwapEntry == 0); - if (Page != 0) - { - ASSERT(MmGetReferenceCountPage(Page) == 1); - MmReleasePageMemoryConsumer(MC_CACHE, Page); - } -} - NTSTATUS CcRosInternalFreeVacb ( PROS_VACB Vacb) @@ -1047,6 +889,8 @@ CcRosInternalFreeVacb ( * FUNCTION: Releases a VACB associated with a shared cache map */ { + NTSTATUS Status; + DPRINT("Freeing VACB 0x%p\n", Vacb); #if DBG if (Vacb->SharedCacheMap->Trace) @@ -1055,12 +899,14 @@ CcRosInternalFreeVacb ( } #endif - MmLockAddressSpace(MmGetKernelAddressSpace()); - MmFreeMemoryArea(MmGetKernelAddressSpace(), - Vacb->MemoryArea, - CcFreeCachePage, - NULL); - MmUnlockAddressSpace(MmGetKernelAddressSpace()); + /* Delete the mapping */ + Status = MmUnmapViewInSystemSpace(Vacb->BaseAddress); + if (!NT_SUCCESS(Status)) + { + DPRINT1("Failed to unmap VACB from System address space! Status 0x%08X\n", Status); + ASSERT(FALSE); + /* Proceed with the deĺetion anyway */ + } if (Vacb->ReferenceCount != 0) { @@ -1097,11 +943,8 @@ CcFlushCache ( PROS_VACB current; NTSTATUS Status; - CCTRACE(CC_API_DEBUG, "SectionObjectPointers=%p FileOffset=%p Length=%lu\n", - SectionObjectPointers, FileOffset, Length); - - DPRINT("CcFlushCache(SectionObjectPointers 0x%p, FileOffset 0x%p, Length %lu, IoStatus 0x%p)\n", - SectionObjectPointers, FileOffset, Length, IoStatus); + CCTRACE(CC_API_DEBUG, "SectionObjectPointers=%p FileOffset=0x%I64X Length=%lu\n", + SectionObjectPointers, FileOffset ? FileOffset->QuadPart : 0LL, Length); if (SectionObjectPointers && SectionObjectPointers->SharedCacheMap) { @@ -1217,6 +1060,8 @@ CcRosDeleteFileCache ( KeReleaseSpinLockFromDpcLevel(&SharedCacheMap->CacheMapLock); KeReleaseQueuedSpinLock(LockQueueMasterLock, *OldIrql); + if(SharedCacheMap->Section) + ObDereferenceObject(SharedCacheMap->Section); ObDereferenceObject(SharedCacheMap->FileObject); while (!IsListEmpty(&FreeList)) @@ -1299,36 +1144,6 @@ CcRosRemoveIfClosed ( KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); } - -VOID -NTAPI -CcRosDereferenceCache ( - PFILE_OBJECT FileObject) -{ - PROS_SHARED_CACHE_MAP SharedCacheMap; - KIRQL OldIrql; - - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - ASSERT(SharedCacheMap); - if (SharedCacheMap->OpenCount > 0) - { - SharedCacheMap->OpenCount--; - if (SharedCacheMap->OpenCount == 0) - { - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); - MmFreeSectionSegments(SharedCacheMap->FileObject); - - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - CcRosDeleteFileCache(FileObject, SharedCacheMap, &OldIrql); - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); - - return; - } - } - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); -} - NTSTATUS NTAPI CcRosReleaseFileCache ( @@ -1373,20 +1188,12 @@ CcRosReleaseFileCache ( PrivateMap->NodeTypeCode = 0; } - if (SharedCacheMap->OpenCount > 0) - { - SharedCacheMap->OpenCount--; - if (SharedCacheMap->OpenCount == 0) - { - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); - MmFreeSectionSegments(SharedCacheMap->FileObject); - - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - CcRosDeleteFileCache(FileObject, SharedCacheMap, &OldIrql); - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); + ASSERT(SharedCacheMap->OpenCount > 0); - return STATUS_SUCCESS; - } + SharedCacheMap->OpenCount--; + if (SharedCacheMap->OpenCount == 0) + { + CcRosDeleteFileCache(FileObject, SharedCacheMap, &OldIrql); } } } @@ -1412,6 +1219,8 @@ CcRosInitializeFileCache ( DPRINT("CcRosInitializeFileCache(FileObject 0x%p)\n", FileObject); + OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); + Allocated = FALSE; SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; if (SharedCacheMap == NULL) @@ -1437,27 +1246,37 @@ CcRosInitializeFileCache ( KeInitializeSpinLock(&SharedCacheMap->CacheMapLock); InitializeListHead(&SharedCacheMap->CacheMapVacbListHead); InitializeListHead(&SharedCacheMap->BcbList); - } - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - if (Allocated) + SharedCacheMap->Flags = SHARED_CACHE_MAP_IN_CREATION; + + ObReferenceObjectByPointer(FileObject, + FILE_ALL_ACCESS, + NULL, + KernelMode); + + FileObject->SectionObjectPointer->SharedCacheMap = SharedCacheMap; + + // CcRosTraceCacheMap(SharedCacheMap, TRUE); + } + else if (SharedCacheMap->Flags & SHARED_CACHE_MAP_IN_CREATION) { - if (FileObject->SectionObjectPointer->SharedCacheMap == NULL) - { - ObReferenceObjectByPointer(FileObject, - FILE_ALL_ACCESS, - NULL, - KernelMode); - FileObject->SectionObjectPointer->SharedCacheMap = SharedCacheMap; + /* The shared cache map is being created somewhere else. Wait for that to happen */ + KEVENT Waiter; + PKEVENT PreviousWaiter = SharedCacheMap->CreateEvent; - InsertTailList(&CcCleanSharedCacheMapList, &SharedCacheMap->SharedCacheMapLinks); - } - else - { - ExFreeToNPagedLookasideList(&SharedCacheMapLookasideList, SharedCacheMap); - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - } + KeInitializeEvent(&Waiter, NotificationEvent, FALSE); + SharedCacheMap->CreateEvent = &Waiter; + + KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); + + KeWaitForSingleObject(&Waiter, Executive, KernelMode, FALSE, NULL); + + if (PreviousWaiter) + KeSetEvent(PreviousWaiter, IO_NO_INCREMENT, FALSE); + + OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); } + if (FileObject->PrivateCacheMap == NULL) { PPRIVATE_CACHE_MAP PrivateMap; @@ -1503,8 +1322,48 @@ CcRosInitializeFileCache ( FileObject->PrivateCacheMap = PrivateMap; SharedCacheMap->OpenCount++; } + KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); + /* Create the section */ + if (Allocated) + { + NTSTATUS Status; + + ASSERT(SharedCacheMap->Section == NULL); + + Status = MmCreateSection( + &SharedCacheMap->Section, + SECTION_ALL_ACCESS, + NULL, + &SharedCacheMap->SectionSize, + PAGE_READWRITE, + 0, + NULL, + FileObject); + + ASSERT(NT_SUCCESS(Status)); + + if (!NT_SUCCESS(Status)) + { + CcRosReleaseFileCache(FileObject); + return Status; + } + + OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); + + InsertTailList(&CcCleanSharedCacheMapList, &SharedCacheMap->SharedCacheMapLinks); + SharedCacheMap->Flags &= ~SHARED_CACHE_MAP_IN_CREATION; + + if (SharedCacheMap->CreateEvent) + { + KeSetEvent(SharedCacheMap->CreateEvent, IO_NO_INCREMENT, FALSE); + SharedCacheMap->CreateEvent = NULL; + } + + KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); + } + return STATUS_SUCCESS; } @@ -1564,8 +1423,6 @@ CcInitView ( TAG_VACB, 20); - MmInitializeMemoryConsumer(MC_CACHE, CcRosTrimCache); - CcInitCacheZeroPage(); } diff --git a/ntoskrnl/ex/init.c b/ntoskrnl/ex/init.c index 8eb867b09cd..fc98cccd430 100644 --- a/ntoskrnl/ex/init.c +++ b/ntoskrnl/ex/init.c @@ -1970,7 +1970,6 @@ Phase1InitializationDiscard(IN PVOID Context) InbvEnableDisplayString(TRUE); /* Launch initial process */ - DPRINT("Free non-cache pages: %lx\n", MmAvailablePages + MiMemoryConsumers[MC_CACHE].PagesUsed); ProcessInfo = &InitBuffer->ProcessInfo; ExpLoadInitialProcess(InitBuffer, &ProcessParameters, &Environment); @@ -2009,7 +2008,6 @@ Phase1InitializationDiscard(IN PVOID Context) /* Free the boot buffer */ ExFreePoolWithTag(InitBuffer, TAG_INIT); - DPRINT("Free non-cache pages: %lx\n", MmAvailablePages + MiMemoryConsumers[MC_CACHE].PagesUsed); } VOID diff --git a/ntoskrnl/ex/sysinfo.c b/ntoskrnl/ex/sysinfo.c index 64cbffc5ba1..f595623b4c8 100644 --- a/ntoskrnl/ex/sysinfo.c +++ b/ntoskrnl/ex/sysinfo.c @@ -719,7 +719,6 @@ QSI_DEF(SystemPerformanceInformation) * Not sure this is right. 8^\ */ Spi->CommittedPages = MiMemoryConsumers[MC_SYSTEM].PagesUsed + - MiMemoryConsumers[MC_CACHE].PagesUsed + MiMemoryConsumers[MC_USER].PagesUsed + MiUsedSwapPages; /* @@ -767,7 +766,7 @@ QSI_DEF(SystemPerformanceInformation) Spi->TotalSystemDriverPages = 0; /* FIXME */ Spi->Spare3Count = 0; /* FIXME */ - Spi->ResidentSystemCachePage = MiMemoryConsumers[MC_CACHE].PagesUsed; + Spi->ResidentSystemCachePage = 0; /* FIXME */ Spi->ResidentPagedPoolPage = 0; /* FIXME */ Spi->ResidentSystemDriverPage = 0; /* FIXME */ @@ -1477,13 +1476,10 @@ QSI_DEF(SystemFileCacheInformation) RtlZeroMemory(Sci, sizeof(SYSTEM_FILECACHE_INFORMATION)); /* Return the Byte size not the page size. */ - Sci->CurrentSize = - MiMemoryConsumers[MC_CACHE].PagesUsed * PAGE_SIZE; - Sci->PeakSize = - MiMemoryConsumers[MC_CACHE].PagesUsed * PAGE_SIZE; /* FIXME */ + Sci->CurrentSize = 0; /* FIXME */ + Sci->PeakSize = 0; /* FIXME */ /* Taskmgr multiplies this one by page size right away */ - Sci->CurrentSizeIncludingTransitionInPages = - MiMemoryConsumers[MC_CACHE].PagesUsed; /* FIXME: Should be */ + Sci->CurrentSizeIncludingTransitionInPages = 0; /* FIXME: Should be */ /* system working set and standby pages. */ Sci->PageFaultCount = 0; /* FIXME */ Sci->MinimumWorkingSet = 0; /* FIXME */ diff --git a/ntoskrnl/include/internal/cc.h b/ntoskrnl/include/internal/cc.h index 18c0b9effa2..9e186a61d11 100644 --- a/ntoskrnl/include/internal/cc.h +++ b/ntoskrnl/include/internal/cc.h @@ -3,7 +3,7 @@ // // Define this if you want debugging support // -#define _CC_DEBUG_ 0x00 +#define _CC_DEBUG_ 0x0 // // These define the Debug Masks Supported @@ -179,6 +179,8 @@ typedef struct _ROS_SHARED_CACHE_MAP ULONG DirtyPages; LIST_ENTRY SharedCacheMapLinks; ULONG Flags; + PVOID Section; + PKEVENT CreateEvent; PCACHE_MANAGER_CALLBACKS Callbacks; PVOID LazyWriteContext; LIST_ENTRY PrivateList; @@ -197,13 +199,12 @@ typedef struct _ROS_SHARED_CACHE_MAP #define READAHEAD_DISABLED 0x1 #define WRITEBEHIND_DISABLED 0x2 +#define SHARED_CACHE_MAP_IN_CREATION 0x4 typedef struct _ROS_VACB { /* Base address of the region where the view's data is mapped. */ PVOID BaseAddress; - /* Memory area representing the region where the view's data is mapped. */ - struct _MEMORY_AREA* MemoryArea; /* Are the contents of the view valid. */ BOOLEAN Valid; /* Are the contents of the view newer than those on disk. */ @@ -316,12 +317,19 @@ NTAPI CcRosGetVacb( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, - PLONGLONG BaseOffset, - PVOID *BaseAddress, - PBOOLEAN UptoDate, PROS_VACB *Vacb ); +BOOLEAN +NTAPI +CcRosEnsureVacbResident( + _In_ PROS_VACB Vacb, + _In_ BOOLEAN Wait, + _In_ BOOLEAN NoRead, + _In_ ULONG Offset, + _In_ ULONG Length +); + VOID NTAPI CcInitView(VOID); @@ -330,14 +338,6 @@ VOID NTAPI CcShutdownLazyWriter(VOID); -NTSTATUS -NTAPI -CcReadVirtualAddress(PROS_VACB Vacb); - -NTSTATUS -NTAPI -CcWriteVirtualAddress(PROS_VACB Vacb); - BOOLEAN NTAPI CcInitializeCacheManager(VOID); @@ -415,8 +415,6 @@ NTAPI CcRosRequestVacb( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, - PVOID* BaseAddress, - PBOOLEAN UptoDate, PROS_VACB *Vacb ); diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index be8b0acc4b0..55e9fcfc29c 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -89,12 +89,9 @@ typedef ULONG_PTR SWAPENTRY; #define SEC_PHYSICALMEMORY (0x80000000) -#define MM_DATAFILE_SEGMENT (0x2) - -#define MC_CACHE (0) -#define MC_USER (1) -#define MC_SYSTEM (2) -#define MC_MAXIMUM (3) +#define MC_USER (0) +#define MC_SYSTEM (1) +#define MC_MAXIMUM (2) #define PAGED_POOL_MASK 1 #define MUST_SUCCEED_POOL_MASK 2 @@ -171,10 +168,10 @@ typedef struct _MM_SECTION_SEGMENT FAST_MUTEX Lock; /* lock which protects the page directory */ LARGE_INTEGER RawLength; /* length of the segment which is part of the mapped file */ LARGE_INTEGER Length; /* absolute length of the segment */ - ULONG ReferenceCount; - ULONG CacheCount; + PULONG ReferenceCount; + ULONG SectionCount; ULONG Protection; - ULONG Flags; + PULONG Flags; BOOLEAN WriteCopy; BOOLEAN Locked; @@ -185,6 +182,9 @@ typedef struct _MM_SECTION_SEGMENT ULONG Characteristics; } Image; + ULONG RefCount; + ULONG SegFlags; + LIST_ENTRY ListOfSegments; RTL_GENERIC_TABLE PageTable; } MM_SECTION_SEGMENT, *PMM_SECTION_SEGMENT; @@ -193,12 +193,20 @@ typedef struct _MM_IMAGE_SECTION_OBJECT { PFILE_OBJECT FileObject; + ULONG RefCount; + ULONG SegFlags; + SECTION_IMAGE_INFORMATION ImageInformation; PVOID BasedAddress; ULONG NrSegments; PMM_SECTION_SEGMENT Segments; } MM_IMAGE_SECTION_OBJECT, *PMM_IMAGE_SECTION_OBJECT; +#define MM_DATAFILE_SEGMENT (0x2) +#define MM_SEGMENT_INDELETE (0x4) +#define MM_SEGMENT_INCREATE (0x8) + + #define MA_GetStartingAddress(_MemoryArea) ((_MemoryArea)->VadNode.StartingVpn << PAGE_SHIFT) #define MA_GetEndingAddress(_MemoryArea) (((_MemoryArea)->VadNode.EndingVpn + 1) << PAGE_SHIFT) @@ -862,11 +870,6 @@ MmInitializeRmapList(VOID); VOID NTAPI MmSetCleanAllRmaps(PFN_NUMBER Page); - -VOID -NTAPI -MmSetDirtyAllRmaps(PFN_NUMBER Page); - BOOLEAN NTAPI MmIsDirtyPageRmap(PFN_NUMBER Page); @@ -1288,15 +1291,6 @@ MmNotPresentFaultSectionView( BOOLEAN Locked ); -NTSTATUS -NTAPI -MmPageOutSectionView( - PMMSUPPORT AddressSpace, - PMEMORY_AREA MemoryArea, - PVOID Address, - ULONG_PTR Entry -); - NTSTATUS NTAPI MmCreatePhysicalMemorySection(VOID); @@ -1337,6 +1331,48 @@ MmMakePagesResident( _In_ PVOID Address, _In_ ULONG Length); +NTSTATUS +NTAPI +MmMakePagesDirty( + _In_ PEPROCESS Process, + _In_ PVOID Address, + _In_ ULONG Length); + +NTSTATUS +NTAPI +MmRosFlushVirtualMemory( + _In_ PEPROCESS Process, + _Inout_ PVOID* Address, + _Inout_ PSIZE_T Length, + _Out_ PIO_STATUS_BLOCK Iosb); + +BOOLEAN +NTAPI +MmCheckDirtySegment( + PMM_SECTION_SEGMENT Segment, + PLARGE_INTEGER Offset, + BOOLEAN ForceDirty, + BOOLEAN PageOut); + +BOOLEAN +NTAPI +MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, + PMM_SECTION_SEGMENT Segment, + PLARGE_INTEGER Offset, + BOOLEAN Dirty, + BOOLEAN PageOut, + ULONG_PTR *InEntry); + +VOID +NTAPI +MmDereferenceSegment(PMM_SECTION_SEGMENT Segment); + +NTSTATUS +NTAPI +MmExtendSection( + _In_ PVOID Section, + _Inout_ PLARGE_INTEGER NewSize); + /* sysldr.c ******************************************************************/ VOID diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index 938a08d6037..9ea36637a22 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -2934,7 +2934,7 @@ MmMapViewOfArm3Section(IN PVOID SectionObject, if (!(*ViewSize)) { /* Compute it for the caller */ - CalculatedViewSize = Section->SizeOfSection.QuadPart - + CalculatedViewSize = Section->SizeOfSection.QuadPart - SectionOffset->QuadPart; /* Check if it's larger than 4GB or overflows into kernel-mode */ @@ -3891,30 +3891,24 @@ NtExtendSection(IN HANDLE SectionHandle, NULL); if (!NT_SUCCESS(Status)) return Status; - /* Really this should go in MmExtendSection */ - if (!Section->u.Flags.File || Section->u.Flags.Image) - { - DPRINT1("Not extending a file\n"); - ObDereferenceObject(Section); - return STATUS_SECTION_NOT_EXTENDED; - } - - /* FIXME: Do the work */ + Status = MmExtendSection(Section, &SafeNewMaximumSize); /* Dereference the section */ ObDereferenceObject(Section); - /* Enter SEH */ - _SEH2_TRY - { - /* Write back the new size */ - *NewMaximumSize = SafeNewMaximumSize; - } - _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) + if (NT_SUCCESS(Status)) { - /* Nothing to do */ + _SEH2_TRY + { + /* Write back the new size */ + *NewMaximumSize = SafeNewMaximumSize; + } + _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) + { + Status = _SEH2_GetExceptionCode(); + } + _SEH2_END; } - _SEH2_END; /* Return the status */ return STATUS_NOT_IMPLEMENTED; diff --git a/ntoskrnl/mm/ARM3/virtual.c b/ntoskrnl/mm/ARM3/virtual.c index 19eb8a69b01..00271e03e7b 100644 --- a/ntoskrnl/mm/ARM3/virtual.c +++ b/ntoskrnl/mm/ARM3/virtual.c @@ -1356,12 +1356,8 @@ MmFlushVirtualMemory(IN PEPROCESS Process, OUT PIO_STATUS_BLOCK IoStatusBlock) { PAGED_CODE(); - UNIMPLEMENTED; - - // - // Fake success - // - return STATUS_SUCCESS; + /* For now we call the old Mm */ + return MmRosFlushVirtualMemory(Process, BaseAddress, RegionSize, IoStatusBlock); } ULONG diff --git a/ntoskrnl/mm/ARM3/zeropage.c b/ntoskrnl/mm/ARM3/zeropage.c index 5fe22294199..80725e8a5a7 100644 --- a/ntoskrnl/mm/ARM3/zeropage.c +++ b/ntoskrnl/mm/ARM3/zeropage.c @@ -46,7 +46,7 @@ MmZeroPageThread(VOID) /* Get the discardable sections to free them */ MiFindInitializationCode(&StartAddress, &EndAddress); if (StartAddress) MiFreeInitializationCode(StartAddress, EndAddress); - DPRINT("Free non-cache pages: %lx\n", MmAvailablePages + MiMemoryConsumers[MC_CACHE].PagesUsed); + DPRINT("Free pages: %lx\n", MmAvailablePages); /* Set our priority to 0 */ Thread->BasePriority = 0; diff --git a/ntoskrnl/mm/balance.c b/ntoskrnl/mm/balance.c index 5cd5e267a23..29c0dfc6f47 100644 --- a/ntoskrnl/mm/balance.c +++ b/ntoskrnl/mm/balance.c @@ -54,18 +54,6 @@ MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages) /* Set up targets. */ MiMinimumAvailablePages = 256; MiMinimumPagesPerRun = 256; - if ((NrAvailablePages + NrSystemPages) >= 8192) - { - MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3; - } - else if ((NrAvailablePages + NrSystemPages) >= 4096) - { - MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2; - } - else - { - MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8; - } MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages; } diff --git a/ntoskrnl/mm/pagefile.c b/ntoskrnl/mm/pagefile.c index 37870520998..e0f9e087150 100644 --- a/ntoskrnl/mm/pagefile.c +++ b/ntoskrnl/mm/pagefile.c @@ -94,9 +94,6 @@ NTAPI MmBuildMdlFromPages(PMDL Mdl, PPFN_NUMBER Pages) { memcpy(Mdl + 1, Pages, sizeof(PFN_NUMBER) * (PAGE_ROUND_UP(Mdl->ByteOffset+Mdl->ByteCount)/PAGE_SIZE)); - - /* FIXME: this flag should be set by the caller perhaps? */ - Mdl->MdlFlags |= MDL_IO_PAGE_READ; } @@ -230,7 +227,7 @@ MiReadPageFile( MmInitializeMdl(Mdl, NULL, PAGE_SIZE); MmBuildMdlFromPages(Mdl, &Page); - Mdl->MdlFlags |= MDL_PAGES_LOCKED; + Mdl->MdlFlags |= MDL_PAGES_LOCKED | MDL_IO_PAGE_READ; file_offset.QuadPart = PageFileOffset * PAGE_SIZE; diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index 9f5938d7496..d1cfb584dce 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -19,7 +19,6 @@ /* GLOBALS ******************************************************************/ static NPAGED_LOOKASIDE_LIST RmapLookasideList; -FAST_MUTEX RmapListLock; /* FUNCTIONS ****************************************************************/ @@ -38,7 +37,6 @@ VOID NTAPI MmInitializeRmapList(VOID) { - ExInitializeFastMutex(&RmapListLock); ExInitializeNPagedLookasideList (&RmapLookasideList, NULL, RmapListFree, @@ -55,37 +53,27 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) PMM_RMAP_ENTRY entry; PMEMORY_AREA MemoryArea; PMMSUPPORT AddressSpace; - ULONG Type; PVOID Address; PEPROCESS Process; - ULONGLONG Offset; NTSTATUS Status = STATUS_SUCCESS; + PMM_SECTION_SEGMENT Segment; + LARGE_INTEGER SegmentOffset; + KIRQL OldIrql; - ExAcquireFastMutex(&RmapListLock); - entry = MmGetRmapListHeadPage(Page); + OldIrql = MiAcquirePfnLock(); -#ifdef NEWCC - // Special case for NEWCC: we can have a page that's only in a segment - // page table - if (entry && RMAP_IS_SEGMENT(entry->Address) && entry->Next == NULL) - { - /* NEWCC does locking itself */ - ExReleaseFastMutex(&RmapListLock); - return MmpPageOutPhysicalAddress(Page); - } -#endif + entry = MmGetRmapListHeadPage(Page); while (entry && RMAP_IS_SEGMENT(entry->Address)) entry = entry->Next; if (entry == NULL) { - ExReleaseFastMutex(&RmapListLock); - return(STATUS_UNSUCCESSFUL); + MiReleasePfnLock(OldIrql); + return STATUS_UNSUCCESSFUL; } Process = entry->Process; - Address = entry->Address; if ((((ULONG_PTR)Address) & 0xFFF) != 0) @@ -97,12 +85,12 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) { if (!ExAcquireRundownProtection(&Process->RundownProtect)) { - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); return STATUS_PROCESS_IS_TERMINATING; } Status = ObReferenceObjectByPointer(Process, PROCESS_ALL_ACCESS, NULL, KernelMode); - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); if (!NT_SUCCESS(Status)) { ExReleaseRundownProtection(&Process->RundownProtect); @@ -112,11 +100,12 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) } else { - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); AddressSpace = MmGetKernelAddressSpace(); } MmLockAddressSpace(AddressSpace); + MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); if (MemoryArea == NULL || MemoryArea->DeleteInProgress) { @@ -128,23 +117,27 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) } return(STATUS_UNSUCCESSFUL); } - Type = MemoryArea->Type; - if (Type == MEMORY_AREA_SECTION_VIEW) + + if (MemoryArea->Type == MEMORY_AREA_SECTION_VIEW) { ULONG_PTR Entry; - Offset = MemoryArea->SectionData.ViewOffset.QuadPart + + BOOLEAN Dirty; + PFN_NUMBER MapPage; + LARGE_INTEGER Offset; + BOOLEAN Released; + + Offset.QuadPart = MemoryArea->SectionData.ViewOffset.QuadPart + ((ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea)); - MmLockSectionSegment(MemoryArea->SectionData.Segment); + Segment = MemoryArea->SectionData.Segment; - /* - * Get or create a pageop - */ - Entry = MmGetPageEntrySectionSegment(MemoryArea->SectionData.Segment, - (PLARGE_INTEGER)&Offset); + MmLockSectionSegment(Segment); + + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); if (Entry && MM_IS_WAIT_PTE(Entry)) { - MmUnlockSectionSegment(MemoryArea->SectionData.Segment); + /* The segment is being read or something. Give up */ + MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); if (Address < MmSystemRangeStart) { @@ -154,18 +147,101 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) return(STATUS_UNSUCCESSFUL); } - MmSetPageEntrySectionSegment(MemoryArea->SectionData.Segment, (PLARGE_INTEGER)&Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); + /* Delete this virtual mapping in the process */ + MmDeleteVirtualMapping(Process, Address, &Dirty, &MapPage); + ASSERT(MapPage == Page); + + if (Page != PFN_FROM_SSE(Entry)) + { + SWAPENTRY SwapEntry; + + /* This page is private to the process */ + MmUnlockSectionSegment(Segment); + + /* Check if we should write it back to the page file */ + SwapEntry = MmGetSavedSwapEntryPage(Page); + + if ((SwapEntry == 0) && Dirty) + { + /* We don't have a Swap entry, yet the page is dirty. Get one */ + SwapEntry = MmAllocSwapPage(); + if (!SwapEntry) + { + PMM_REGION Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), + &MemoryArea->SectionData.RegionListHead, + Address, NULL); + + /* We can't, so let this page in the Process VM */ + MmCreateVirtualMapping(Process, Address, Region->Protect, &Page, 1); + MmSetDirtyPage(Process, Address); + + MmUnlockAddressSpace(AddressSpace); + return STATUS_UNSUCCESSFUL; + } + } + + if (Dirty) + { + Status = MmWriteToSwapPage(SwapEntry, Page); + if (!NT_SUCCESS(Status)) + { + /* We failed at saving the content of this page. Keep it in */ + PMM_REGION Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), + &MemoryArea->SectionData.RegionListHead, + Address, NULL); + + /* This Swap Entry is useless to us */ + MmSetSavedSwapEntryPage(Page, 0); + MmFreeSwapPage(SwapEntry); + + /* We can't, so let this page in the Process VM */ + MmCreateVirtualMapping(Process, Address, Region->Protect, &Page, 1); + MmSetDirtyPage(Process, Address); + + MmUnlockAddressSpace(AddressSpace); + return STATUS_UNSUCCESSFUL; + } + } + + if (SwapEntry) + { + /* Keep this in the process VM */ + MmCreatePageFileMapping(Process, Address, SwapEntry); + MmSetSavedSwapEntryPage(Page, 0); + } + + MmUnlockAddressSpace(AddressSpace); + + /* We can finally let this page go */ + MmDeleteRmap(Page, Process, Address); + MmReleasePageMemoryConsumer(MC_USER, Page); + + ASSERT(MmGetRmapListHeadPage(Page) == NULL); + + if (Address < MmSystemRangeStart) + { + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + } + return STATUS_SUCCESS; + } + + /* Delete this RMAP */ + MmDeleteRmap(Page, Process, Address); - /* - * Release locks now we have a page op. - */ - MmUnlockSectionSegment(MemoryArea->SectionData.Segment); + /* One less mapping referencing this segment */ + Released = MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, FALSE, NULL); + + MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); - /* - * Do the actual page out work. - */ - Status = MmPageOutSectionView(AddressSpace, MemoryArea, Address, Entry); + if (Address < MmSystemRangeStart) + { + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + } + + if (Released) return STATUS_SUCCESS; } #ifdef NEWCC else if (Type == MEMORY_AREA_CACHE) @@ -185,51 +261,52 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) ExReleaseRundownProtection(&Process->RundownProtect); ObDereferenceObject(Process); } - return(Status); -} - -VOID -NTAPI -MmSetCleanAllRmaps(PFN_NUMBER Page) -{ - PMM_RMAP_ENTRY current_entry; - ExAcquireFastMutex(&RmapListLock); - current_entry = MmGetRmapListHeadPage(Page); - if (current_entry == NULL) - { - DPRINT1("MmIsDirtyRmap: No rmaps.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - while (current_entry != NULL) + /* Now write this page to file, if needed */ + Segment = MmGetSectionAssociation(Page, &SegmentOffset); + if (Segment) { - if (!RMAP_IS_SEGMENT(current_entry->Address)) - MmSetCleanPage(current_entry->Process, current_entry->Address); - current_entry = current_entry->Next; + BOOLEAN Released; + + MmLockSectionSegment(Segment); + + Released = MmCheckDirtySegment(Segment, &SegmentOffset, FALSE, TRUE); + + MmUnlockSectionSegment(Segment); + + MmDereferenceSegment(Segment); + + if (Released) + { + return STATUS_SUCCESS; + } } - ExReleaseFastMutex(&RmapListLock); + + /* If we are here, then we didn't release the page */ + return STATUS_UNSUCCESSFUL; } VOID NTAPI -MmSetDirtyAllRmaps(PFN_NUMBER Page) +MmSetCleanAllRmaps(PFN_NUMBER Page) { PMM_RMAP_ENTRY current_entry; + KIRQL OldIrql; - ExAcquireFastMutex(&RmapListLock); + OldIrql = MiAcquirePfnLock(); current_entry = MmGetRmapListHeadPage(Page); if (current_entry == NULL) { - DPRINT1("MmIsDirtyRmap: No rmaps.\n"); + DPRINT1("MmSetCleanAllRmaps: No rmaps.\n"); KeBugCheck(MEMORY_MANAGEMENT); } while (current_entry != NULL) { if (!RMAP_IS_SEGMENT(current_entry->Address)) - MmSetDirtyPage(current_entry->Process, current_entry->Address); + MmSetCleanPage(current_entry->Process, current_entry->Address); current_entry = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); } BOOLEAN @@ -237,27 +314,31 @@ NTAPI MmIsDirtyPageRmap(PFN_NUMBER Page) { PMM_RMAP_ENTRY current_entry; + KIRQL OldIrql; + BOOLEAN Dirty = FALSE; - ExAcquireFastMutex(&RmapListLock); + OldIrql = MiAcquirePfnLock(); current_entry = MmGetRmapListHeadPage(Page); if (current_entry == NULL) { - ExReleaseFastMutex(&RmapListLock); - return(FALSE); + DPRINT1("MmIsDirtyPageRmap: No rmaps.\n"); + KeBugCheck(MEMORY_MANAGEMENT); } while (current_entry != NULL) { - if ( - !RMAP_IS_SEGMENT(current_entry->Address) && - MmIsDirtyPage(current_entry->Process, current_entry->Address)) + if (!RMAP_IS_SEGMENT(current_entry->Address)) { - ExReleaseFastMutex(&RmapListLock); - return(TRUE); + if (MmIsDirtyPage(current_entry->Process, current_entry->Address)) + { + Dirty = TRUE; + break; + } } current_entry = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); - return(FALSE); + MiReleasePfnLock(OldIrql); + + return Dirty; } VOID @@ -268,6 +349,8 @@ MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process, PMM_RMAP_ENTRY current_entry; PMM_RMAP_ENTRY new_entry; ULONG PrevSize; + KIRQL OldIrql; + if (!RMAP_IS_SEGMENT(Address)) Address = (PVOID)PAGE_ROUND_DOWN(Address); @@ -298,7 +381,7 @@ MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process, KeBugCheck(MEMORY_MANAGEMENT); } - ExAcquireFastMutex(&RmapListLock); + OldIrql = MiAcquirePfnLock(); current_entry = MmGetRmapListHeadPage(Page); new_entry->Next = current_entry; #if DBG @@ -318,7 +401,8 @@ MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process, } #endif MmSetRmapListHeadPage(Page, new_entry); - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); + if (!RMAP_IS_SEGMENT(Address)) { if (Process == NULL) @@ -336,63 +420,15 @@ MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process, } } -VOID -NTAPI -MmDeleteAllRmaps(PFN_NUMBER Page, PVOID Context, - VOID (*DeleteMapping)(PVOID Context, PEPROCESS Process, - PVOID Address)) -{ - PMM_RMAP_ENTRY current_entry; - PMM_RMAP_ENTRY previous_entry; - PEPROCESS Process; - - ExAcquireFastMutex(&RmapListLock); - current_entry = MmGetRmapListHeadPage(Page); - if (current_entry == NULL) - { - DPRINT1("MmDeleteAllRmaps: No rmaps.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - MmSetRmapListHeadPage(Page, NULL); - ExReleaseFastMutex(&RmapListLock); - - while (current_entry != NULL) - { - previous_entry = current_entry; - current_entry = current_entry->Next; - if (!RMAP_IS_SEGMENT(previous_entry->Address)) - { - if (DeleteMapping) - { - DeleteMapping(Context, previous_entry->Process, - previous_entry->Address); - } - Process = previous_entry->Process; - ExFreeToNPagedLookasideList(&RmapLookasideList, previous_entry); - if (Process == NULL) - { - Process = PsInitialSystemProcess; - } - if (Process) - { - (void)InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, -PAGE_SIZE); - } - } - else - { - ExFreeToNPagedLookasideList(&RmapLookasideList, previous_entry); - } - } -} - VOID NTAPI MmDeleteRmap(PFN_NUMBER Page, PEPROCESS Process, PVOID Address) { PMM_RMAP_ENTRY current_entry, previous_entry; + KIRQL OldIrql; - ExAcquireFastMutex(&RmapListLock); + OldIrql = MiAcquirePfnLock(); previous_entry = NULL; current_entry = MmGetRmapListHeadPage(Page); @@ -409,7 +445,8 @@ MmDeleteRmap(PFN_NUMBER Page, PEPROCESS Process, { previous_entry->Next = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); + ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry); if (!RMAP_IS_SEGMENT(Address)) { @@ -450,8 +487,8 @@ MmGetSegmentRmap(PFN_NUMBER Page, PULONG RawOffset) { PCACHE_SECTION_PAGE_TABLE Result = NULL; PMM_RMAP_ENTRY current_entry;//, previous_entry; + KIRQL OldIrql = MiAcquirePfnLock(); - ExAcquireFastMutex(&RmapListLock); //previous_entry = NULL; current_entry = MmGetRmapListHeadPage(Page); while (current_entry != NULL) @@ -460,14 +497,20 @@ MmGetSegmentRmap(PFN_NUMBER Page, PULONG RawOffset) { Result = (PCACHE_SECTION_PAGE_TABLE)current_entry->Process; *RawOffset = (ULONG_PTR)current_entry->Address & ~RMAP_SEGMENT_MASK; - InterlockedIncrementUL(&Result->Segment->ReferenceCount); - ExReleaseFastMutex(&RmapListLock); + if (*Result->Segment->Flags & MM_SEGMENT_INDELETE) + { + MiReleasePfnLock(OldIrql); + return NULL; + } + + InterlockedIncrementUL(Result->Segment->ReferenceCount); + MiReleasePfnLock(OldIrql); return Result; } //previous_entry = current_entry; current_entry = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); return NULL; } @@ -482,8 +525,8 @@ NTAPI MmDeleteSectionAssociation(PFN_NUMBER Page) { PMM_RMAP_ENTRY current_entry, previous_entry; + KIRQL OldIrql = MiAcquirePfnLock(); - ExAcquireFastMutex(&RmapListLock); previous_entry = NULL; current_entry = MmGetRmapListHeadPage(Page); while (current_entry != NULL) @@ -498,12 +541,12 @@ MmDeleteSectionAssociation(PFN_NUMBER Page) { previous_entry->Next = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry); return; } previous_entry = current_entry; current_entry = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); } diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 4d29f5a9ff4..39ae8bf75f5 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -193,6 +193,58 @@ static GENERIC_MAPPING MmpSectionMapping = /* FUNCTIONS *****************************************************************/ + +NTSTATUS +NTAPI +MiWritePage(PMM_SECTION_SEGMENT Segment, + LONGLONG SegOffset, + PFN_NUMBER Page) +/* + * FUNCTION: write a page for a section backed memory area. + * PARAMETERS: + * MemoryArea - Memory area to write the page for. + * Offset - Offset of the page to write. + * Page - Page which contains the data to write. + */ +{ + NTSTATUS Status; + IO_STATUS_BLOCK IoStatus; + KEVENT Event; + UCHAR MdlBase[sizeof(MDL) + sizeof(PFN_NUMBER)]; + PMDL Mdl = (PMDL)MdlBase; + PFILE_OBJECT FileObject = Segment->FileObject; + LARGE_INTEGER FileOffset; + PFSRTL_COMMON_FCB_HEADER Fcb = FileObject->FsContext; + + FileOffset.QuadPart = Segment->Image.FileOffset + SegOffset; + + /* Check if we are not writing off-limit */ + if (FileOffset.QuadPart >= Fcb->AllocationSize.QuadPart) + { + return STATUS_SUCCESS; + } + + RtlZeroMemory(MdlBase, sizeof(MdlBase)); + MmInitializeMdl(Mdl, NULL, PAGE_SIZE); + MmBuildMdlFromPages(Mdl, &Page); + Mdl->MdlFlags |= MDL_PAGES_LOCKED; + + KeInitializeEvent(&Event, NotificationEvent, FALSE); + Status = IoSynchronousPageWrite(FileObject, Mdl, &FileOffset, &Event, &IoStatus); + if (Status == STATUS_PENDING) + { + KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); + Status = IoStatus.Status; + } + if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) + { + MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl); + } + + return Status; +} + + /* References: [1] Microsoft Corporation, "Microsoft Portable Executable and Common Object @@ -713,6 +765,8 @@ l_ReadHeaderFromFile: nStatus = STATUS_INVALID_IMAGE_FORMAT; + ASSERT(ImageSectionObject->RefCount > 0); + /* convert the executable sections into segments. See also [1], section 4 */ for(i = 0; i < ImageSectionObject->NrSegments - 1; ++ i) { @@ -824,51 +878,146 @@ MmspWaitForFileLock(PFILE_OBJECT File) //return KeWaitForSingleObject(&File->Lock, 0, KernelMode, FALSE, NULL); } + + VOID NTAPI -MmFreeSectionSegments(PFILE_OBJECT FileObject) +MmpFreePageFileSegment(PMM_SECTION_SEGMENT Segment) { - if (FileObject->SectionObjectPointer->ImageSectionObject != NULL) + ULONG Length; + LARGE_INTEGER Offset; + ULONG_PTR Entry; + SWAPENTRY SavedSwapEntry; + PFN_NUMBER Page; + + Page = 0; + + MmLockSectionSegment(Segment); + + Length = PAGE_ROUND_UP(Segment->Length.QuadPart); + for (Offset.QuadPart = 0; Offset.QuadPart < Length; Offset.QuadPart += PAGE_SIZE) { - PMM_IMAGE_SECTION_OBJECT ImageSectionObject; + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); + if (Entry) + { + MmSetPageEntrySectionSegment(Segment, &Offset, 0); + if (IS_SWAP_FROM_SSE(Entry)) + { + MmFreeSwapPage(SWAPENTRY_FROM_SSE(Entry)); + } + else + { + Page = PFN_FROM_SSE(Entry); + SavedSwapEntry = MmGetSavedSwapEntryPage(Page); + if (SavedSwapEntry != 0) + { + MmSetSavedSwapEntryPage(Page, 0); + MmFreeSwapPage(SavedSwapEntry); + } + MmReleasePageMemoryConsumer(MC_USER, Page); + } + } + } + + MmUnlockSectionSegment(Segment); +} + +static +VOID +NTAPI +FreeSegmentPage(PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset) +{ + ULONG_PTR Entry; + PFN_NUMBER Page; + + MmLockSectionSegment(Segment); + + Entry = MmGetPageEntrySectionSegment(Segment, Offset); + + MmUnlockSectionSegment(Segment); + + /* This must be either a valid entry or nothing */ + ASSERT(!IS_SWAP_FROM_SSE(Entry)); + + /* There should be no reference anymore */ + ASSERT(SHARE_COUNT_FROM_SSE(Entry) == 0); + + Page = PFN_FROM_SSE(Entry); + /* If there is a page, this must be because it's still dirty */ + ASSERT(Page != 0); + + /* Write the page */ + if (IS_DIRTY_SSE(Entry)) + MiWritePage(Segment, Offset->QuadPart, Page); + + MmReleasePageMemoryConsumer(MC_USER, Page); +} + +VOID +NTAPI +MmDereferenceSegment(PMM_SECTION_SEGMENT Segment) +{ + KIRQL OldIrql; + + /* Lock the PFN lock because we mess around with SectionObjectPointers */ + OldIrql = MiAcquirePfnLock(); + + if (InterlockedDecrementUL(Segment->ReferenceCount) > 0) + { + /* Nothing to do yet */ + MiReleasePfnLock(OldIrql); + return; + } + + *Segment->Flags |= MM_SEGMENT_INDELETE; + + MiReleasePfnLock(OldIrql); + + /* Flush the segment */ + if (*Segment->Flags & MM_DATAFILE_SEGMENT) + { + /* Free the page table. This will flush any remaining dirty data */ + MmFreePageTablesSectionSegment(Segment, FreeSegmentPage); + + OldIrql = MiAcquirePfnLock(); + /* Delete the pointer on the file */ + ASSERT(Segment->FileObject->SectionObjectPointer->DataSectionObject == Segment); + Segment->FileObject->SectionObjectPointer->DataSectionObject = NULL; + MiReleasePfnLock(OldIrql); + ObDereferenceObject(Segment->FileObject); + + ExFreePoolWithTag(Segment, TAG_MM_SECTION_SEGMENT); + } + else + { + /* Most grotesque thing ever */ + PMM_IMAGE_SECTION_OBJECT ImageSectionObject = CONTAINING_RECORD(Segment->ReferenceCount, MM_IMAGE_SECTION_OBJECT, RefCount); PMM_SECTION_SEGMENT SectionSegments; ULONG NrSegments; ULONG i; - ImageSectionObject = (PMM_IMAGE_SECTION_OBJECT)FileObject->SectionObjectPointer->ImageSectionObject; + OldIrql = MiAcquirePfnLock(); + /* Delete the pointer on the file */ + ASSERT(ImageSectionObject->FileObject->SectionObjectPointer->ImageSectionObject == ImageSectionObject); + ImageSectionObject->FileObject->SectionObjectPointer->ImageSectionObject = NULL; + MiReleasePfnLock(OldIrql); + + ObDereferenceObject(ImageSectionObject->FileObject); + NrSegments = ImageSectionObject->NrSegments; SectionSegments = ImageSectionObject->Segments; for (i = 0; i < NrSegments; i++) { - if (SectionSegments[i].ReferenceCount != 0) + if (SectionSegments[i].Image.Characteristics & IMAGE_SCN_MEM_SHARED) { - DPRINT1("Image segment %lu still referenced (was %lu)\n", i, - SectionSegments[i].ReferenceCount); - KeBugCheck(MEMORY_MANAGEMENT); + MmpFreePageFileSegment(&SectionSegments[i]); } + MmFreePageTablesSectionSegment(&SectionSegments[i], NULL); } - ObDereferenceObject(ImageSectionObject->FileObject); - ExFreePool(ImageSectionObject->Segments); - ExFreePool(ImageSectionObject); - FileObject->SectionObjectPointer->ImageSectionObject = NULL; - } - if (FileObject->SectionObjectPointer->DataSectionObject != NULL) - { - PMM_SECTION_SEGMENT Segment; - Segment = (PMM_SECTION_SEGMENT)FileObject->SectionObjectPointer-> - DataSectionObject; - - if (Segment->ReferenceCount != 0) - { - DPRINT1("Data segment still referenced\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - ObDereferenceObject(Segment->FileObject); - MmFreePageTablesSectionSegment(Segment, NULL); - ExFreePool(Segment); - FileObject->SectionObjectPointer->DataSectionObject = NULL; + ExFreePoolWithTag(ImageSectionObject->Segments, TAG_MM_SECTION_SEGMENT); + ExFreePoolWithTag(ImageSectionObject, TAG_MM_SECTION_SEGMENT); } } @@ -908,7 +1057,9 @@ MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, ULONG_PTR *InEntry) { ULONG_PTR Entry = InEntry ? *InEntry : MmGetPageEntrySectionSegment(Segment, Offset); - BOOLEAN IsDirectMapped = FALSE; + PFN_NUMBER Page = PFN_FROM_SSE(Entry); + ULONG_PTR NewEntry = 0; + SWAPENTRY SwapEntry; if (Entry == 0) { @@ -924,142 +1075,66 @@ MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, { KeBugCheck(MEMORY_MANAGEMENT); } + Dirty = Dirty || IS_DIRTY_SSE(Entry); Entry = MAKE_SSE(PAGE_FROM_SSE(Entry), SHARE_COUNT_FROM_SSE(Entry) - 1); - /* - * If we reducing the share count of this entry to zero then set the entry - * to zero and tell the cache the page is no longer mapped. - */ - if (SHARE_COUNT_FROM_SSE(Entry) == 0) - { - PFILE_OBJECT FileObject; - SWAPENTRY SavedSwapEntry; - PFN_NUMBER Page; -#ifndef NEWCC - PROS_SHARED_CACHE_MAP SharedCacheMap; - BOOLEAN IsImageSection; - LARGE_INTEGER FileOffset; + if (Dirty) Entry = DIRTY_SSE(Entry); - FileOffset.QuadPart = Offset->QuadPart + Segment->Image.FileOffset; - IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; -#endif + if (SHARE_COUNT_FROM_SSE(Entry) > 0) + { + /* Update the page mapping in the segment and we're done */ + if (InEntry) + *InEntry = Entry; + else + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; + } - Page = PFN_FROM_SSE(Entry); - FileObject = Segment->FileObject; - if (FileObject != NULL && - !(Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { + if (IS_DIRTY_SSE(Entry) && !(Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) + { + ASSERT(!Segment->WriteCopy); + ASSERT(MmGetSavedSwapEntryPage(Page) == 0); -#ifndef NEWCC - if ((FileOffset.QuadPart % PAGE_SIZE) == 0 && - (Offset->QuadPart + PAGE_SIZE <= Segment->RawLength.QuadPart || !IsImageSection)) - { - NTSTATUS Status; - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - IsDirectMapped = TRUE; -#ifndef NEWCC - Status = CcRosUnmapVacb(SharedCacheMap, FileOffset.QuadPart, Dirty); -#else - Status = STATUS_SUCCESS; -#endif - if (!NT_SUCCESS(Status)) - { - DPRINT1("CcRosUnmapVacb failed, status = %x\n", Status); - KeBugCheck(MEMORY_MANAGEMENT); - } - } -#endif - } + /* The entry must be written back to the disk, so let this in the segment, the page-out thread will take care of this */ + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; + } - SavedSwapEntry = MmGetSavedSwapEntryPage(Page); - if (SavedSwapEntry == 0) - { - if (!PageOut && (Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { - /* - * FIXME: - * Try to page out this page and set the swap entry - * within the section segment. There exist no rmap entry - * for this page. The pager thread can't page out a - * page without a rmap entry. - */ - MmSetPageEntrySectionSegment(Segment, Offset, Entry); - if (InEntry) *InEntry = Entry; - MiSetPageEvent(NULL, NULL); - } - else - { - MmSetPageEntrySectionSegment(Segment, Offset, 0); - if (InEntry) *InEntry = 0; - MiSetPageEvent(NULL, NULL); - if (!IsDirectMapped) - { - MmReleasePageMemoryConsumer(MC_USER, Page); - } - } - } - else + SwapEntry = MmGetSavedSwapEntryPage(Page); + if (Dirty && !SwapEntry) + { + SwapEntry = MmAllocSwapPage(); + if (!SwapEntry) { - if (Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - if (!PageOut) - { - if (Dirty) - { - /* - * FIXME: - * We hold all locks. Nobody can do something with the current - * process and the current segment (also not within an other process). - */ - NTSTATUS Status; - Status = MmWriteToSwapPage(SavedSwapEntry, Page); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MM: Failed to write to swap page (Status was 0x%.8X)\n", Status); - KeBugCheck(MEMORY_MANAGEMENT); - } - } - MmSetPageEntrySectionSegment(Segment, Offset, MAKE_SWAP_SSE(SavedSwapEntry)); - if (InEntry) *InEntry = MAKE_SWAP_SSE(SavedSwapEntry); - MmSetSavedSwapEntryPage(Page, 0); - MiSetPageEvent(NULL, NULL); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - } - else - { - DPRINT1("Found a swapentry for a non private page in an image or data file sgment\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } + /* We can't have a swap entry for this page. Let the segment keep it */ + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; } } - else + + if (Dirty) { - if (InEntry) - *InEntry = Entry; - else + NTSTATUS Status = MmWriteToSwapPage(SwapEntry, Page); + if (!NT_SUCCESS(Status)) + { + /* We failed. Clean up */ + MmSetSavedSwapEntryPage(Page, 0); + MmFreeSwapPage(SwapEntry); MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; + } } - return(SHARE_COUNT_FROM_SSE(Entry) > 0); -} -BOOLEAN MiIsPageFromCache(PMEMORY_AREA MemoryArea, - LONGLONG SegOffset) -{ -#ifndef NEWCC - if (!(MemoryArea->SectionData.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) + if (SwapEntry) { - PROS_SHARED_CACHE_MAP SharedCacheMap; - PROS_VACB Vacb; - SharedCacheMap = MemoryArea->SectionData.Segment->FileObject->SectionObjectPointer->SharedCacheMap; - Vacb = CcRosLookupVacb(SharedCacheMap, SegOffset + MemoryArea->SectionData.Segment->Image.FileOffset); - if (Vacb) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, Vacb->Valid, FALSE, TRUE); - return TRUE; - } + NewEntry = MAKE_SWAP_SSE(SwapEntry); + MmSetSavedSwapEntryPage(Page, 0); } -#endif - return FALSE; + + /* We can let this go */ + MmSetPageEntrySectionSegment(Segment, Offset, NewEntry); + MmReleasePageMemoryConsumer(MC_USER, Page); + MiSetPageEvent(NULL, NULL); + return TRUE; } NTSTATUS @@ -1097,175 +1172,46 @@ MiReadPage(PMEMORY_AREA MemoryArea, * Page - Variable that receives a page contains the read data. */ { - LONGLONG BaseOffset; - LONGLONG FileOffset; - PVOID BaseAddress; - BOOLEAN UptoDate; - PROS_VACB Vacb; - PFILE_OBJECT FileObject; NTSTATUS Status; - LONGLONG RawLength; - PROS_SHARED_CACHE_MAP SharedCacheMap; - BOOLEAN IsImageSection; - LONGLONG Length; - - FileObject = MemoryArea->SectionData.Segment->FileObject; - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - RawLength = MemoryArea->SectionData.Segment->RawLength.QuadPart; - FileOffset = SegOffset + MemoryArea->SectionData.Segment->Image.FileOffset; - IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; - - ASSERT(SharedCacheMap); + IO_STATUS_BLOCK IoStatus; + KEVENT Event; + UCHAR MdlBase[sizeof(MDL) + sizeof(PFN_NUMBER)]; + PMDL Mdl = (PMDL)MdlBase; + PFILE_OBJECT FileObject = MemoryArea->SectionData.Segment->FileObject; + LARGE_INTEGER FileOffset; + PFSRTL_COMMON_FCB_HEADER Fcb = FileObject->FsContext; - DPRINT("%S %I64x\n", FileObject->FileName.Buffer, FileOffset); + FileOffset.QuadPart = MemoryArea->SectionData.Segment->Image.FileOffset + SegOffset; - /* - * If the file system is letting us go directly to the cache and the - * memory area was mapped at an offset in the file which is page aligned - * then get the related VACB. - */ - if (((FileOffset % PAGE_SIZE) == 0) && - ((SegOffset + PAGE_SIZE <= RawLength) || !IsImageSection) && - !(MemoryArea->SectionData.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { + Status = MmRequestPageMemoryConsumer(MC_USER, FALSE, Page); + if (!NT_SUCCESS(Status)) + return Status; - /* - * Get the related VACB; we use a lower level interface than - * filesystems do because it is safe for us to use an offset with an - * alignment less than the file system block size. - */ - Status = CcRosGetVacb(SharedCacheMap, - FileOffset, - &BaseOffset, - &BaseAddress, - &UptoDate, - &Vacb); - if (!NT_SUCCESS(Status)) - { - return(Status); - } - if (!UptoDate) - { - /* - * If the VACB isn't up to date then call the file - * system to read in the data. - */ - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - return Status; - } - } + /* Check if we are beyond the file */ + if (FileOffset.QuadPart > Fcb->FileSize.QuadPart) + return STATUS_SUCCESS; - /* Probe the page, since it's PDE might not be synced */ - (void)*((volatile char*)BaseAddress + FileOffset - BaseOffset); + RtlZeroMemory(MdlBase, sizeof(MdlBase)); + MmInitializeMdl(Mdl, NULL, PAGE_SIZE); + MmBuildMdlFromPages(Mdl, Page); + Mdl->MdlFlags |= MDL_PAGES_LOCKED | MDL_IO_PAGE_READ; - /* - * Retrieve the page from the view that we actually want. - */ - (*Page) = MmGetPhysicalAddress((char*)BaseAddress + - FileOffset - BaseOffset).LowPart >> PAGE_SHIFT; + KeInitializeEvent(&Event, NotificationEvent, FALSE); - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, TRUE); + Status = IoPageRead(FileObject, Mdl, &FileOffset, &Event, &IoStatus); + if (Status == STATUS_PENDING) + { + KeWaitForSingleObject(&Event, WrPageIn, KernelMode, FALSE, NULL); + Status = IoStatus.Status; } - else + if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) { - PEPROCESS Process; - KIRQL Irql; - PVOID PageAddr; - LONGLONG VacbOffset; + MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl); + } - /* - * Allocate a page, this is rather complicated by the possibility - * we might have to move other things out of memory - */ - MI_SET_USAGE(MI_USAGE_SECTION); - MI_SET_PROCESS2(PsGetCurrentProcess()->ImageFileName); - Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, Page); - if (!NT_SUCCESS(Status)) - { - return(Status); - } - Status = CcRosGetVacb(SharedCacheMap, - FileOffset, - &BaseOffset, - &BaseAddress, - &UptoDate, - &Vacb); - if (!NT_SUCCESS(Status)) - { - return(Status); - } - if (!UptoDate) - { - /* - * If the VACB isn't up to date then call the file - * system to read in the data. - */ - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - return Status; - } - } + return Status; +} - Process = PsGetCurrentProcess(); - PageAddr = MiMapPageInHyperSpace(Process, *Page, &Irql); - VacbOffset = BaseOffset + VACB_MAPPING_GRANULARITY - FileOffset; - Length = RawLength - SegOffset; - if (Length <= VacbOffset && Length <= PAGE_SIZE) - { - memcpy(PageAddr, (char*)BaseAddress + FileOffset - BaseOffset, Length); - } - else if (VacbOffset >= PAGE_SIZE) - { - memcpy(PageAddr, (char*)BaseAddress + FileOffset - BaseOffset, PAGE_SIZE); - } - else - { - memcpy(PageAddr, (char*)BaseAddress + FileOffset - BaseOffset, VacbOffset); - MiUnmapPageInHyperSpace(Process, PageAddr, Irql); - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); - Status = CcRosGetVacb(SharedCacheMap, - FileOffset + VacbOffset, - &BaseOffset, - &BaseAddress, - &UptoDate, - &Vacb); - if (!NT_SUCCESS(Status)) - { - return(Status); - } - if (!UptoDate) - { - /* - * If the VACB isn't up to date then call the file - * system to read in the data. - */ - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - return Status; - } - } - PageAddr = MiMapPageInHyperSpace(Process, *Page, &Irql); - if (Length < PAGE_SIZE) - { - memcpy((char*)PageAddr + VacbOffset, BaseAddress, Length - VacbOffset); - } - else - { - memcpy((char*)PageAddr + VacbOffset, BaseAddress, PAGE_SIZE - VacbOffset); - } - } - MiUnmapPageInHyperSpace(Process, PageAddr, Irql); - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); - } - return(STATUS_SUCCESS); -} #else NTSTATUS NTAPI @@ -1571,7 +1517,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, /* * Add the page to the process's working set */ - MmInsertRmap(Page, Process, Address); + if (Process) MmInsertRmap(Page, Process, Address); /* * Finish the operation */ @@ -1684,7 +1630,8 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, KeBugCheck(MEMORY_MANAGEMENT); } ASSERT(MmIsPagePresent(Process, PAddress)); - MmInsertRmap(Page, Process, Address); + if (Process) + MmInsertRmap(Page, Process, Address); /* Set this section offset has being backed by our new page. */ Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); @@ -1765,7 +1712,8 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, DPRINT1("Unable to create virtual mapping\n"); KeBugCheck(MEMORY_MANAGEMENT); } - MmInsertRmap(Page, Process, Address); + if (Process) + MmInsertRmap(Page, Process, Address); /* * Mark the offset within the section as having valid, in-memory @@ -1794,7 +1742,9 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, DPRINT1("Unable to create virtual mapping\n"); KeBugCheck(MEMORY_MANAGEMENT); } - MmInsertRmap(Page, Process, Address); + + if (Process) + MmInsertRmap(Page, Process, Address); /* Take a reference on it */ MmSharePageEntrySectionSegment(Segment, &Offset); @@ -1878,609 +1828,58 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, PFN_FROM_SSE(Entry) != OldPage) { MmUnlockSectionSegment(Segment); - /* This is a private page. We must only change the page protection. */ - MmSetPageProtect(Process, PAddress, Region->Protect); - return(STATUS_SUCCESS); - } - - /* - * Allocate a page - */ - MI_SET_USAGE(MI_USAGE_SECTION); - if (Process) MI_SET_PROCESS2(Process->ImageFileName); - if (!Process) MI_SET_PROCESS2("Kernel Section"); - Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &NewPage); - if (!NT_SUCCESS(Status)) - { - KeBugCheck(MEMORY_MANAGEMENT); - } - - /* - * Copy the old page - */ - NT_VERIFY(NT_SUCCESS(MiCopyFromUserPage(NewPage, PAddress))); - - /* - * Unshare the old page. - */ - DPRINT("Swapping page (Old %x New %x)\n", OldPage, NewPage); - MmDeleteVirtualMapping(Process, PAddress, NULL, NULL); - MmDeleteRmap(OldPage, Process, PAddress); - MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, FALSE, FALSE, NULL); - MmUnlockSectionSegment(Segment); - - /* - * Set the PTE to point to the new page - */ - Status = MmCreateVirtualMapping(Process, - PAddress, - Region->Protect, - &NewPage, - 1); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MmCreateVirtualMapping failed, unable to create virtual mapping, not out of memory\n"); - KeBugCheck(MEMORY_MANAGEMENT); - return(Status); - } - MmInsertRmap(NewPage, Process, PAddress); - - MiSetPageEvent(Process, Address); - DPRINT("Address 0x%p\n", Address); - return(STATUS_SUCCESS); -} - -VOID -MmPageOutDeleteMapping(PVOID Context, PEPROCESS Process, PVOID Address) -{ - MM_SECTION_PAGEOUT_CONTEXT* PageOutContext; - BOOLEAN WasDirty; - PFN_NUMBER Page = 0; - - PageOutContext = (MM_SECTION_PAGEOUT_CONTEXT*)Context; - if (Process) - { - MmLockAddressSpace(&Process->Vm); - } - - MmDeleteVirtualMapping(Process, - Address, - &WasDirty, - &Page); - if (WasDirty) - { - PageOutContext->WasDirty = TRUE; - } - if (!PageOutContext->Private) - { - MmLockSectionSegment(PageOutContext->Segment); - MmUnsharePageEntrySectionSegment(PageOutContext->MemoryArea, - PageOutContext->Segment, - &PageOutContext->Offset, - PageOutContext->WasDirty, - TRUE, - &PageOutContext->SectionEntry); - MmUnlockSectionSegment(PageOutContext->Segment); - } - if (Process) - { - MmUnlockAddressSpace(&Process->Vm); - } - - if (PageOutContext->Private) - { - MmReleasePageMemoryConsumer(MC_USER, Page); - } -} - -NTSTATUS -NTAPI -MmPageOutSectionView(PMMSUPPORT AddressSpace, - MEMORY_AREA* MemoryArea, - PVOID Address, ULONG_PTR Entry) -{ - PFN_NUMBER Page; - MM_SECTION_PAGEOUT_CONTEXT Context; - SWAPENTRY SwapEntry; - NTSTATUS Status; -#ifndef NEWCC - ULONGLONG FileOffset; - PFILE_OBJECT FileObject; - PROS_SHARED_CACHE_MAP SharedCacheMap = NULL; - BOOLEAN IsImageSection; -#endif - BOOLEAN DirectMapped; - PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); - KIRQL OldIrql; - - Address = (PVOID)PAGE_ROUND_DOWN(Address); - - /* - * Get the segment and section. - */ - Context.Segment = MemoryArea->SectionData.Segment; - Context.MemoryArea = MemoryArea; - Context.SectionEntry = Entry; - Context.CallingProcess = Process; - - Context.Offset.QuadPart = (ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; - - DirectMapped = FALSE; - - MmLockSectionSegment(Context.Segment); - -#ifndef NEWCC - FileOffset = Context.Offset.QuadPart + Context.Segment->Image.FileOffset; - IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; - FileObject = Context.Segment->FileObject; - - if (FileObject != NULL && - !(Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - - /* - * If the file system is letting us go directly to the cache and the - * memory area was mapped at an offset in the file which is page aligned - * then note this is a direct mapped page. - */ - if ((FileOffset % PAGE_SIZE) == 0 && - (Context.Offset.QuadPart + PAGE_SIZE <= Context.Segment->RawLength.QuadPart || !IsImageSection)) - { - DirectMapped = TRUE; - } - } -#endif - - /* - * Get the section segment entry and the physical address. - */ - if (!MmIsPagePresent(Process, Address)) - { - DPRINT1("Trying to page out not-present page at (%p,0x%p).\n", - Process ? Process->UniqueProcessId : 0, Address); - KeBugCheck(MEMORY_MANAGEMENT); - } - Page = MmGetPfnForProcess(Process, Address); - SwapEntry = MmGetSavedSwapEntryPage(Page); - - /* - * Check the reference count to ensure this page can be paged out - */ - if (MmGetReferenceCountPage(Page) != 1) - { - DPRINT("Cannot page out locked section page: 0x%lu (RefCount: %lu)\n", - Page, MmGetReferenceCountPage(Page)); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - return STATUS_UNSUCCESSFUL; - } - - /* - * Prepare the context structure for the rmap delete call. - */ - MmUnlockSectionSegment(Context.Segment); - Context.WasDirty = FALSE; - if (IS_SWAP_FROM_SSE(Entry) || PFN_FROM_SSE(Entry) != Page) - { - Context.Private = TRUE; - } - else - { - Context.Private = FALSE; - } - - /* - * Take an additional reference to the page or the VACB. - */ - if (DirectMapped && !Context.Private) - { - if(!MiIsPageFromCache(MemoryArea, Context.Offset.QuadPart)) - { - DPRINT1("Direct mapped non private page is not associated with the cache.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - } - else - { - OldIrql = MiAcquirePfnLock(); - MmReferencePage(Page); - MiReleasePfnLock(OldIrql); - } - - MmDeleteAllRmaps(Page, (PVOID)&Context, MmPageOutDeleteMapping); - - /* Since we passed in a surrogate, we'll get back the page entry - * state in our context. This is intended to make intermediate - * decrements of share count not release the wait entry. - */ - Entry = Context.SectionEntry; - - /* - * If this wasn't a private page then we should have reduced the entry to - * zero by deleting all the rmaps. - */ - if (!Context.Private && Entry != 0) - { - if (!(Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { - KeBugCheckEx(MEMORY_MANAGEMENT, Entry, (ULONG_PTR)Process, (ULONG_PTR)Address, 0); - } - } - - /* - * If the page wasn't dirty then we can just free it as for a readonly page. - * Since we unmapped all the mappings above we know it will not suddenly - * become dirty. - * If the page is from a pagefile section and has no swap entry, - * we can't free the page at this point. - */ - SwapEntry = MmGetSavedSwapEntryPage(Page); - if (Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - if (Context.Private) - { - DPRINT1("Found a %s private page (address %p) in a shared section segment.\n", - Context.WasDirty ? "dirty" : "clean", Address); - KeBugCheckEx(MEMORY_MANAGEMENT, Page, (ULONG_PTR)Process, (ULONG_PTR)Address, 0); - } - if (!Context.WasDirty || SwapEntry != 0) - { - MmSetSavedSwapEntryPage(Page, 0); - if (SwapEntry != 0) - { - MmLockSectionSegment(Context.Segment); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry)); - MmUnlockSectionSegment(Context.Segment); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - } - else if (!Context.Private && DirectMapped) - { - if (SwapEntry != 0) - { - DPRINT1("Found a swapentry for a non private and direct mapped page (address %p)\n", - Address); - KeBugCheckEx(MEMORY_MANAGEMENT, STATUS_UNSUCCESSFUL, SwapEntry, (ULONG_PTR)Process, (ULONG_PTR)Address); - } -#ifndef NEWCC - Status = CcRosUnmapVacb(SharedCacheMap, FileOffset, FALSE); -#else - Status = STATUS_SUCCESS; -#endif -#ifndef NEWCC - if (!NT_SUCCESS(Status)) - { - DPRINT1("CcRosUnmapVacb failed, status = %x\n", Status); - KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)SharedCacheMap, (ULONG_PTR)FileOffset, (ULONG_PTR)Address); - } -#endif - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - else if (!Context.WasDirty && !DirectMapped && !Context.Private) - { - if (SwapEntry != 0) - { - DPRINT1("Found a swap entry for a non dirty, non private and not direct mapped page (address %p)\n", - Address); - KeBugCheckEx(MEMORY_MANAGEMENT, SwapEntry, Page, (ULONG_PTR)Process, (ULONG_PTR)Address); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - else if (!Context.WasDirty && Context.Private && SwapEntry != 0) - { - DPRINT("Not dirty and private and not swapped (%p:%p)\n", Process, Address); - MmSetSavedSwapEntryPage(Page, 0); - MmLockAddressSpace(AddressSpace); - Status = MmCreatePageFileMapping(Process, - Address, - SwapEntry); - MmUnlockAddressSpace(AddressSpace); - if (!NT_SUCCESS(Status)) - { - DPRINT1("Status %x Swapping out %p:%p\n", Status, Process, Address); - KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)Process, (ULONG_PTR)Address, SwapEntry); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - - /* - * If necessary, allocate an entry in the paging file for this page - */ - if (SwapEntry == 0) - { - SwapEntry = MmAllocSwapPage(); - if (SwapEntry == 0) - { - MmShowOutOfSpaceMessagePagingFile(); - MmLockAddressSpace(AddressSpace); - /* - * For private pages restore the old mappings. - */ - if (Context.Private) - { - Status = MmCreateVirtualMapping(Process, - Address, - MmProtectToValue[MemoryArea->VadNode.u.VadFlags.Protection], - &Page, - 1); - MmSetDirtyPage(Process, Address); - MmInsertRmap(Page, - Process, - Address); - } - else - { - ULONG_PTR OldEntry; - - MmLockSectionSegment(Context.Segment); - - /* - * For non-private pages if the page wasn't direct mapped then - * set it back into the section segment entry so we don't loose - * our copy. Otherwise it will be handled by the cache manager. - */ - Status = MmCreateVirtualMapping(Process, - Address, - MmProtectToValue[MemoryArea->VadNode.u.VadFlags.Protection], - &Page, - 1); - MmSetDirtyPage(Process, Address); - MmInsertRmap(Page, - Process, - Address); - // If we got here, the previous entry should have been a wait - Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); - OldEntry = MmGetPageEntrySectionSegment(Context.Segment, &Context.Offset); - ASSERT(OldEntry == 0 || OldEntry == MAKE_SWAP_SSE(MM_WAIT_ENTRY)); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - } - MmUnlockAddressSpace(AddressSpace); - MiSetPageEvent(NULL, NULL); - return(STATUS_PAGEFILE_QUOTA); - } - } - - /* - * Write the page to the pagefile - */ - Status = MmWriteToSwapPage(SwapEntry, Page); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MM: Failed to write to swap page (Status was 0x%.8X)\n", - Status); - /* - * As above: undo our actions. - * FIXME: Also free the swap page. - */ - MmLockAddressSpace(AddressSpace); - if (Context.Private) - { - Status = MmCreateVirtualMapping(Process, - Address, - MmProtectToValue[MemoryArea->VadNode.u.VadFlags.Protection], - &Page, - 1); - MmSetDirtyPage(Process, Address); - MmInsertRmap(Page, - Process, - Address); - } - else - { - MmLockSectionSegment(Context.Segment); - Status = MmCreateVirtualMapping(Process, - Address, - MmProtectToValue[MemoryArea->VadNode.u.VadFlags.Protection], - &Page, - 1); - MmSetDirtyPage(Process, Address); - MmInsertRmap(Page, - Process, - Address); - Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - } - MmUnlockAddressSpace(AddressSpace); - MiSetPageEvent(NULL, NULL); - return(STATUS_UNSUCCESSFUL); - } - - /* - * Otherwise we have succeeded. - */ - DPRINT("MM: Wrote section page 0x%.8X to swap!\n", Page << PAGE_SHIFT); - MmSetSavedSwapEntryPage(Page, 0); - if (Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - MmLockSectionSegment(Context.Segment); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry)); - MmUnlockSectionSegment(Context.Segment); - } - else - { - MmReleasePageMemoryConsumer(MC_USER, Page); - } - - if (Context.Private) - { - MmLockAddressSpace(AddressSpace); - MmLockSectionSegment(Context.Segment); - Status = MmCreatePageFileMapping(Process, - Address, - SwapEntry); - /* We had placed a wait entry upon entry ... replace it before leaving */ - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - MmUnlockAddressSpace(AddressSpace); - if (!NT_SUCCESS(Status)) - { - DPRINT1("Status %x Creating page file mapping for %p:%p\n", Status, Process, Address); - KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)Process, (ULONG_PTR)Address, SwapEntry); - } - } - else - { - MmLockAddressSpace(AddressSpace); - MmLockSectionSegment(Context.Segment); - Entry = MAKE_SWAP_SSE(SwapEntry); - /* We had placed a wait entry upon entry ... replace it before leaving */ - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - MmUnlockAddressSpace(AddressSpace); - } - - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); -} - -NTSTATUS -NTAPI -MmWritePageSectionView(PMMSUPPORT AddressSpace, - PMEMORY_AREA MemoryArea, - PVOID Address, - ULONG PageEntry) -{ - LARGE_INTEGER Offset; - PMM_SECTION_SEGMENT Segment; - PFN_NUMBER Page; - SWAPENTRY SwapEntry; - ULONG_PTR Entry; - BOOLEAN Private; - NTSTATUS Status; - PFILE_OBJECT FileObject; -#ifndef NEWCC - PROS_SHARED_CACHE_MAP SharedCacheMap = NULL; -#endif - BOOLEAN DirectMapped; - BOOLEAN IsImageSection; - PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); - - Address = (PVOID)PAGE_ROUND_DOWN(Address); - - Offset.QuadPart = (ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; - - /* - * Get the segment and section. - */ - Segment = MemoryArea->SectionData.Segment; - IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; - - FileObject = Segment->FileObject; - DirectMapped = FALSE; - if (FileObject != NULL && - !(Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { -#ifndef NEWCC - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; -#endif - - /* - * If the file system is letting us go directly to the cache and the - * memory area was mapped at an offset in the file which is page aligned - * then note this is a direct mapped page. - */ - if (((Offset.QuadPart + Segment->Image.FileOffset) % PAGE_SIZE) == 0 && - (Offset.QuadPart + PAGE_SIZE <= Segment->RawLength.QuadPart || !IsImageSection)) - { - DirectMapped = TRUE; - } - } - - /* - * Get the section segment entry and the physical address. - */ - Entry = MmGetPageEntrySectionSegment(Segment, &Offset); - if (!MmIsPagePresent(Process, Address)) - { - DPRINT1("Trying to page out not-present page at (%p,0x%p).\n", - Process ? Process->UniqueProcessId : 0, Address); - KeBugCheck(MEMORY_MANAGEMENT); + /* This is a private page. We must only change the page protection. */ + MmSetPageProtect(Process, PAddress, Region->Protect); + return(STATUS_SUCCESS); } - Page = MmGetPfnForProcess(Process, Address); - SwapEntry = MmGetSavedSwapEntryPage(Page); /* - * Check for a private (COWed) page. + * Allocate a page */ - if (IS_SWAP_FROM_SSE(Entry) || PFN_FROM_SSE(Entry) != Page) - { - Private = TRUE; - } - else + MI_SET_USAGE(MI_USAGE_SECTION); + if (Process) MI_SET_PROCESS2(Process->ImageFileName); + if (!Process) MI_SET_PROCESS2("Kernel Section"); + Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &NewPage); + if (!NT_SUCCESS(Status)) { - Private = FALSE; + KeBugCheck(MEMORY_MANAGEMENT); } /* - * Speculatively set all mappings of the page to clean. - */ - MmSetCleanAllRmaps(Page); - - /* - * If this page was direct mapped from the cache then the cache manager - * will take care of writing it back to disk. + * Copy the old page */ - if (DirectMapped && !Private) - { - //LARGE_INTEGER SOffset; - ASSERT(SwapEntry == 0); - //SOffset.QuadPart = Offset.QuadPart + Segment->Image.FileOffset; -#ifndef NEWCC - CcRosMarkDirtyFile(SharedCacheMap, Offset.QuadPart); -#endif - MmLockSectionSegment(Segment); - MmSetPageEntrySectionSegment(Segment, &Offset, PageEntry); - MmUnlockSectionSegment(Segment); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } + NT_VERIFY(NT_SUCCESS(MiCopyFromUserPage(NewPage, PAddress))); /* - * If necessary, allocate an entry in the paging file for this page + * Unshare the old page. */ - if (SwapEntry == 0) - { - SwapEntry = MmAllocSwapPage(); - if (SwapEntry == 0) - { - MmSetDirtyAllRmaps(Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_PAGEFILE_QUOTA); - } - MmSetSavedSwapEntryPage(Page, SwapEntry); - } + DPRINT("Swapping page (Old %x New %x)\n", OldPage, NewPage); + MmDeleteVirtualMapping(Process, PAddress, NULL, NULL); + if (Process) + MmDeleteRmap(OldPage, Process, PAddress); + MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, FALSE, FALSE, NULL); + MmUnlockSectionSegment(Segment); /* - * Write the page to the pagefile + * Set the PTE to point to the new page */ - Status = MmWriteToSwapPage(SwapEntry, Page); + Status = MmCreateVirtualMapping(Process, + PAddress, + Region->Protect, + &NewPage, + 1); if (!NT_SUCCESS(Status)) { - DPRINT1("MM: Failed to write to swap page (Status was 0x%.8X)\n", - Status); - MmSetDirtyAllRmaps(Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_UNSUCCESSFUL); + DPRINT1("MmCreateVirtualMapping failed, unable to create virtual mapping, not out of memory\n"); + KeBugCheck(MEMORY_MANAGEMENT); + return(Status); } - /* - * Otherwise we have succeeded. - */ - DPRINT("MM: Wrote section page 0x%.8X to swap!\n", Page << PAGE_SHIFT); - MiSetPageEvent(NULL, NULL); + if (Process) + MmInsertRmap(NewPage, Process, PAddress); + + MiSetPageEvent(Process, Address); + DPRINT("Address 0x%p\n", Address); return(STATUS_SUCCESS); } @@ -2560,48 +1959,6 @@ MmQuerySectionView(PMEMORY_AREA MemoryArea, return(STATUS_SUCCESS); } -VOID -NTAPI -MmpFreePageFileSegment(PMM_SECTION_SEGMENT Segment) -{ - ULONG Length; - LARGE_INTEGER Offset; - ULONG_PTR Entry; - SWAPENTRY SavedSwapEntry; - PFN_NUMBER Page; - - Page = 0; - - MmLockSectionSegment(Segment); - - Length = PAGE_ROUND_UP(Segment->Length.QuadPart); - for (Offset.QuadPart = 0; Offset.QuadPart < Length; Offset.QuadPart += PAGE_SIZE) - { - Entry = MmGetPageEntrySectionSegment(Segment, &Offset); - if (Entry) - { - MmSetPageEntrySectionSegment(Segment, &Offset, 0); - if (IS_SWAP_FROM_SSE(Entry)) - { - MmFreeSwapPage(SWAPENTRY_FROM_SSE(Entry)); - } - else - { - Page = PFN_FROM_SSE(Entry); - SavedSwapEntry = MmGetSavedSwapEntryPage(Page); - if (SavedSwapEntry != 0) - { - MmSetSavedSwapEntryPage(Page, 0); - MmFreeSwapPage(SavedSwapEntry); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - } - } - } - - MmUnlockSectionSegment(Segment); -} - VOID NTAPI MmpDeleteSection(PVOID ObjectBody) { @@ -2617,10 +1974,7 @@ MmpDeleteSection(PVOID ObjectBody) DPRINT("MmpDeleteSection(ObjectBody %p)\n", ObjectBody); if (Section->u.Flags.Image) { - ULONG i; - ULONG NrSegments; - ULONG RefCount; - PMM_SECTION_SEGMENT SectionSegments; + PMM_IMAGE_SECTION_OBJECT ImageSectionObject = (PMM_IMAGE_SECTION_OBJECT)Section->Segment; /* * NOTE: Section->ImageSection can be NULL for short time @@ -2631,25 +1985,9 @@ MmpDeleteSection(PVOID ObjectBody) if (Section->Segment == NULL) return; - SectionSegments = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment)->Segments; - NrSegments = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment)->NrSegments; - - for (i = 0; i < NrSegments; i++) - { - if (SectionSegments[i].Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - MmLockSectionSegment(&SectionSegments[i]); - } - RefCount = InterlockedDecrementUL(&SectionSegments[i].ReferenceCount); - if (SectionSegments[i].Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - MmUnlockSectionSegment(&SectionSegments[i]); - if (RefCount == 0) - { - MmpFreePageFileSegment(&SectionSegments[i]); - } - } - } + /* We just dereference the first segment */ + ASSERT(ImageSectionObject->RefCount > 0); + MmDereferenceSegment(ImageSectionObject->Segments); } #ifdef NEWCC else if (Section->Segment && Section->Segment->Flags & MM_DATAFILE_SEGMENT) @@ -2672,25 +2010,17 @@ MmpDeleteSection(PVOID ObjectBody) #endif else { + PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; + /* * NOTE: Section->Segment can be NULL for short time * during the section creating. */ - if (Section->Segment == NULL) + if (Segment == NULL) return; - (void)InterlockedDecrementUL(&((PMM_SECTION_SEGMENT)Section->Segment)->ReferenceCount); - } - - if (Section->Segment) - { - PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; - if (Segment->FileObject != NULL) - { - #ifndef NEWCC - CcRosDereferenceCache(Segment->FileObject); - #endif - } + Segment->SectionCount--; + MmDereferenceSegment(Segment); } } @@ -2763,13 +2093,17 @@ MmCreatePhysicalMemorySection(VOID) } RtlZeroMemory(Segment, sizeof(MM_SECTION_SEGMENT)); PhysSection->Segment = (PSEGMENT)Segment; - Segment->ReferenceCount = 1; + Segment->RefCount = 1; + + Segment->ReferenceCount = &Segment->RefCount; + Segment->Flags = &Segment->SegFlags; + ExInitializeFastMutex(&Segment->Lock); Segment->Image.FileOffset = 0; Segment->Protection = PAGE_EXECUTE_READWRITE; Segment->RawLength = SectionSize; Segment->Length = SectionSize; - Segment->Flags = 0; + Segment->SegFlags = 0; Segment->WriteCopy = FALSE; Segment->Image.VirtualAddress = 0; Segment->Image.Characteristics = 0; @@ -2823,6 +2157,7 @@ MmInitSectionImplementation(VOID) return(STATUS_SUCCESS); } +static NTSTATUS NTAPI MmCreateDataFileSection(PSECTION *SectionObject, @@ -2831,7 +2166,8 @@ MmCreateDataFileSection(PSECTION *SectionObject, PLARGE_INTEGER UMaximumSize, ULONG SectionPageProtection, ULONG AllocationAttributes, - PFILE_OBJECT FileObject) + PFILE_OBJECT FileObject, + BOOLEAN GotFileHandle) /* * Create a section backed by a data file */ @@ -2840,8 +2176,7 @@ MmCreateDataFileSection(PSECTION *SectionObject, NTSTATUS Status; LARGE_INTEGER MaximumSize; PMM_SECTION_SEGMENT Segment; - FILE_STANDARD_INFORMATION FileInfo; - ULONG Length; + KIRQL OldIrql; /* * Create the section @@ -2869,63 +2204,63 @@ MmCreateDataFileSection(PSECTION *SectionObject, Section->u.Flags.filler = 1; Section->InitialPageProtection = SectionPageProtection; Section->u.Flags.File = 1; + if (AllocationAttributes & SEC_NO_CHANGE) Section->u.Flags.NoChange = 1; - /* - * FIXME: This is propably not entirely correct. We can't look into - * the standard FCB header because it might not be initialized yet - * (as in case of the EXT2FS driver by Manoj Paul Joseph where the - * standard file information is filled on first request). - */ - Status = IoQueryFileInformation(FileObject, - FileStandardInformation, - sizeof(FILE_STANDARD_INFORMATION), - &FileInfo, - &Length); - if (!NT_SUCCESS(Status)) - { - ObDereferenceObject(Section); - ObDereferenceObject(FileObject); - return Status; - } - - /* - * FIXME: Revise this once a locking order for file size changes is - * decided - */ - if ((UMaximumSize != NULL) && (UMaximumSize->QuadPart != 0)) + if (!GotFileHandle) { + ASSERT(UMaximumSize != NULL); + ASSERT(UMaximumSize->QuadPart != 0); MaximumSize = *UMaximumSize; } else { - MaximumSize = FileInfo.EndOfFile; - /* Mapping zero-sized files isn't allowed. */ - if (MaximumSize.QuadPart == 0) + LARGE_INTEGER FileSize; + Status = FsRtlGetFileSize(FileObject, &FileSize); + if (!NT_SUCCESS(Status)) { ObDereferenceObject(Section); ObDereferenceObject(FileObject); - return STATUS_MAPPED_FILE_SIZE_ZERO; + return Status; } - } - if (MaximumSize.QuadPart > FileInfo.EndOfFile.QuadPart) - { - Status = IoSetInformation(FileObject, - FileEndOfFileInformation, - sizeof(LARGE_INTEGER), - &MaximumSize); - if (!NT_SUCCESS(Status)) + /* + * FIXME: Revise this once a locking order for file size changes is + * decided + */ + if ((UMaximumSize != NULL) && (UMaximumSize->QuadPart != 0)) { - ObDereferenceObject(Section); - ObDereferenceObject(FileObject); - return(STATUS_SECTION_NOT_EXTENDED); + MaximumSize = *UMaximumSize; + } + else + { + MaximumSize = FileSize; + /* Mapping zero-sized files isn't allowed. */ + if (MaximumSize.QuadPart == 0) + { + ObDereferenceObject(Section); + ObDereferenceObject(FileObject); + return STATUS_MAPPED_FILE_SIZE_ZERO; + } + } + + if (MaximumSize.QuadPart > FileSize.QuadPart) + { + Status = IoSetInformation(FileObject, + FileEndOfFileInformation, + sizeof(LARGE_INTEGER), + &MaximumSize); + if (!NT_SUCCESS(Status)) + { + ObDereferenceObject(Section); + ObDereferenceObject(FileObject); + return(STATUS_SECTION_NOT_EXTENDED); + } } } - if (FileObject->SectionObjectPointer == NULL || - FileObject->SectionObjectPointer->SharedCacheMap == NULL) + if (FileObject->SectionObjectPointer == NULL) { ObDereferenceObject(Section); ObDereferenceObject(FileObject); @@ -2943,34 +2278,61 @@ MmCreateDataFileSection(PSECTION *SectionObject, return(Status); } + /* Lock the PFN lock while messing with Section Object pointers */ + OldIrql = MiAcquirePfnLock(); + Segment = FileObject->SectionObjectPointer->DataSectionObject; + + while (Segment && (Segment->SegFlags & (MM_SEGMENT_INDELETE | MM_SEGMENT_INCREATE))) + { + LARGE_INTEGER ShortTime = {{-10 * 100 * 1000, -1}}; + + MiReleasePfnLock(OldIrql); + KeDelayExecutionThread(KernelMode, FALSE, &ShortTime); + OldIrql = MiAcquirePfnLock(); + Segment = FileObject->SectionObjectPointer->DataSectionObject; + } + /* * If this file hasn't been mapped as a data file before then allocate a * section segment to describe the data file mapping */ - if (FileObject->SectionObjectPointer->DataSectionObject == NULL) + if (Segment == NULL) { Segment = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_SECTION_SEGMENT), TAG_MM_SECTION_SEGMENT); if (Segment == NULL) { //KeSetEvent((PVOID)&FileObject->Lock, IO_NO_INCREMENT, FALSE); + MiReleasePfnLock(OldIrql); ObDereferenceObject(Section); ObDereferenceObject(FileObject); return(STATUS_NO_MEMORY); } + + /* We are creating it */ + RtlZeroMemory(Segment, sizeof(*Segment)); + Segment->SegFlags = MM_DATAFILE_SEGMENT | MM_SEGMENT_INCREATE; + Segment->RefCount = 1; + + FileObject->SectionObjectPointer->DataSectionObject = Segment; + + /* We're safe to release the lock now */ + MiReleasePfnLock(OldIrql); + Section->Segment = (PSEGMENT)Segment; - Segment->ReferenceCount = 1; + + /* Self-referencing segment */ + Segment->Flags = &Segment->SegFlags; + Segment->ReferenceCount = &Segment->RefCount; + + Segment->SectionCount = 1; + ExInitializeFastMutex(&Segment->Lock); Segment->FileObject = FileObject; - /* - * Set the lock before assigning the segment to the file object - */ - ExAcquireFastMutex(&Segment->Lock); - FileObject->SectionObjectPointer->DataSectionObject = (PVOID)Segment; Segment->Image.FileOffset = 0; Segment->Protection = SectionPageProtection; - Segment->Flags = MM_DATAFILE_SEGMENT; + Segment->Image.Characteristics = 0; Segment->WriteCopy = (SectionPageProtection & (PAGE_WRITECOPY | PAGE_EXECUTE_WRITECOPY)); if (AllocationAttributes & SEC_RESERVE) @@ -2983,20 +2345,21 @@ MmCreateDataFileSection(PSECTION *SectionObject, Segment->Length.QuadPart = PAGE_ROUND_UP(Segment->RawLength.QuadPart); } Segment->Image.VirtualAddress = 0; - Segment->Locked = TRUE; MiInitializeSectionPageTable(Segment); + + /* We're good to use it now */ + OldIrql = MiAcquirePfnLock(); + Segment->SegFlags &= ~MM_SEGMENT_INCREATE; + MiReleasePfnLock(OldIrql); } else { - /* - * If the file is already mapped as a data file then we may need - * to extend it - */ - Segment = - (PMM_SECTION_SEGMENT)FileObject->SectionObjectPointer-> - DataSectionObject; Section->Segment = (PSEGMENT)Segment; - (void)InterlockedIncrementUL(&Segment->ReferenceCount); + Segment->RefCount++; + InterlockedIncrementUL(&Segment->SectionCount); + + MiReleasePfnLock(OldIrql); + MmLockSectionSegment(Segment); if (MaximumSize.QuadPart > Segment->RawLength.QuadPart && @@ -3006,15 +2369,13 @@ MmCreateDataFileSection(PSECTION *SectionObject, Segment->Length.QuadPart = PAGE_ROUND_UP(Segment->RawLength.QuadPart); } - /* We let the segment reference the file object */ + MmUnlockSectionSegment(Segment); + + /* The segment already has a reference to a file object. Don't bother keeping one.*/ ObDereferenceObject(FileObject); - FileObject = Segment->FileObject; } - MmUnlockSectionSegment(Segment); Section->SizeOfSection = MaximumSize; -#ifndef NEWCC - CcRosReferenceCache(FileObject); -#endif + //KeSetEvent((PVOID)&FileObject->Lock, IO_NO_INCREMENT, FALSE); *SectionObject = Section; return(STATUS_SUCCESS); @@ -3541,7 +2902,6 @@ ExeFmtpCreateImageSection(PFILE_OBJECT FileObject, */ for (i = 0; i < RTL_NUMBER_OF(ExeFmtpLoaders); ++ i) { - RtlZeroMemory(ImageSectionObject, sizeof(*ImageSectionObject)); Flags = 0; Status = ExeFmtpLoaders[i](FileHeader, @@ -3580,6 +2940,7 @@ ExeFmtpCreateImageSection(PFILE_OBJECT FileObject, return Status; ASSERT(ImageSectionObject->Segments != NULL); + ASSERT(ImageSectionObject->RefCount > 0); /* * Some defaults @@ -3640,11 +3001,14 @@ ExeFmtpCreateImageSection(PFILE_OBJECT FileObject, for ( i = 0; i < ImageSectionObject->NrSegments; ++ i ) { ExInitializeFastMutex(&ImageSectionObject->Segments[i].Lock); - ImageSectionObject->Segments[i].ReferenceCount = 1; + ImageSectionObject->Segments[i].ReferenceCount = &ImageSectionObject->RefCount; + ImageSectionObject->Segments[i].Flags = &ImageSectionObject->SegFlags; MiInitializeSectionPageTable(&ImageSectionObject->Segments[i]); ImageSectionObject->Segments[i].FileObject = FileObject; } + ASSERT(ImageSectionObject->RefCount > 0); + ImageSectionObject->FileObject = FileObject; ASSERT(NT_SUCCESS(Status)); @@ -3662,20 +3026,18 @@ MmCreateImageSection(PSECTION *SectionObject, { PSECTION Section; NTSTATUS Status; - PMM_SECTION_SEGMENT SectionSegments; PMM_IMAGE_SECTION_OBJECT ImageSectionObject; - ULONG i; + KIRQL OldIrql; + if (FileObject == NULL) return STATUS_INVALID_FILE_FOR_SECTION; -#ifndef NEWCC - if (!CcIsFileCached(FileObject)) + if (FileObject->SectionObjectPointer == NULL) { DPRINT1("Denying section creation due to missing cache initialization\n"); return STATUS_INVALID_FILE_FOR_SECTION; } -#endif /* * Create the section @@ -3709,13 +3071,31 @@ MmCreateImageSection(PSECTION *SectionObject, if (AllocationAttributes & SEC_NO_CHANGE) Section->u.Flags.NoChange = 1; - if (FileObject->SectionObjectPointer->ImageSectionObject == NULL) + OldIrql = MiAcquirePfnLock(); + + /* Wait for it to be properly created or deleted */ + ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; + while(ImageSectionObject && (ImageSectionObject->SegFlags & (MM_SEGMENT_INDELETE | MM_SEGMENT_INCREATE))) + { + LARGE_INTEGER ShortTime; + + MiReleasePfnLock(OldIrql); + + ShortTime.QuadPart = - 10 * 100 * 1000; + KeDelayExecutionThread(KernelMode, FALSE, &ShortTime); + + OldIrql = MiAcquirePfnLock(); + ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; + } + + if (ImageSectionObject == NULL) { NTSTATUS StatusExeFmt; - ImageSectionObject = ExAllocatePoolWithTag(PagedPool, sizeof(MM_IMAGE_SECTION_OBJECT), TAG_MM_SECTION_SEGMENT); + ImageSectionObject = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_IMAGE_SECTION_OBJECT), TAG_MM_SECTION_SEGMENT); if (ImageSectionObject == NULL) { + MiReleasePfnLock(OldIrql); ObDereferenceObject(FileObject); ObDereferenceObject(Section); return(STATUS_NO_MEMORY); @@ -3723,10 +3103,21 @@ MmCreateImageSection(PSECTION *SectionObject, RtlZeroMemory(ImageSectionObject, sizeof(MM_IMAGE_SECTION_OBJECT)); + ImageSectionObject->SegFlags = MM_SEGMENT_INCREATE; + ImageSectionObject->RefCount = 1; + FileObject->SectionObjectPointer->ImageSectionObject = ImageSectionObject; + + MiReleasePfnLock(OldIrql); + StatusExeFmt = ExeFmtpCreateImageSection(FileObject, ImageSectionObject); if (!NT_SUCCESS(StatusExeFmt)) { + /* Unset */ + OldIrql = MiAcquirePfnLock(); + FileObject->SectionObjectPointer->ImageSectionObject = NULL; + MiReleasePfnLock(OldIrql); + if(ImageSectionObject->Segments != NULL) ExFreePool(ImageSectionObject->Segments); @@ -3747,6 +3138,7 @@ MmCreateImageSection(PSECTION *SectionObject, Section->Segment = (PSEGMENT)ImageSectionObject; ASSERT(ImageSectionObject->Segments); + ASSERT(ImageSectionObject->RefCount > 0); /* * Lock the file @@ -3754,6 +3146,11 @@ MmCreateImageSection(PSECTION *SectionObject, Status = MmspWaitForFileLock(FileObject); if (!NT_SUCCESS(Status)) { + /* Unset */ + OldIrql = MiAcquirePfnLock(); + FileObject->SectionObjectPointer->ImageSectionObject = NULL; + MiReleasePfnLock(OldIrql); + ExFreePool(ImageSectionObject->Segments); ExFreePool(ImageSectionObject); ObDereferenceObject(Section); @@ -3761,66 +3158,30 @@ MmCreateImageSection(PSECTION *SectionObject, return(Status); } - if (NULL != InterlockedCompareExchangePointer(&FileObject->SectionObjectPointer->ImageSectionObject, - ImageSectionObject, NULL)) - { - /* - * An other thread has initialized the same image in the background - */ - ExFreePool(ImageSectionObject->Segments); - ExFreePool(ImageSectionObject); - ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; - Section->Segment = (PSEGMENT)ImageSectionObject; - SectionSegments = ImageSectionObject->Segments; - - for (i = 0; i < ImageSectionObject->NrSegments; i++) - { - (void)InterlockedIncrementUL(&SectionSegments[i].ReferenceCount); - } - - /* We let the Image Section Object hold the reference */ - ObDereferenceObject(FileObject); - FileObject = ImageSectionObject->FileObject; - } + OldIrql = MiAcquirePfnLock(); + ImageSectionObject->SegFlags &= ~MM_SEGMENT_INCREATE; + MiReleasePfnLock(OldIrql); Status = StatusExeFmt; } else { - /* - * Lock the file - */ - Status = MmspWaitForFileLock(FileObject); - if (Status != STATUS_SUCCESS) - { - ObDereferenceObject(Section); - ObDereferenceObject(FileObject); - return(Status); - } + /* Take one ref */ + ImageSectionObject->RefCount++; - ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; - Section->Segment = (PSEGMENT)ImageSectionObject; - SectionSegments = ImageSectionObject->Segments; + MiReleasePfnLock(OldIrql); - /* - * Otherwise just reference all the section segments - */ - for (i = 0; i < ImageSectionObject->NrSegments; i++) - { - (void)InterlockedIncrementUL(&SectionSegments[i].ReferenceCount); - } + Section->Segment = (PSEGMENT)ImageSectionObject; /* We let the Image Section Object hold the reference */ ObDereferenceObject(FileObject); - FileObject = ImageSectionObject->FileObject; Status = STATUS_SUCCESS; } -#ifndef NEWCC - CcRosReferenceCache(FileObject); -#endif //KeSetEvent((PVOID)&FileObject->Lock, IO_NO_INCREMENT, FALSE); *SectionObject = Section; + ASSERT(ImageSectionObject->RefCount > 0); + return(Status); } @@ -3908,10 +3269,6 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, PFN_NUMBER Page, SWAPENTRY SwapEntry, BOOLEAN Dirty) { ULONG_PTR Entry; -#ifndef NEWCC - PFILE_OBJECT FileObject; - PROS_SHARED_CACHE_MAP SharedCacheMap; -#endif LARGE_INTEGER Offset; SWAPENTRY SavedSwapEntry; PMM_SECTION_SEGMENT Segment; @@ -3942,18 +3299,12 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, } /* - * For a dirty, datafile, non-private page mark it as dirty in the - * cache manager. + * For a dirty, datafile, non-private page, there shoulkd be no swap entry */ - if (Segment->Flags & MM_DATAFILE_SEGMENT) + if (*Segment->Flags & MM_DATAFILE_SEGMENT) { if (Page == PFN_FROM_SSE(Entry) && Dirty) { -#ifndef NEWCC - FileObject = MemoryArea->SectionData.Segment->FileObject; - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - CcRosMarkDirtyFile(SharedCacheMap, Offset.QuadPart + Segment->Image.FileOffset); -#endif ASSERT(SwapEntry == 0); } } @@ -3970,6 +3321,8 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, if (IS_SWAP_FROM_SSE(Entry) || Page != PFN_FROM_SSE(Entry)) { + ASSERT(Process != NULL); + /* * Just dereference private pages */ @@ -3984,7 +3337,8 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, } else { - MmDeleteRmap(Page, Process, Address); + if (Process) + MmDeleteRmap(Page, Process, Address); MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, FALSE, NULL); } } @@ -4467,6 +3821,8 @@ MmMapViewOfSection(IN PVOID SectionObject, SectionSegments = ImageSectionObject->Segments; NrSegments = ImageSectionObject->NrSegments; + ASSERT(ImageSectionObject->RefCount > 0); + ImageBase = (ULONG_PTR)*BaseAddress; if (ImageBase == 0) { @@ -4550,6 +3906,8 @@ MmMapViewOfSection(IN PVOID SectionObject, PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; LONGLONG ViewOffset; + ASSERT(Segment->RefCount > 0); + /* check for write access */ if ((Protect & (PAGE_READWRITE|PAGE_EXECUTE_READWRITE)) && !(Section->InitialPageProtection & (PAGE_READWRITE|PAGE_EXECUTE_READWRITE))) @@ -4633,68 +3991,58 @@ BOOLEAN NTAPI MmCanFileBeTruncated (IN PSECTION_OBJECT_POINTERS SectionObjectPointer, IN PLARGE_INTEGER NewFileSize) { + KIRQL OldIrql = MiAcquirePfnLock(); + BOOLEAN Ret; + PMM_SECTION_SEGMENT Segment; + +CheckSectionPointer: /* Check whether an ImageSectionObject exists */ if (SectionObjectPointer->ImageSectionObject != NULL) { DPRINT1("ERROR: File can't be truncated because it has an image section\n"); + MiReleasePfnLock(OldIrql); + return FALSE; } - if (SectionObjectPointer->DataSectionObject != NULL) + Segment = (PMM_SECTION_SEGMENT)SectionObjectPointer->DataSectionObject; + /* Wait for it to be created/deleted properly */ + while (Segment && (Segment->SegFlags & (MM_SEGMENT_INCREATE | MM_SEGMENT_INDELETE))) { - PMM_SECTION_SEGMENT Segment; + LARGE_INTEGER ShortTime; + + ShortTime.QuadPart = -10 * 100 * 1000; - Segment = (PMM_SECTION_SEGMENT)SectionObjectPointer-> - DataSectionObject; + /* Bad luck. Wait a bit for the operation to finish */ + MiReleasePfnLock(OldIrql); + KeDelayExecutionThread(KernelMode, FALSE, &ShortTime); + OldIrql = MiAcquirePfnLock(); + goto CheckSectionPointer; + } - if (Segment->ReferenceCount != 0) + if (Segment) + { + if ((Segment->SectionCount == 1) && (SectionObjectPointer->SharedCacheMap != NULL)) { -#ifdef NEWCC - CC_FILE_SIZES FileSizes; - CcpLock(); - if (SectionObjectPointer->SharedCacheMap && (Segment->ReferenceCount > CcpCountCacheSections((PNOCC_CACHE_MAP)SectionObjectPointer->SharedCacheMap))) - { - CcpUnlock(); - /* Check size of file */ - if (SectionObjectPointer->SharedCacheMap) - { - if (!CcGetFileSizes(Segment->FileObject, &FileSizes)) - { - return FALSE; - } - - if (NewFileSize->QuadPart <= FileSizes.FileSize.QuadPart) - { - return FALSE; - } - } - } - else - CcpUnlock(); -#else - /* Check size of file */ - if (SectionObjectPointer->SharedCacheMap) - { - PROS_SHARED_CACHE_MAP SharedCacheMap = SectionObjectPointer->SharedCacheMap; - if (NewFileSize->QuadPart <= SharedCacheMap->FileSize.QuadPart) - { - return FALSE; - } - } -#endif + /* If the cache is the only one holding a reference to the segment, then it's fine to resize */ + Ret = TRUE; } else { - /* Something must gone wrong - * how can we have a Section but no - * reference? */ - DPRINT("ERROR: DataSectionObject without reference!\n"); + /* We can't shrink, but we can extend */ + Ret = NewFileSize->QuadPart > Segment->RawLength.QuadPart; } } + else + { + Ret = TRUE; + } + + MiReleasePfnLock(OldIrql); DPRINT("FIXME: didn't check for outstanding write probes\n"); - return TRUE; + return Ret; } @@ -4797,8 +4145,7 @@ MmMapViewInSystemSpaceEx ( MmLockAddressSpace(AddressSpace); - - if ((*ViewSize == 0) || ((SectionOffset->QuadPart + *ViewSize) > Section->SizeOfSection.QuadPart)) + if (*ViewSize == 0) { *ViewSize = MIN((Section->SizeOfSection.QuadPart - SectionOffset->QuadPart), SIZE_T_MAX); } @@ -4812,7 +4159,7 @@ MmMapViewInSystemSpaceEx ( *ViewSize, PAGE_READWRITE, SectionOffset->QuadPart, - 0); + SEC_RESERVE); MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); @@ -4946,13 +4293,8 @@ MmCreateSection (OUT PVOID * Section, return STATUS_INVALID_PARAMETER_6; } - /* Did the caller pass an object? */ - if (FileObject) - { - /* Reference the object directly */ - ObReferenceObject(FileObject); - } - else + /* Did the caller pass a handle? */ + if (FileHandle) { /* Reference the file handle to get the object */ Status = ObReferenceObjectByHandle(FileHandle, @@ -4967,6 +4309,11 @@ MmCreateSection (OUT PVOID * Section, return Status; } } + else + { + /* Reference the object directly */ + ObReferenceObject(FileObject); + } } else { @@ -4974,63 +4321,6 @@ MmCreateSection (OUT PVOID * Section, if (AllocationAttributes & SEC_IMAGE) return STATUS_INVALID_FILE_FOR_SECTION; } -#ifndef NEWCC // A hack for initializing caching. - // This is needed only in the old case. - if (FileHandle) - { - IO_STATUS_BLOCK Iosb; - NTSTATUS Status; - CHAR Buffer; - LARGE_INTEGER ByteOffset; - ByteOffset.QuadPart = 0; - Status = ZwReadFile(FileHandle, - NULL, - NULL, - NULL, - &Iosb, - &Buffer, - sizeof(Buffer), - &ByteOffset, - NULL); - if (!NT_SUCCESS(Status) && Status != STATUS_END_OF_FILE) - { - DPRINT1("CC failure: %lx\n", Status); - if (FileObject) - ObDereferenceObject(FileObject); - return Status; - } - // Caching is initialized... - - // Hack of the hack: actually, it might not be initialized if FSD init on effective right and if file is null-size - // In such case, force cache by initiating a write IRP - if (Status == STATUS_END_OF_FILE && !(AllocationAttributes & SEC_IMAGE) && FileObject != NULL && - (FileObject->SectionObjectPointer == NULL || FileObject->SectionObjectPointer->SharedCacheMap == NULL)) - { - Buffer = 0xdb; - Status = ZwWriteFile(FileHandle, - NULL, - NULL, - NULL, ... 433 lines suppressed ...

3 years, 10 months

[reactos] 20/100: [NTOS/MM] Introduce MmArePagesResident and MmMakePagesResident

by Jérôme Gardou

https://git.reactos.org/?p=reactos.git;a=commitdiff;h=e4047d1521cd75ffded65… commit e4047d1521cd75ffded656aef221d7eb8285a04e Author: Jérôme Gardou <jerome.gardou(a)reactos.org> AuthorDate: Tue Oct 27 17:37:38 2020 +0100 Commit: Jérôme Gardou <jerome.gardou(a)reactos.org> CommitDate: Wed Feb 3 09:41:22 2021 +0100 [NTOS/MM] Introduce MmArePagesResident and MmMakePagesResident --- ntoskrnl/include/internal/mm.h | 14 +++++ ntoskrnl/mm/section.c | 140 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 662f49c9bd7..be8b0acc4b0 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -1323,6 +1323,20 @@ MmMapViewInSystemSpaceEx ( _Inout_ PLARGE_INTEGER SectionOffset ); +BOOLEAN +NTAPI +MmArePagesResident( + _In_ PEPROCESS Process, + _In_ PVOID BaseAddress, + _In_ ULONG Length); + +NTSTATUS +NTAPI +MmMakePagesResident( + _In_ PEPROCESS Process, + _In_ PVOID Address, + _In_ ULONG Length); + /* sysldr.c ******************************************************************/ VOID diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index fc2c1a3e904..4d29f5a9ff4 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -5076,4 +5076,144 @@ MmCreateSection (OUT PVOID * Section, return Status; } +BOOLEAN +NTAPI +MmArePagesResident( + _In_ PEPROCESS Process, + _In_ PVOID Address, + _In_ ULONG Length) +{ + PMEMORY_AREA MemoryArea; + BOOLEAN Ret = TRUE; + PMM_SECTION_SEGMENT Segment; + LARGE_INTEGER SegmentOffset, RangeEnd; + + MmLockAddressSpace(&Process->Vm); + + MemoryArea = MmLocateMemoryAreaByAddress(&Process->Vm, Address); + if (MemoryArea == NULL) + { + MmUnlockAddressSpace(&Process->Vm); + return FALSE; + } + + /* Only supported in old Mm for now */ + ASSERT(MemoryArea->Type == MEMORY_AREA_SECTION_VIEW); + /* For file mappings */ + ASSERT(MemoryArea->VadNode.u.VadFlags.VadType != VadImageMap); + + Segment = MemoryArea->SectionData.Segment; + MmLockSectionSegment(Segment); + + SegmentOffset.QuadPart = PAGE_ROUND_DOWN(Address) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset.QuadPart; + RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)Address + Length) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset.QuadPart; + + while (SegmentOffset.QuadPart < RangeEnd.QuadPart) + { + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + if ((Entry == 0) || IS_SWAP_FROM_SSE(Entry)) + { + Ret = FALSE; + break; + } + SegmentOffset.QuadPart += PAGE_SIZE; + } + + MmUnlockSectionSegment(Segment); + + MmUnlockAddressSpace(&Process->Vm); + return Ret; +} + +NTSTATUS +NTAPI +MmMakePagesResident( + _In_ PEPROCESS Process, + _In_ PVOID Address, + _In_ ULONG Length) +{ + PMEMORY_AREA MemoryArea; + PMM_SECTION_SEGMENT Segment; + LARGE_INTEGER SegmentOffset, RangeEnd; + + MmLockAddressSpace(&Process->Vm); + + MemoryArea = MmLocateMemoryAreaByAddress(&Process->Vm, Address); + if (MemoryArea == NULL) + { + MmUnlockAddressSpace(&Process->Vm); + return FALSE; + } + + /* Only supported in old Mm for now */ + ASSERT(MemoryArea->Type == MEMORY_AREA_SECTION_VIEW); + /* For file mappings */ + ASSERT(MemoryArea->VadNode.u.VadFlags.VadType != VadImageMap); + + Segment = MemoryArea->SectionData.Segment; + MmLockSectionSegment(Segment); + + SegmentOffset.QuadPart = PAGE_ROUND_DOWN(Address) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset.QuadPart; + RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)Address + Length) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset.QuadPart; + + while (SegmentOffset.QuadPart < RangeEnd.QuadPart) + { + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + + /* Let any pending read proceed */ + while (MM_IS_WAIT_PTE(Entry)) + { + MmUnlockSectionSegment(Segment); + MmUnlockAddressSpace(&Process->Vm); + MiWaitForPageEvent(NULL, NULL); + MmLockAddressSpace(&Process->Vm); + MmLockSectionSegment(Segment); + Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + } + + /* We are called from Cc, this can't be backed by the page files */ + ASSERT(!IS_SWAP_FROM_SSE(Entry)); + + /* At this point, there may be a valid page there */ + if (Entry == 0) + { + PFN_NUMBER Page; + NTSTATUS Status; + + /* + * Release all our locks and read in the page from disk + */ + MmSetPageEntrySectionSegment(Segment, &SegmentOffset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); + MmUnlockSectionSegment(Segment); + MmUnlockAddressSpace(&Process->Vm); + + /* FIXME: Read the whole range at once instead of one page at a time */ + Status = MiReadPage(MemoryArea, SegmentOffset.QuadPart, &Page); + if (!NT_SUCCESS(Status)) + { + /* Reset the Segment entry and fail */ + MmLockSectionSegment(Segment); + MmSetPageEntrySectionSegment(Segment, &SegmentOffset, 0); + MmUnlockSectionSegment(Segment); + MiSetPageEvent(Process, Address); + return Status; + } + + MmLockAddressSpace(&Process->Vm); + MmLockSectionSegment(Segment); + MmSetPageEntrySectionSegment(Segment, &SegmentOffset, MAKE_SSE(Page << PAGE_SHIFT, 1)); + } + SegmentOffset.QuadPart += PAGE_SIZE; + } + + MmUnlockSectionSegment(Segment); + + MmUnlockAddressSpace(&Process->Vm); + return STATUS_SUCCESS; +} + /* EOF */

3 years, 10 months

[reactos] 19/100: [NTOS/MM] Introduce MmMapViewInSystemSpaceEx

by Jérôme Gardou

https://git.reactos.org/?p=reactos.git;a=commitdiff;h=f1631b44e120f2c6319fb… commit f1631b44e120f2c6319fb4d4b113b0a43336b413 Author: Jérôme Gardou <jerome.gardou(a)reactos.org> AuthorDate: Tue Oct 27 17:36:18 2020 +0100 Commit: Jérôme Gardou <jerome.gardou(a)reactos.org> CommitDate: Wed Feb 3 09:41:22 2021 +0100 [NTOS/MM] Introduce MmMapViewInSystemSpaceEx --- ntoskrnl/include/internal/mm.h | 10 +++++++++ ntoskrnl/mm/ARM3/section.c | 38 +++++++++++++++++++++++-------- ntoskrnl/mm/section.c | 51 ++++++++++++++++++++++++++---------------- 3 files changed, 71 insertions(+), 28 deletions(-) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index a8536c3d786..662f49c9bd7 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -1313,6 +1313,16 @@ VOID NTAPI MmFreeSectionSegments(PFILE_OBJECT FileObject); +/* Exported from NT 6.2 Onward. We keep it internal. */ +NTSTATUS +NTAPI +MmMapViewInSystemSpaceEx ( + _In_ PVOID Section, + _Outptr_result_bytebuffer_ (*ViewSize) PVOID *MappedBase, + _Inout_ PSIZE_T ViewSize, + _Inout_ PLARGE_INTEGER SectionOffset + ); + /* sysldr.c ******************************************************************/ VOID diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index 511f11a5101..938a08d6037 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -416,12 +416,16 @@ NTSTATUS NTAPI MiAddMappedPtes(IN PMMPTE FirstPte, IN PFN_NUMBER PteCount, - IN PCONTROL_AREA ControlArea) + IN PCONTROL_AREA ControlArea, + IN PLARGE_INTEGER SectionOffset) { MMPTE TempPte; PMMPTE PointerPte, ProtoPte, LastProtoPte, LastPte; PSUBSECTION Subsection; + /* Mapping at offset not supported yet */ + ASSERT(SectionOffset->QuadPart == 0); + /* ARM3 doesn't support this yet */ ASSERT(ControlArea->u.Flags.GlobalOnlyPerSession == 0); ASSERT(ControlArea->u.Flags.Rom == 0); @@ -1052,11 +1056,13 @@ NTAPI MiMapViewInSystemSpace(IN PVOID Section, IN PMMSESSION Session, OUT PVOID *MappedBase, - IN OUT PSIZE_T ViewSize) + IN OUT PSIZE_T ViewSize, + IN PLARGE_INTEGER SectionOffset) { PVOID Base; PCONTROL_AREA ControlArea; - ULONG Buckets, SectionSize; + ULONG Buckets; + LONGLONG SectionSize; NTSTATUS Status; PAGED_CODE(); @@ -1073,13 +1079,23 @@ MiMapViewInSystemSpace(IN PVOID Section, ASSERT(NT_SUCCESS(Status)); /* Get the section size at creation time */ - SectionSize = ((PSECTION)Section)->SizeOfSection.LowPart; + SectionSize = ((PSECTION)Section)->SizeOfSection.QuadPart; - /* If the caller didn't specify a view size, assume the whole section */ - if (!(*ViewSize)) *ViewSize = SectionSize; + /* If the caller didn't specify a view size, assume until the end of the section */ + if (!(*ViewSize)) + { + /* Check for overflow first */ + if ((SectionSize - SectionOffset->QuadPart) > SIZE_T_MAX) + { + DPRINT1("Section end is too far away from the specified offset.\n"); + MiDereferenceControlArea(ControlArea); + return STATUS_INVALID_VIEW_SIZE; + } + *ViewSize = SectionSize - SectionOffset->QuadPart; + } /* Check if the caller wanted a larger section than the view */ - if (*ViewSize > SectionSize) + if (SectionOffset->QuadPart + *ViewSize > SectionSize) { /* Fail */ DPRINT1("View is too large\n"); @@ -1129,7 +1145,8 @@ MiMapViewInSystemSpace(IN PVOID Section, /* Create the actual prototype PTEs for this mapping */ Status = MiAddMappedPtes(MiAddressToPte(Base), BYTES_TO_PAGES(*ViewSize), - ControlArea); + ControlArea, + SectionOffset); ASSERT(NT_SUCCESS(Status)); /* Return the base adress of the mapping and success */ @@ -3016,6 +3033,7 @@ MmMapViewInSessionSpace(IN PVOID Section, IN OUT PSIZE_T ViewSize) { PAGED_CODE(); + LARGE_INTEGER SectionOffset; // HACK if (MiIsRosSectionObject(Section)) @@ -3032,10 +3050,12 @@ MmMapViewInSessionSpace(IN PVOID Section, /* Use the system space API, but with the session view instead */ ASSERT(MmIsAddressValid(MmSessionSpace) == TRUE); + SectionOffset.QuadPart = 0; return MiMapViewInSystemSpace(Section, &MmSessionSpace->Session, MappedBase, - ViewSize); + ViewSize, + &SectionOffset); } /* diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 04cba04d612..fc2c1a3e904 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -88,7 +88,8 @@ NTAPI MiMapViewInSystemSpace(IN PVOID Section, IN PVOID Session, OUT PVOID *MappedBase, - IN OUT PSIZE_T ViewSize); + IN OUT PSIZE_T ViewSize, + IN PLARGE_INTEGER SectionOffset); NTSTATUS NTAPI @@ -3832,7 +3833,7 @@ MmMapViewOfSegment(PMMSUPPORT AddressSpace, PVOID* BaseAddress, SIZE_T ViewSize, ULONG Protect, - ULONG ViewOffset, + LONGLONG ViewOffset, ULONG AllocationType) { PMEMORY_AREA MArea; @@ -4417,7 +4418,6 @@ MmMapViewOfSection(IN PVOID SectionObject, { PSECTION Section; PMMSUPPORT AddressSpace; - ULONG ViewOffset; NTSTATUS Status = STATUS_SUCCESS; BOOLEAN NotAtBase = FALSE; @@ -4530,7 +4530,7 @@ MmMapViewOfSection(IN PVOID SectionObject, Section, &SectionSegments[i], &SBaseAddress, - SectionSegments[i].Length.LowPart, + SectionSegments[i].Length.QuadPart, SectionSegments[i].Protection, 0, 0); @@ -4548,6 +4548,7 @@ MmMapViewOfSection(IN PVOID SectionObject, else { PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; + LONGLONG ViewOffset; /* check for write access */ if ((Protect & (PAGE_READWRITE|PAGE_EXECUTE_READWRITE)) && @@ -4577,7 +4578,7 @@ MmMapViewOfSection(IN PVOID SectionObject, } else { - ViewOffset = SectionOffset->u.LowPart; + ViewOffset = SectionOffset->QuadPart; } if ((ViewOffset % PAGE_SIZE) != 0) @@ -4588,11 +4589,11 @@ MmMapViewOfSection(IN PVOID SectionObject, if ((*ViewSize) == 0) { - (*ViewSize) = Section->SizeOfSection.u.LowPart - ViewOffset; + (*ViewSize) = Section->SizeOfSection.QuadPart - ViewOffset; } - else if (((*ViewSize)+ViewOffset) > Section->SizeOfSection.u.LowPart) + else if (((*ViewSize)+ViewOffset) > Section->SizeOfSection.QuadPart) { - (*ViewSize) = Section->SizeOfSection.u.LowPart - ViewOffset; + (*ViewSize) = MIN(Section->SizeOfSection.QuadPart - ViewOffset, SIZE_T_MAX - PAGE_SIZE); } *ViewSize = PAGE_ROUND_UP(*ViewSize); @@ -4756,7 +4757,23 @@ MmMapViewInSystemSpace (IN PVOID SectionObject, OUT PVOID * MappedBase, IN OUT PSIZE_T ViewSize) { - PSECTION Section; + LARGE_INTEGER SectionOffset; + + SectionOffset.QuadPart = 0; + + return MmMapViewInSystemSpaceEx(SectionObject, MappedBase, ViewSize, &SectionOffset); +} + +NTSTATUS +NTAPI +MmMapViewInSystemSpaceEx ( + _In_ PVOID SectionObject, + _Outptr_result_bytebuffer_ (*ViewSize) PVOID *MappedBase, + _Inout_ PSIZE_T ViewSize, + _Inout_ PLARGE_INTEGER SectionOffset + ) +{ + PSECTION Section = SectionObject; PMM_SECTION_SEGMENT Segment; PMMSUPPORT AddressSpace; NTSTATUS Status; @@ -4767,10 +4784,11 @@ MmMapViewInSystemSpace (IN PVOID SectionObject, return MiMapViewInSystemSpace(SectionObject, &MmSession, MappedBase, - ViewSize); + ViewSize, + SectionOffset); } - DPRINT("MmMapViewInSystemSpace() called\n"); + DPRINT("MmMapViewInSystemSpaceEx() called\n"); Section = SectionObject; Segment = (PMM_SECTION_SEGMENT)Section->Segment; @@ -4780,25 +4798,20 @@ MmMapViewInSystemSpace (IN PVOID SectionObject, MmLockAddressSpace(AddressSpace); - if ((*ViewSize) == 0) + if ((*ViewSize == 0) || ((SectionOffset->QuadPart + *ViewSize) > Section->SizeOfSection.QuadPart)) { - (*ViewSize) = Section->SizeOfSection.u.LowPart; - } - else if ((*ViewSize) > Section->SizeOfSection.u.LowPart) - { - (*ViewSize) = Section->SizeOfSection.u.LowPart; + *ViewSize = MIN((Section->SizeOfSection.QuadPart - SectionOffset->QuadPart), SIZE_T_MAX); } MmLockSectionSegment(Segment); - Status = MmMapViewOfSegment(AddressSpace, Section, Segment, MappedBase, *ViewSize, PAGE_READWRITE, - 0, + SectionOffset->QuadPart, 0); MmUnlockSectionSegment(Segment);

3 years, 10 months

[reactos] 18/100: [NTOS/MM] Infer that the Memory Area is an image mapping from its VAD node

by Jérôme Gardou

https://git.reactos.org/?p=reactos.git;a=commitdiff;h=4f6fd6c42b4fa712dad5c… commit 4f6fd6c42b4fa712dad5cffc9522e801a38279c1 Author: Jérôme Gardou <jerome.gardou(a)reactos.org> AuthorDate: Mon Oct 26 17:49:16 2020 +0100 Commit: Jérôme Gardou <jerome.gardou(a)reactos.org> CommitDate: Wed Feb 3 09:41:21 2021 +0100 [NTOS/MM] Infer that the Memory Area is an image mapping from its VAD node --- ntoskrnl/mm/section.c | 57 +++++++++++---------------------------------------- 1 file changed, 12 insertions(+), 45 deletions(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index f65c5c1ff64..04cba04d612 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -138,7 +138,7 @@ C_ASSERT(PEFMT_FIELDS_EQUAL(IMAGE_OPTIONAL_HEADER32, IMAGE_OPTIONAL_HEADER64, Si typedef struct { - PSECTION Section; + PMEMORY_AREA MemoryArea; PMM_SECTION_SEGMENT Segment; LARGE_INTEGER Offset; BOOLEAN WasDirty; @@ -899,7 +899,7 @@ MmSharePageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, BOOLEAN NTAPI -MmUnsharePageEntrySectionSegment(PSECTION Section, +MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset, BOOLEAN Dirty, @@ -939,7 +939,7 @@ MmUnsharePageEntrySectionSegment(PSECTION Section, LARGE_INTEGER FileOffset; FileOffset.QuadPart = Offset->QuadPart + Segment->Image.FileOffset; - IsImageSection = Section->u.Flags.Image; + IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; #endif Page = PFN_FROM_SSE(Entry); @@ -1112,7 +1112,7 @@ MiReadPage(PMEMORY_AREA MemoryArea, SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; RawLength = MemoryArea->SectionData.Segment->RawLength.QuadPart; FileOffset = SegOffset + MemoryArea->SectionData.Segment->Image.FileOffset; - IsImageSection = MemoryArea->SectionData.Section->u.Flags.Image; + IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; ASSERT(SharedCacheMap); @@ -1631,7 +1631,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, MmCreatePageFileMapping(Process, PAddress, MM_WAIT_ENTRY); MmUnlockAddressSpace(AddressSpace); - if ((Offset.QuadPart >= (LONGLONG)PAGE_ROUND_UP(Segment->RawLength.QuadPart)) && Section->u.Flags.Image) + if ((Offset.QuadPart >= (LONGLONG)PAGE_ROUND_UP(Segment->RawLength.QuadPart)) && (MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap)) { MI_SET_USAGE(MI_USAGE_SECTION); if (Process) MI_SET_PROCESS2(Process->ImageFileName); @@ -1812,7 +1812,6 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, PVOID Address) { PMM_SECTION_SEGMENT Segment; - PSECTION Section; PFN_NUMBER OldPage; PFN_NUMBER NewPage; NTSTATUS Status; @@ -1849,7 +1848,6 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, + MemoryArea->SectionData.ViewOffset.QuadPart; Segment = MemoryArea->SectionData.Segment; - Section = MemoryArea->SectionData.Section; Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), &MemoryArea->SectionData.RegionListHead, Address, NULL); @@ -1907,7 +1905,7 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, DPRINT("Swapping page (Old %x New %x)\n", OldPage, NewPage); MmDeleteVirtualMapping(Process, PAddress, NULL, NULL); MmDeleteRmap(OldPage, Process, PAddress); - MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, FALSE, FALSE, NULL); + MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, FALSE, FALSE, NULL); MmUnlockSectionSegment(Segment); /* @@ -1955,7 +1953,7 @@ MmPageOutDeleteMapping(PVOID Context, PEPROCESS Process, PVOID Address) if (!PageOutContext->Private) { MmLockSectionSegment(PageOutContext->Segment); - MmUnsharePageEntrySectionSegment(PageOutContext->Section, + MmUnsharePageEntrySectionSegment(PageOutContext->MemoryArea, PageOutContext->Segment, &PageOutContext->Offset, PageOutContext->WasDirty, @@ -2000,7 +1998,7 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, * Get the segment and section. */ Context.Segment = MemoryArea->SectionData.Segment; - Context.Section = MemoryArea->SectionData.Section; + Context.MemoryArea = MemoryArea; Context.SectionEntry = Entry; Context.CallingProcess = Process; @@ -2013,7 +2011,7 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, #ifndef NEWCC FileOffset = Context.Offset.QuadPart + Context.Segment->Image.FileOffset; - IsImageSection = Context.Section->u.Flags.Image; + IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; FileObject = Context.Segment->FileObject; if (FileObject != NULL && @@ -2034,19 +2032,6 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, } #endif - - /* - * This should never happen since mappings of physical memory are never - * placed in the rmap lists. - */ - if (Context.Section->u.Flags.PhysicalMemory) - { - DPRINT1("Trying to page out from physical memory section address 0x%p " - "process %p\n", Address, - Process ? Process->UniqueProcessId : 0); - KeBugCheck(MEMORY_MANAGEMENT); - } - /* * Get the section segment entry and the physical address. */ @@ -2367,7 +2352,6 @@ MmWritePageSectionView(PMMSUPPORT AddressSpace, ULONG PageEntry) { LARGE_INTEGER Offset; - PSECTION Section; PMM_SECTION_SEGMENT Segment; PFN_NUMBER Page; SWAPENTRY SwapEntry; @@ -2391,8 +2375,7 @@ MmWritePageSectionView(PMMSUPPORT AddressSpace, * Get the segment and section. */ Segment = MemoryArea->SectionData.Segment; - Section = MemoryArea->SectionData.Section; - IsImageSection = Section->u.Flags.Image; + IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; FileObject = Segment->FileObject; DirectMapped = FALSE; @@ -2415,18 +2398,6 @@ MmWritePageSectionView(PMMSUPPORT AddressSpace, } } - /* - * This should never happen since mappings of physical memory are never - * placed in the rmap lists. - */ - if (Section->u.Flags.PhysicalMemory) - { - DPRINT1("Trying to write back page from physical memory mapped at %p " - "process %p\n", Address, - Process ? Process->UniqueProcessId : 0); - KeBugCheck(MEMORY_MANAGEMENT); - } - /* * Get the section segment entry and the physical address. */ @@ -2557,7 +2528,6 @@ MmQuerySectionView(PMEMORY_AREA MemoryArea, { PMM_REGION Region; PVOID RegionBaseAddress; - PSECTION Section; PMM_SECTION_SEGMENT Segment; Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), @@ -2568,8 +2538,7 @@ MmQuerySectionView(PMEMORY_AREA MemoryArea, return STATUS_UNSUCCESSFUL; } - Section = MemoryArea->SectionData.Section; - if (Section->u.Flags.Image) + if (MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap) { Segment = MemoryArea->SectionData.Segment; Info->AllocationBase = (PUCHAR)MA_GetStartingAddress(MemoryArea) - Segment->Image.VirtualAddress; @@ -3944,7 +3913,6 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, #endif LARGE_INTEGER Offset; SWAPENTRY SavedSwapEntry; - PSECTION Section; PMM_SECTION_SEGMENT Segment; PMMSUPPORT AddressSpace; PEPROCESS Process; @@ -3957,7 +3925,6 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, Offset.QuadPart = ((ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea)) + MemoryArea->SectionData.ViewOffset.QuadPart; - Section = MemoryArea->SectionData.Section; Segment = MemoryArea->SectionData.Segment; Entry = MmGetPageEntrySectionSegment(Segment, &Offset); @@ -4017,7 +3984,7 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, else { MmDeleteRmap(Page, Process, Address); - MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, Dirty, FALSE, NULL); + MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, FALSE, NULL); } } }

3 years, 10 months

← Newer
1
...
36
37
38
39
40
41
42
43
44
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

2006

2005

2004

Ros-diffs February 2021