3 Feb
2021
3 Feb
'21
9:32 a.m.
https://git.reactos.org/?p=reactos.git;a=commitdiff;h=2ba1926037223899fa205…
commit 2ba1926037223899fa2052d9a72663f095d37f54
Author: Jérôme Gardou <jerome.gardou(a)reactos.org>
AuthorDate: Fri Jan 29 18:48:32 2021 +0100
Commit: Jérôme Gardou <jerome.gardou(a)reactos.org>
CommitDate: Wed Feb 3 09:41:23 2021 +0100
[NTOS:MM][NTOS:CC] Performance improvement again
Read files by 64kb chunks instead of page-sized chunks.
---
ntoskrnl/cc/copy.c | 48 +---
ntoskrnl/cc/pin.c | 5 -
ntoskrnl/cc/view.c | 40 +++-
ntoskrnl/include/internal/cc.h | 2 +-
ntoskrnl/include/internal/mm.h | 15 +-
ntoskrnl/mm/section.c | 499 +++++++++++++++++++++--------------------
6 files changed, 298 insertions(+), 311 deletions(-)
diff --git a/ntoskrnl/cc/copy.c b/ntoskrnl/cc/copy.c
index 8ed24f2bd92..cdab94c8313 100644
--- a/ntoskrnl/cc/copy.c
+++ b/ntoskrnl/cc/copy.c
@@ -504,24 +504,6 @@ CcCopyRead (
CurrentOffset = FileOffset->QuadPart;
while(CurrentOffset < ReadEnd)
{
- if (CurrentOffset >= SharedCacheMap->ValidDataLength.QuadPart)
- {
- DPRINT1("Zeroing buffer because we are beyond the VDL.\n");
- /* We are beyond what is valid. Just zero this out */
- _SEH2_TRY
- {
- RtlZeroMemory(Buffer, Length);
- }
- _SEH2_EXCEPT(CcpCheckInvalidUserBuffer(_SEH2_GetExceptionInformation(),
Buffer, Length))
- {
- ExRaiseStatus(STATUS_INVALID_USER_BUFFER);
- }
- _SEH2_END;
-
- ReadLength += Length;
- break;
- }
-
Status = CcRosGetVacb(SharedCacheMap, CurrentOffset, &Vacb);
if (!NT_SUCCESS(Status))
{
@@ -538,25 +520,15 @@ CcCopyRead (
if (!CcRosEnsureVacbResident(Vacb, Wait, FALSE, VacbOffset, VacbLength))
return FALSE;
- /* Do not copy past the section */
- if (CurrentOffset + VacbLength > SharedCacheMap->SectionSize.QuadPart)
- CopyLength = SharedCacheMap->SectionSize.QuadPart - CurrentOffset;
- if (CopyLength != 0)
+ _SEH2_TRY
{
- _SEH2_TRY
- {
- RtlCopyMemory(Buffer, (PUCHAR)Vacb->BaseAddress + VacbOffset,
CopyLength);
- }
- _SEH2_EXCEPT(CcpCheckInvalidUserBuffer(_SEH2_GetExceptionInformation(),
Buffer, VacbLength))
- {
- ExRaiseStatus(STATUS_INVALID_USER_BUFFER);
- }
- _SEH2_END;
+ RtlCopyMemory(Buffer, (PUCHAR)Vacb->BaseAddress + VacbOffset,
CopyLength);
}
-
- /* Zero-out the buffer tail if needed */
- if (CopyLength < VacbLength)
- RtlZeroMemory((PUCHAR)Buffer + CopyLength, VacbLength - CopyLength);
+ _SEH2_EXCEPT(CcpCheckInvalidUserBuffer(_SEH2_GetExceptionInformation(),
Buffer, VacbLength))
+ {
+ ExRaiseStatus(STATUS_INVALID_USER_BUFFER);
+ }
+ _SEH2_END;
ReadLength += VacbLength;
@@ -684,10 +656,6 @@ CcCopyWrite (
if (FileObject->Flags & FO_WRITE_THROUGH)
CcFlushCache(FileObject->SectionObjectPointer, FileOffset, Length, NULL);
- /* Update VDL */
- if (WriteEnd > SharedCacheMap->ValidDataLength.QuadPart)
- SharedCacheMap->ValidDataLength.QuadPart = WriteEnd;
-
return TRUE;
}
@@ -898,7 +866,7 @@ CcZeroData (
}
/* See if we should simply truncate the valid data length */
- if ((StartOffset->QuadPart < SharedCacheMap->ValidDataLength.QuadPart)
&& (EndOffset->QuadPart > SharedCacheMap->ValidDataLength.QuadPart))
+ if ((StartOffset->QuadPart < SharedCacheMap->ValidDataLength.QuadPart)
&& (EndOffset->QuadPart >= SharedCacheMap->ValidDataLength.QuadPart))
{
DPRINT1("Truncating VDL.\n");
SharedCacheMap->ValidDataLength = *StartOffset;
diff --git a/ntoskrnl/cc/pin.c b/ntoskrnl/cc/pin.c
index af7b3aa71a8..c48f1068042 100644
--- a/ntoskrnl/cc/pin.c
+++ b/ntoskrnl/cc/pin.c
@@ -543,7 +543,6 @@ CcSetDirtyPinnedData (
IN PLARGE_INTEGER Lsn)
{
PINTERNAL_BCB iBcb = CONTAINING_RECORD(Bcb, INTERNAL_BCB, PFCB);
- PROS_SHARED_CACHE_MAP SharedCacheMap = iBcb->Vacb->SharedCacheMap;
CCTRACE(CC_API_DEBUG, "Bcb=%p Lsn=%p\n", Bcb, Lsn);
@@ -556,10 +555,6 @@ CcSetDirtyPinnedData (
{
CcRosMarkDirtyVacb(iBcb->Vacb);
}
-
- /* Update VDL */
- if (SharedCacheMap->ValidDataLength.QuadPart <
(iBcb->PFCB.MappedFileOffset.QuadPart + iBcb->PFCB.MappedLength))
- SharedCacheMap->ValidDataLength.QuadPart =
iBcb->PFCB.MappedFileOffset.QuadPart + iBcb->PFCB.MappedLength;
}
diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c
index 591168eeba0..c0f1458287d 100644
--- a/ntoskrnl/cc/view.c
+++ b/ntoskrnl/cc/view.c
@@ -166,12 +166,12 @@ MmFlushVirtualMemory(IN PEPROCESS Process,
NTSTATUS
NTAPI
CcRosFlushVacb (
- PROS_VACB Vacb)
+ _In_ PROS_VACB Vacb,
+ _In_ PIO_STATUS_BLOCK Iosb)
{
- IO_STATUS_BLOCK Iosb;
- SIZE_T FlushSize = VACB_MAPPING_GRANULARITY;
NTSTATUS Status;
BOOLEAN HaveLock = FALSE;
+ PROS_SHARED_CACHE_MAP SharedCacheMap = Vacb->SharedCacheMap;
CcRosUnmarkDirtyVacb(Vacb, TRUE);
@@ -184,7 +184,10 @@ CcRosFlushVacb (
HaveLock = TRUE;
}
- Status = MmFlushVirtualMemory(NULL, &Vacb->BaseAddress, &FlushSize,
&Iosb);
+ Status = MmFlushSegment(SharedCacheMap->FileObject->SectionObjectPointer,
+ &Vacb->FileOffset,
+ VACB_MAPPING_GRANULARITY,
+ Iosb);
if (HaveLock)
{
@@ -194,6 +197,14 @@ CcRosFlushVacb (
quit:
if (!NT_SUCCESS(Status))
CcRosMarkDirtyVacb(Vacb);
+ else
+ {
+ /* Update VDL */
+ if (SharedCacheMap->ValidDataLength.QuadPart <
(Vacb->FileOffset.QuadPart + VACB_MAPPING_GRANULARITY))
+ {
+ SharedCacheMap->ValidDataLength.QuadPart = Vacb->FileOffset.QuadPart +
VACB_MAPPING_GRANULARITY;
+ }
+ }
return Status;
}
@@ -285,7 +296,8 @@ CcRosFlushDirtyPages (
continue;
}
- Status = CcRosFlushVacb(current);
+ IO_STATUS_BLOCK Iosb;
+ Status = CcRosFlushVacb(current, &Iosb);
SharedCacheMap->Callbacks->ReleaseFromLazyWrite(SharedCacheMap->LazyWriteContext);
@@ -308,7 +320,7 @@ CcRosFlushDirtyPages (
ULONG PagesFreed;
/* How many pages did we free? */
- PagesFreed = VACB_MAPPING_GRANULARITY / PAGE_SIZE;
+ PagesFreed = Iosb.Information / PAGE_SIZE;
(*Count) += PagesFreed;
if (!Wait)
@@ -756,7 +768,11 @@ CcRosEnsureVacbResident(
if (!NoRead)
{
- NTSTATUS Status = MmMakePagesResident(NULL, BaseAddress, Length);
+ PROS_SHARED_CACHE_MAP SharedCacheMap = Vacb->SharedCacheMap;
+ NTSTATUS Status =
MmMakeDataSectionResident(SharedCacheMap->FileObject->SectionObjectPointer,
+ Vacb->FileOffset.QuadPart +
Offset,
+ Length,
+
&SharedCacheMap->ValidDataLength);
if (!NT_SUCCESS(Status))
ExRaiseStatus(Status);
}
@@ -935,7 +951,6 @@ CcFlushCache (
}
Status = STATUS_SUCCESS;
-
if (IoStatus)
{
IoStatus->Information = 0;
@@ -953,9 +968,10 @@ CcFlushCache (
if (vacb != NULL)
{
+ IO_STATUS_BLOCK VacbIosb;
if (vacb->Dirty)
{
- Status = CcRosFlushVacb(vacb);
+ Status = CcRosFlushVacb(vacb, &VacbIosb);
if (!NT_SUCCESS(Status))
{
goto quit;
@@ -966,7 +982,7 @@ CcFlushCache (
CcRosReleaseVacb(SharedCacheMap, vacb, FALSE, FALSE);
if (IoStatus)
- IoStatus->Information += VACB_MAPPING_GRANULARITY;
+ IoStatus->Information += VacbIosb.Information;
}
if (!DirtyVacb)
@@ -992,6 +1008,10 @@ CcFlushCache (
if (IoStatus)
IoStatus->Information += MmIosb.Information;
+
+ /* Update VDL */
+ if (SharedCacheMap->ValidDataLength.QuadPart < FlushEnd)
+ SharedCacheMap->ValidDataLength.QuadPart = FlushEnd;
}
if (!NT_SUCCESS(RtlLongLongAdd(FlushStart, VACB_MAPPING_GRANULARITY,
&FlushStart)))
diff --git a/ntoskrnl/include/internal/cc.h b/ntoskrnl/include/internal/cc.h
index 7384d9be0d3..cf57319449b 100644
--- a/ntoskrnl/include/internal/cc.h
+++ b/ntoskrnl/include/internal/cc.h
@@ -310,7 +310,7 @@ CcMdlWriteComplete2(
NTSTATUS
NTAPI
-CcRosFlushVacb(PROS_VACB Vacb);
+CcRosFlushVacb(PROS_VACB Vacb, PIO_STATUS_BLOCK Iosb);
NTSTATUS
NTAPI
diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h
index 0243ac55177..cec0d6e13d1 100644
--- a/ntoskrnl/include/internal/mm.h
+++ b/ntoskrnl/include/internal/mm.h
@@ -1369,13 +1369,6 @@ MmArePagesResident(
_In_ PVOID BaseAddress,
_In_ ULONG Length);
-NTSTATUS
-NTAPI
-MmMakePagesResident(
- _In_ PEPROCESS Process,
- _In_ PVOID Address,
- _In_ ULONG Length);
-
NTSTATUS
NTAPI
MmMakePagesDirty(
@@ -1399,6 +1392,14 @@ MmFlushSegment(
_In_ ULONG Length,
_In_opt_ PIO_STATUS_BLOCK Iosb);
+NTSTATUS
+NTAPI
+MmMakeDataSectionResident(
+ _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer,
+ _In_ LONGLONG Offset,
+ _In_ ULONG Length,
+ _In_ PLARGE_INTEGER ValidDataLength);
+
BOOLEAN
NTAPI
MmPurgeSegment(
diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c
index 94ea1844d2d..d848e337038 100644
--- a/ntoskrnl/mm/section.c
+++ b/ntoskrnl/mm/section.c
@@ -1181,126 +1181,234 @@ MiCopyFromUserPage(PFN_NUMBER DestPage, const VOID *SrcAddress)
return(STATUS_SUCCESS);
}
-#ifndef NEWCC
static
NTSTATUS
NTAPI
-MiReadPage(PMEMORY_AREA MemoryArea,
- LONGLONG SegOffset,
- PPFN_NUMBER Page,
- BOOLEAN IgnoreSize)
-/*
- * FUNCTION: Read a page for a section backed memory area.
- * PARAMETERS:
- * MemoryArea - Memory area to read the page for.
- * Offset - Offset of the page to read.
- * Page - Variable that receives a page contains the read data.
- */
+MmMakeSegmentResident(
+ _In_ PMM_SECTION_SEGMENT Segment,
+ _In_ LONGLONG Offset,
+ _In_ ULONG Length,
+ _In_opt_ PLARGE_INTEGER ValidDataLength)
{
+ /* Let's use a 64K granularity. */
+ LONGLONG RangeStart, RangeEnd;
NTSTATUS Status;
- IO_STATUS_BLOCK IoStatus;
- KEVENT Event;
- UCHAR MdlBase[sizeof(MDL) + sizeof(PFN_NUMBER)];
- PMDL Mdl = (PMDL)MdlBase;
- PFILE_OBJECT FileObject = MemoryArea->SectionData.Segment->FileObject;
- LARGE_INTEGER FileOffset;
- KIRQL OldIrql;
- PFSRTL_ADVANCED_FCB_HEADER FcbHeader = FileObject->FsContext;
-
- FileOffset.QuadPart = MemoryArea->SectionData.Segment->Image.FileOffset +
SegOffset;
-
- DPRINT("Reading file at offset %08x:%08x\n", FileOffset.HighPart,
FileOffset.LowPart);
+ PFILE_OBJECT FileObject = Segment->FileObject;
- Status = MmRequestPageMemoryConsumer(MC_USER, FALSE, Page);
+ /* Calculate our range, aligned on 64K if possible. */
+ Status = RtlLongLongAdd(Offset, Length, &RangeEnd);
+ ASSERT(NT_SUCCESS(Status));
if (!NT_SUCCESS(Status))
return Status;
- if ((FileOffset.QuadPart > FcbHeader->ValidDataLength.QuadPart) &&
!IgnoreSize)
+ RangeStart = Offset - (Offset % _64K);
+ if (RangeEnd % _64K)
+ RangeEnd += _64K - (RangeEnd % _64K);
+
+ /* Clamp if needed */
+ if (!FlagOn(*Segment->Flags, MM_DATAFILE_SEGMENT))
{
- /* Quick path : data is not valid; return a zero-page */
- return STATUS_SUCCESS;
+ if (RangeEnd > Segment->RawLength.QuadPart)
+ RangeEnd = Segment->RawLength.QuadPart;
}
- RtlZeroMemory(MdlBase, sizeof(MdlBase));
- MmInitializeMdl(Mdl, NULL, PAGE_SIZE);
- MmBuildMdlFromPages(Mdl, Page);
- Mdl->MdlFlags |= MDL_PAGES_LOCKED | MDL_IO_PAGE_READ;
+ /* Let's gooooooooo */
+ for ( ; RangeStart < RangeEnd; RangeStart += _64K)
+ {
+ /* First take a look at where we miss pages */
+ ULONG ToReadPageBits = 0;
+ LONGLONG ChunkEnd = RangeStart + _64K;
- KeInitializeEvent(&Event, NotificationEvent, FALSE);
+ if (ChunkEnd > RangeEnd)
+ ChunkEnd = RangeEnd;
- /* Disable APCs */
- KeRaiseIrql(APC_LEVEL, &OldIrql);
+ MmLockSectionSegment(Segment);
+ for (LONGLONG ChunkOffset = RangeStart; ChunkOffset < ChunkEnd; ChunkOffset +=
PAGE_SIZE)
+ {
+ LARGE_INTEGER CurrentOffset;
- Status = IoPageRead(FileObject, Mdl, &FileOffset, &Event, &IoStatus);
- if (Status == STATUS_PENDING)
- {
- KeWaitForSingleObject(&Event, WrPageIn, KernelMode, FALSE, NULL);
- Status = IoStatus.Status;
- }
+ CurrentOffset.QuadPart = ChunkOffset;
+ ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &CurrentOffset);
- if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA)
- {
- MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl);
- }
+ /* Let any pending read proceed */
+ while (MM_IS_WAIT_PTE(Entry))
+ {
+ MmUnlockSectionSegment(Segment);
- KeLowerIrql(OldIrql);
+ KeDelayExecutionThread(KernelMode, FALSE, &TinyTime);
- if (Status == STATUS_END_OF_FILE)
- {
- DPRINT1("Got STATUS_END_OF_FILE at offset %I64d for file %wZ.\n",
SegOffset, &FileObject->FileName);
- Status = STATUS_SUCCESS;
- }
+ MmLockSectionSegment(Segment);
+ Entry = MmGetPageEntrySectionSegment(Segment, &CurrentOffset);
+ }
- if ((MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap)
- && ((SegOffset + PAGE_SIZE) >
MemoryArea->SectionData.Segment->RawLength.QuadPart))
- {
- KIRQL OldIrql;
- PUCHAR PageMap;
+ if (Entry != 0)
+ {
+ /* There is a page here. Or a swap entry. Or whatever... */
+ continue;
+ }
- DPRINT("Zeroing at offset %I64d for file %wZ.\n", SegOffset,
&FileObject->FileName);
+ ToReadPageBits |= 1UL << ((ChunkOffset - RangeStart) >>
PAGE_SHIFT);
- /* Zero out the end of it */
- PageMap = MiMapPageInHyperSpace(PsGetCurrentProcess(), *Page, &OldIrql);
- RtlZeroMemory(PageMap + MemoryArea->SectionData.Segment->RawLength.QuadPart
- SegOffset,
- PAGE_SIZE -
(MemoryArea->SectionData.Segment->RawLength.QuadPart - SegOffset));
- MiUnmapPageInHyperSpace(PsGetCurrentProcess(), PageMap, OldIrql);
- }
+ /* Put a wait entry here */
+ MmSetPageEntrySectionSegment(Segment, &CurrentOffset,
MAKE_SWAP_SSE(MM_WAIT_ENTRY));
+ }
+ MmUnlockSectionSegment(Segment);
- return Status;
-}
+ if (ToReadPageBits == 0)
+ {
+ /* Nothing to do for this chunk */
+ continue;
+ }
-#else
-NTSTATUS
-NTAPI
-MiReadPage(PMEMORY_AREA MemoryArea,
- LONGLONG SegOffset,
- PPFN_NUMBER Page)
-/*
- * FUNCTION: Read a page for a section backed memory area.
- * PARAMETERS:
- * MemoryArea - Memory area to read the page for.
- * Offset - Offset of the page to read.
- * Page - Variable that receives a page contains the read data.
- */
-{
- MM_REQUIRED_RESOURCES Resources;
- NTSTATUS Status;
+ /* Now perform the actual read */
+ LONGLONG ChunkOffset = RangeStart;
+ while (ChunkOffset < ChunkEnd)
+ {
+ /* Move forward if there is a hole */
+ ULONG BitSet;
+ if (!_BitScanForward(&BitSet, ToReadPageBits))
+ {
+ /* Nothing more to read */
+ break;
+ }
+ ToReadPageBits >>= BitSet;
+ ChunkOffset += BitSet * PAGE_SIZE;
+ ASSERT(ChunkOffset < ChunkEnd);
- RtlZeroMemory(&Resources, sizeof(MM_REQUIRED_RESOURCES));
+ /* Get the range we have to read */
+ _BitScanForward(&BitSet, ~ToReadPageBits);
+ ULONG ReadLength = BitSet * PAGE_SIZE;
- Resources.Context = MemoryArea->SectionData.Section->FileObject;
- Resources.FileOffset.QuadPart = SegOffset +
-
MemoryArea->SectionData.Segment->Image.FileOffset;
- Resources.Consumer = MC_USER;
- Resources.Amount = PAGE_SIZE;
+ ASSERT(ReadLength <= _64K);
- DPRINT("%S, offset 0x%x, len 0x%x, page 0x%x\n",
((PFILE_OBJECT)Resources.Context)->FileName.Buffer, Resources.FileOffset.LowPart,
Resources.Amount, Resources.Page[0]);
+ /* Clamp (This is for image mappings */
+ if ((ChunkOffset + ReadLength) > ChunkEnd)
+ ReadLength = ChunkEnd - ChunkOffset;
- Status = MiReadFilePage(MmGetKernelAddressSpace(), MemoryArea, &Resources);
- *Page = Resources.Page[0];
- return Status;
+ ASSERT(ReadLength != 0);
+
+ /* Allocate a MDL */
+ PMDL Mdl = IoAllocateMdl(NULL, ReadLength, FALSE, FALSE, NULL);
+ if (!Mdl)
+ {
+ /* Damn. Roll-back. */
+ MmLockSectionSegment(Segment);
+ while (ChunkOffset < ChunkEnd)
+ {
+ if (ToReadPageBits & 1)
+ {
+ LARGE_INTEGER CurrentOffset;
+ CurrentOffset.QuadPart = ChunkOffset;
+ ASSERT(MM_IS_WAIT_PTE(MmGetPageEntrySectionSegment(Segment,
&CurrentOffset)));
+ MmSetPageEntrySectionSegment(Segment, &CurrentOffset, 0);
+ }
+ ToReadPageBits >>= 1;
+ ChunkOffset += PAGE_SIZE;
+ }
+ MmUnlockSectionSegment(Segment);
+ return STATUS_INSUFFICIENT_RESOURCES;
+ }
+
+ /* Get our pages */
+ PPFN_NUMBER Pages = MmGetMdlPfnArray(Mdl);
+ RtlZeroMemory(Pages, BYTES_TO_PAGES(ReadLength) * sizeof(PFN_NUMBER));
+ for (UINT i = 0; i < BYTES_TO_PAGES(ReadLength); i++)
+ {
+ /* MmRequestPageMemoryConsumer succeeds or bugchecks */
+ (void)MmRequestPageMemoryConsumer(MC_USER, FALSE, &Pages[i]);
+ }
+ Mdl->MdlFlags |= MDL_PAGES_LOCKED | MDL_IO_PAGE_READ;
+
+ LARGE_INTEGER FileOffset;
+ FileOffset.QuadPart = Segment->Image.FileOffset + ChunkOffset;
+
+ /* Clamp to VDL */
+ if (ValidDataLength && ((FileOffset.QuadPart + ReadLength) >
ValidDataLength->QuadPart))
+ {
+ if (FileOffset.QuadPart > ValidDataLength->QuadPart)
+ {
+ /* Great, nothing to read. */
+ goto AssignPagesToSegment;
+ }
+
+ Mdl->Size = (FileOffset.QuadPart + ReadLength) -
ValidDataLength->QuadPart;
+ }
+
+ KEVENT Event;
+ KeInitializeEvent(&Event, NotificationEvent, FALSE);
+
+ /* Disable APCs */
+ KIRQL OldIrql;
+ KeRaiseIrql(APC_LEVEL, &OldIrql);
+
+ IO_STATUS_BLOCK Iosb;
+ Status = IoPageRead(FileObject, Mdl, &FileOffset, &Event,
&Iosb);
+ if (Status == STATUS_PENDING)
+ {
+ KeWaitForSingleObject(&Event, WrPageIn, KernelMode, FALSE, NULL);
+ Status = Iosb.Status;
+ }
+
+ if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA)
+ {
+ MmUnmapLockedPages(Mdl->MappedSystemVa, Mdl);
+ }
+
+ KeLowerIrql(OldIrql);
+
+ if (Status == STATUS_END_OF_FILE)
+ {
+ DPRINT1("Got STATUS_END_OF_FILE at offset %I64d for file
%wZ.\n", FileOffset.QuadPart, &FileObject->FileName);
+ Status = STATUS_SUCCESS;
+ }
+
+ if (!NT_SUCCESS(Status))
+ {
+ /* Damn. Roll back. */
+ for (UINT i = 0; i < BYTES_TO_PAGES(ReadLength); i++)
+ MmReleasePageMemoryConsumer(MC_USER, Pages[i]);
+
+ MmLockSectionSegment(Segment);
+ while (ChunkOffset < ChunkEnd)
+ {
+ if (ToReadPageBits & 1)
+ {
+ LARGE_INTEGER CurrentOffset;
+ CurrentOffset.QuadPart = ChunkOffset;
+ ASSERT(MM_IS_WAIT_PTE(MmGetPageEntrySectionSegment(Segment,
&CurrentOffset)));
+ MmSetPageEntrySectionSegment(Segment, &CurrentOffset, 0);
+ }
+ ToReadPageBits >>= 1;
+ ChunkOffset += PAGE_SIZE;
+ }
+ MmUnlockSectionSegment(Segment);
+ IoFreeMdl(Mdl);;
+ return Status;
+ }
+
+AssignPagesToSegment:
+ MmLockSectionSegment(Segment);
+
+ for (UINT i = 0; i < BYTES_TO_PAGES(ReadLength); i++)
+ {
+ LARGE_INTEGER CurrentOffset;
+ CurrentOffset.QuadPart = ChunkOffset + (i * PAGE_SIZE);
+
+ ASSERT(MM_IS_WAIT_PTE(MmGetPageEntrySectionSegment(Segment,
&CurrentOffset)));
+
+ MmSetPageEntrySectionSegment(Segment, &CurrentOffset,
MAKE_SSE(Pages[i] << PAGE_SHIFT, 0));
+ }
+
+ MmUnlockSectionSegment(Segment);
+
+ IoFreeMdl(Mdl);
+ ToReadPageBits >>= BitSet;
+ ChunkOffset += BitSet * PAGE_SIZE;
+ }
+ }
+
+ return STATUS_SUCCESS;
}
-#endif
static VOID
MmAlterViewAttributes(PMMSUPPORT AddressSpace,
@@ -1610,85 +1718,57 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace,
if (Entry == 0)
{
- SWAPENTRY FakeSwapEntry;
-
- /*
- * If the entry is zero (and it can't change because we have
- * locked the segment) then we need to load the page.
- */
-
/*
- * Release all our locks and read in the page from disk
+ * If the entry is zero, then we need to load the page.
*/
- MmSetPageEntrySectionSegment(Segment, &Offset,
MAKE_SWAP_SSE(MM_WAIT_ENTRY));
- MmUnlockSectionSegment(Segment);
- MmCreatePageFileMapping(Process, PAddress, MM_WAIT_ENTRY);
- MmUnlockAddressSpace(AddressSpace);
-
if ((Offset.QuadPart >=
(LONGLONG)PAGE_ROUND_UP(Segment->RawLength.QuadPart)) &&
(MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap))
{
+ /* We are beyond the data which is on file. Just get a new page. */
MI_SET_USAGE(MI_USAGE_SECTION);
if (Process) MI_SET_PROCESS2(Process->ImageFileName);
if (!Process) MI_SET_PROCESS2("Kernel Section");
- Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &Page);
- if (!NT_SUCCESS(Status))
- {
- DPRINT1("MmRequestPageMemoryConsumer failed (Status %x)\n",
Status);
- }
+ MmRequestPageMemoryConsumer(MC_USER, FALSE, &Page);
+ MmSetPageEntrySectionSegment(Segment, &Offset, MAKE_SSE(Page <<
PAGE_SHIFT, 1));
+ MmUnlockSectionSegment(Segment);
- }
- else
- {
- DPRINT("Getting fresh page for file %wZ at offset %I64d.\n",
&Segment->FileObject->FileName, Offset.QuadPart);
- Status = MiReadPage(MemoryArea, Offset.QuadPart, &Page, FALSE);
+ Status = MmCreateVirtualMapping(Process, PAddress, Attributes, &Page,
1);
if (!NT_SUCCESS(Status))
{
- DPRINT1("MiReadPage failed (Status %x)\n", Status);
+ DPRINT1("Unable to create virtual mapping\n");
+ KeBugCheck(MEMORY_MANAGEMENT);
}
- }
- if (!NT_SUCCESS(Status))
- {
- /*
- * FIXME: What do we know in this case?
- */
- /*
- * Cleanup and release locks
- */
- MmLockAddressSpace(AddressSpace);
+ ASSERT(MmIsPagePresent(Process, PAddress));
+ if (Process)
+ MmInsertRmap(Page, Process, Address);
+
MiSetPageEvent(Process, Address);
DPRINT("Address 0x%p\n", Address);
- return(Status);
+ return(STATUS_SUCCESS);
}
- /* Lock both segment and process address space while we proceed. */
- MmLockAddressSpace(AddressSpace);
- MmLockSectionSegment(Segment);
+ MmUnlockSectionSegment(Segment);
+ MmUnlockAddressSpace(AddressSpace);
- MmDeletePageFileMapping(Process, PAddress, &FakeSwapEntry);
- DPRINT("CreateVirtualMapping Page %x Process %p PAddress %p Attributes
%x\n",
- Page, Process, PAddress, Attributes);
- Status = MmCreateVirtualMapping(Process,
- PAddress,
- Attributes,
- &Page,
- 1);
+ /* The data must be paged in. Lock the file, so that the VDL doesn't get
updated behind us. */
+ FsRtlAcquireFileExclusive(Segment->FileObject);
+
+ PFSRTL_COMMON_FCB_HEADER FcbHeader = Segment->FileObject->FsContext;
+
+ Status = MmMakeSegmentResident(Segment, Offset.QuadPart, PAGE_SIZE,
&FcbHeader->ValidDataLength);
+
+ FsRtlReleaseFile(Segment->FileObject);
+
+ /* Lock address space again */
+ MmLockAddressSpace(AddressSpace);
if (!NT_SUCCESS(Status))
{
- DPRINT1("Unable to create virtual mapping\n");
- KeBugCheck(MEMORY_MANAGEMENT);
+ /* Damn */
+ DPRINT1("Failed to page data in!\n");
+ return STATUS_IN_PAGE_ERROR;
}
- ASSERT(MmIsPagePresent(Process, PAddress));
- if (Process)
- MmInsertRmap(Page, Process, Address);
-
- /* Set this section offset has being backed by our new page. */
- Entry = MAKE_SSE(Page << PAGE_SHIFT, 1);
- MmSetPageEntrySectionSegment(Segment, &Offset, Entry);
- MmUnlockSectionSegment(Segment);
- MiSetPageEvent(Process, Address);
- DPRINT("Address 0x%p\n", Address);
- return(STATUS_SUCCESS);
+ /* Everything went fine. Restart the operation */
+ return STATUS_MM_RESTART_OPERATION;
}
else if (IS_SWAP_FROM_SSE(Entry))
{
@@ -4458,103 +4538,6 @@ MmArePagesResident(
return Ret;
}
-NTSTATUS
-NTAPI
-MmMakePagesResident(
- _In_ PEPROCESS Process,
- _In_ PVOID Address,
- _In_ ULONG Length)
-{
- PMEMORY_AREA MemoryArea;
- PMM_SECTION_SEGMENT Segment;
- LARGE_INTEGER SegmentOffset, RangeEnd;
- PMMSUPPORT AddressSpace = Process ? &Process->Vm : MmGetKernelAddressSpace();
-
- MmLockAddressSpace(AddressSpace);
-
- MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address);
- if (MemoryArea == NULL)
- {
- DPRINT1("Unable to find memory area at address %p.\n", Address);
- MmUnlockAddressSpace(AddressSpace);
- return STATUS_NOT_MAPPED_VIEW;
- }
-
- /* Only supported in old Mm for now */
- ASSERT(MemoryArea->Type == MEMORY_AREA_SECTION_VIEW);
- /* For file mappings */
- ASSERT(MemoryArea->VadNode.u.VadFlags.VadType != VadImageMap);
-
- Segment = MemoryArea->SectionData.Segment;
- MmLockSectionSegment(Segment);
-
- SegmentOffset.QuadPart = PAGE_ROUND_DOWN(Address) -
MA_GetStartingAddress(MemoryArea)
- + MemoryArea->SectionData.ViewOffset.QuadPart;
- RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)Address + Length) -
MA_GetStartingAddress(MemoryArea)
- + MemoryArea->SectionData.ViewOffset.QuadPart;
-
- DPRINT("MmMakePagesResident: Segment %p, 0x%I64x -> 0x%I64x\n", Segment,
SegmentOffset.QuadPart, RangeEnd.QuadPart);
-
- while (SegmentOffset.QuadPart < RangeEnd.QuadPart)
- {
- ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset);
-
- /* Let any pending read proceed */
- while (MM_IS_WAIT_PTE(Entry))
- {
- MmUnlockSectionSegment(Segment);
- MmUnlockAddressSpace(AddressSpace);
- MiWaitForPageEvent(NULL, NULL);
- MmLockAddressSpace(AddressSpace);
- MmLockSectionSegment(Segment);
- Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset);
- }
-
- /* We are called from Cc, this can't be backed by the page files */
- ASSERT(!IS_SWAP_FROM_SSE(Entry));
-
- /* At this point, there may be a valid page there */
- if (Entry == 0)
- {
- PFN_NUMBER Page;
- NTSTATUS Status;
-
- /*
- * Release all our locks and read in the page from disk
- */
- MmSetPageEntrySectionSegment(Segment, &SegmentOffset,
MAKE_SWAP_SSE(MM_WAIT_ENTRY));
- MmUnlockSectionSegment(Segment);
- MmUnlockAddressSpace(AddressSpace);
-
- /* FIXME: Read the whole range at once instead of one page at a time */
- /* Ignore file size, as Cc already checked on its side. */
- Status = MiReadPage(MemoryArea, SegmentOffset.QuadPart, &Page, TRUE);
- if (!NT_SUCCESS(Status))
- {
- /* Reset the Segment entry and fail */
- MmLockSectionSegment(Segment);
- MmSetPageEntrySectionSegment(Segment, &SegmentOffset, 0);
- MmUnlockSectionSegment(Segment);
- MiSetPageEvent(Process, Address);
- return Status;
- }
-
- MmLockAddressSpace(AddressSpace);
- MmLockSectionSegment(Segment);
-
- /* We set it with 0 ref count, nobody maps this page yet. */
- MmSetPageEntrySectionSegment(Segment, &SegmentOffset, MAKE_SSE(Page
<< PAGE_SHIFT, 0));
- MiSetPageEvent(Process, Address);
- }
- SegmentOffset.QuadPart += PAGE_SIZE;
- }
-
- MmUnlockSectionSegment(Segment);
-
- MmUnlockAddressSpace(AddressSpace);
- return STATUS_SUCCESS;
-}
-
NTSTATUS
NTAPI
MmRosFlushVirtualMemory(
@@ -4723,6 +4706,26 @@ MmPurgeSegment(
return TRUE;
}
+NTSTATUS
+NTAPI
+MmMakeDataSectionResident(
+ _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer,
+ _In_ LONGLONG Offset,
+ _In_ ULONG Length,
+ _In_ PLARGE_INTEGER ValidDataLength)
+{
+ PMM_SECTION_SEGMENT Segment = MiGrabDataSection(SectionObjectPointer);
+
+ /* There must be a segment for this call */
+ ASSERT(Segment);
+
+ NTSTATUS Status = MmMakeSegmentResident(Segment, Offset, Length, ValidDataLength);
+
+ MmDereferenceSegment(Segment);
+
+ return Status;
+}
+
NTSTATUS
NTAPI
MmFlushSegment(