30 Jun
2006
30 Jun
'06
3:59 p.m.
Author: ion
Date: Fri Jun 30 19:59:06 2006
New Revision: 22717
URL: http://svn.reactos.org/svn/reactos?rev=22717&view=rev
Log:
- Fix IRP/Completion packet lookaside allocation. We weren't setting a zone size, and
I/O Completion packets were also using the wrong size.
- Rewrite I/O MDl support to use lookaside lists for allocations below 23 pages (same as
on NT). This is is an incredible performance optimization because MDLs are often allocated
and de-allocated during I/O operations, and using the lookaside list decreases pool
fragmentation and slowdown.
- Rewrite IoBuildPartialMdl. It did not work like documented in the DDK and also had a bug
documented by Microsoft as being in XP.
Modified:
trunk/reactos/ntoskrnl/include/internal/io.h
trunk/reactos/ntoskrnl/io/iomgr.c
trunk/reactos/ntoskrnl/io/mdl.c (contents, props changed)
Modified: trunk/reactos/ntoskrnl/include/internal/io.h
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/include/internal/…
==============================================================================
--- trunk/reactos/ntoskrnl/include/internal/io.h (original)
+++ trunk/reactos/ntoskrnl/include/internal/io.h Fri Jun 30 19:59:06 2006
@@ -47,6 +47,14 @@
// Number of partition tables in the Boot Record
//
#define PARTITION_TBL_SIZE 4
+
+//
+// We can call the Ob Inlined API, it's the same thing
+//
+#define IopAllocateMdlFromLookaside \
+ ObpAllocateCapturedAttributes
+#define IopFreeMdlFromLookaside \
+ ObpFreeCapturedAttributes
//
// Returns the size of a CM_RESOURCE_LIST
@@ -830,4 +838,4 @@
extern POBJECT_TYPE IoCompletionType;
extern PDEVICE_NODE IopRootDeviceNode;
extern ULONG IopTraceLevel;
-
+extern NPAGED_LOOKASIDE_LIST IopMdlLookasideList;
Modified: trunk/reactos/ntoskrnl/io/iomgr.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/io/iomgr.c?rev=22…
==============================================================================
--- trunk/reactos/ntoskrnl/io/iomgr.c (original)
+++ trunk/reactos/ntoskrnl/io/iomgr.c Fri Jun 30 19:59:06 2006
@@ -41,6 +41,7 @@
extern POBJECT_TYPE IoAdapterObjectType;
NPAGED_LOOKASIDE_LIST IoLargeIrpLookaside;
NPAGED_LOOKASIDE_LIST IoSmallIrpLookaside;
+NPAGED_LOOKASIDE_LIST IopMdlLookasideList;
VOID INIT_FUNCTION IopInitLookasideLists(VOID);
@@ -74,7 +75,7 @@
INIT_FUNCTION
IopInitLookasideLists(VOID)
{
- ULONG LargeIrpSize, SmallIrpSize;
+ ULONG LargeIrpSize, SmallIrpSize, MdlSize;
LONG i;
PKPRCB Prcb;
PNPAGED_LOOKASIDE_LIST CurrentList = NULL;
@@ -82,6 +83,7 @@
/* Calculate the sizes */
LargeIrpSize = sizeof(IRP) + (8 * sizeof(IO_STACK_LOCATION));
SmallIrpSize = sizeof(IRP) + sizeof(IO_STACK_LOCATION);
+ MdlSize = sizeof(MDL) + (23 * sizeof(PFN_NUMBER));
/* Initialize the Lookaside List for Large IRPs */
ExInitializeNPagedLookasideList(&IoLargeIrpLookaside,
@@ -90,7 +92,7 @@
0,
LargeIrpSize,
IO_LARGEIRP,
- 0);
+ 64);
/* Initialize the Lookaside List for Small IRPs */
ExInitializeNPagedLookasideList(&IoSmallIrpLookaside,
@@ -99,7 +101,7 @@
0,
SmallIrpSize,
IO_SMALLIRP,
- 0);
+ 32);
/* Initialize the Lookaside List for I\O Completion */
ExInitializeNPagedLookasideList(&IoCompletionPacketLookaside,
@@ -108,7 +110,16 @@
0,
sizeof(IO_COMPLETION_PACKET),
IOC_TAG1,
- 0);
+ 32);
+
+ /* Initialize the Lookaside List for MDLs */
+ ExInitializeNPagedLookasideList(&IopMdlLookasideList,
+ NULL,
+ NULL,
+ 0,
+ MdlSize,
+ TAG_MDL,
+ 128);
/* Now allocate the per-processor lists */
for (i = 0; i < KeNumberProcessors; i++)
@@ -131,7 +142,7 @@
0,
LargeIrpSize,
IO_LARGEIRP_CPU,
- 0);
+ 64);
}
else
{
@@ -146,14 +157,14 @@
IO_SMALLIRP_CPU);
if (CurrentList)
{
- /* Initialize the Lookaside List for Large IRPs */
+ /* Initialize the Lookaside List for Small IRPs */
ExInitializeNPagedLookasideList(CurrentList,
NULL,
NULL,
0,
SmallIrpSize,
IO_SMALLIRP_CPU,
- 0);
+ 32);
}
else
{
@@ -173,15 +184,37 @@
NULL,
NULL,
0,
+ sizeof(IO_COMPLETION_PACKET),
+ IO_SMALLIRP_CPU,
+ 32);
+ }
+ else
+ {
+ CurrentList = &IoCompletionPacketLookaside;
+ }
+ Prcb->PPLookasideList[LookasideCompletionList].P = &CurrentList->L;
+
+ /* Set the MDL Completion List */
+ Prcb->PPLookasideList[LookasideMdlList].L = &IopMdlLookasideList.L;
+ CurrentList = ExAllocatePoolWithTag(NonPagedPool,
+ sizeof(NPAGED_LOOKASIDE_LIST),
+ TAG_MDL);
+ if (CurrentList)
+ {
+ /* Initialize the Lookaside List for MDLs */
+ ExInitializeNPagedLookasideList(CurrentList,
+ NULL,
+ NULL,
+ 0,
SmallIrpSize,
- IOC_CPU,
- 0);
+ TAG_MDL,
+ 128);
}
else
{
- CurrentList = &IoCompletionPacketLookaside;
- }
- Prcb->PPLookasideList[LookasideCompletionList].P = &CurrentList->L;
+ CurrentList = &IopMdlLookasideList;
+ }
+ Prcb->PPLookasideList[LookasideMdlList].P = &CurrentList->L;
}
DPRINT("Done allocation\n");
Modified: trunk/reactos/ntoskrnl/io/mdl.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/io/mdl.c?rev=2271…
==============================================================================
--- trunk/reactos/ntoskrnl/io/mdl.c (original)
+++ trunk/reactos/ntoskrnl/io/mdl.c Fri Jun 30 19:59:06 2006
@@ -1,11 +1,9 @@
-/* $Id$
- *
- * COPYRIGHT: See COPYING in the top level directory
- * PROJECT: ReactOS kernel
+/*
+ * PROJECT: ReactOS Kernel
+ * LICENSE: GPL - See COPYING in the top level directory
* FILE: ntoskrnl/io/mdl.c
- * PURPOSE: Io manager mdl functions
- *
- * PROGRAMMERS: David Welch (welch(a)mcmail.com)
+ * PURPOSE: I/O Wrappers for MDL Allocation and Deallocation
+ * PROGRAMMERS: Alex Ionescu (alex.ionescu(a)reactos.org)
*/
/* INCLUDES *****************************************************************/
@@ -20,116 +18,143 @@
* @implemented
*/
PMDL
-STDCALL
-IoAllocateMdl(PVOID VirtualAddress,
- ULONG Length,
- BOOLEAN SecondaryBuffer,
- BOOLEAN ChargeQuota,
- PIRP Irp)
+NTAPI
+IoAllocateMdl(IN PVOID VirtualAddress,
+ IN ULONG Length,
+ IN BOOLEAN SecondaryBuffer,
+ IN BOOLEAN ChargeQuota,
+ IN PIRP Irp)
{
- PMDL Mdl;
+ PMDL Mdl = NULL, p;
+ ULONG Flags = 0;
+ ULONG Size;
- if (ChargeQuota)
- {
-// Mdl = ExAllocatePoolWithQuota(NonPagedPool,
-// MmSizeOfMdl(VirtualAddress,Length));
- Mdl = ExAllocatePoolWithTag(NonPagedPool,
- MmSizeOfMdl(VirtualAddress,Length),
- TAG_MDL);
- }
- else
- {
- Mdl = ExAllocatePoolWithTag(NonPagedPool,
- MmSizeOfMdl(VirtualAddress,Length),
- TAG_MDL);
- }
- MmInitializeMdl(Mdl, (char*)VirtualAddress, Length);
+ /* Fail if allocation is over 2GB */
+ if (Length & 0x80000000) return NULL;
- if (Irp)
- {
- if (SecondaryBuffer)
- {
- ASSERT(Irp->MdlAddress);
+ /* Calculate the number of pages for the allocation */
+ Size = ADDRESS_AND_SIZE_TO_SPAN_PAGES(VirtualAddress, Length);
+ if (Size > 23)
+ {
+ /* This is bigger then our fixed-size MDLs. Calculate real size */
+ Size *= sizeof(PFN_NUMBER);
+ Size += sizeof(MDL);
+ if (Size > MAXUSHORT) return NULL;
+ }
+ else
+ {
+ /* Use an internal fixed MDL size */
+ Size = (23 * sizeof(PFN_NUMBER)) + sizeof(MDL);
+ Flags |= MDL_ALLOCATED_FIXED_SIZE;
- /* FIXME: add to end of list maybe?? */
- Mdl->Next = Irp->MdlAddress->Next;
- Irp->MdlAddress->Next = Mdl;
+ /* Allocate one from the lookaside list */
+ Mdl = IopAllocateMdlFromLookaside(LookasideMdlList);
+ }
+
+ /* Check if we don't have an mdl yet */
+ if (!Mdl)
+ {
+ /* Allocate one from pool */
+ Mdl = ExAllocatePoolWithTag(NonPagedPool, Size, TAG_MDL);
+ if (!Mdl) return NULL;
+ }
+
+ /* Initialize it */
+ MmInitializeMdl(Mdl, VirtualAddress, Length);
+ Mdl->MdlFlags |= Flags;
+
+ /* Check if an IRP was given too */
+ if (Irp)
+ {
+ /* Check if it came with a secondary buffer */
+ if (SecondaryBuffer)
+ {
+ /* Insert the MDL at the end */
+ p = Irp->MdlAddress;
+ while (p->Next) p = p->Next;
+ p->Next = Mdl;
}
else
{
- /*
- * What if there's allready an mdl at Irp->MdlAddress?
- * Is that bad and should we do something about it?
- */
- Irp->MdlAddress = Mdl;
+ /* Otherwise, insert it directly */
+ Irp->MdlAddress = Mdl;
}
}
- return(Mdl);
-}
-
-/*
- * @implemented
- *
- * You must IoFreeMdl the slave before freeing the master.
- *
- * IoBuildPartialMdl is more similar to MmBuildMdlForNonPagedPool, the difference
- * is that the former takes the physical addresses from the master MDL, while the
- * latter - from the known location of the NPP.
- */
-VOID
-STDCALL
-IoBuildPartialMdl(PMDL SourceMdl,
- PMDL TargetMdl,
- PVOID VirtualAddress,
- ULONG Length)
-{
- PPFN_TYPE TargetPages = (PPFN_TYPE)(TargetMdl + 1);
- PPFN_TYPE SourcePages = (PPFN_TYPE)(SourceMdl + 1);
- ULONG Count;
- ULONG Delta;
-
- DPRINT("VirtualAddress 0x%p, SourceMdl->StartVa 0x%p,
SourceMdl->MappedSystemVa 0x%p\n",
- VirtualAddress, SourceMdl->StartVa, SourceMdl->MappedSystemVa);
-
- TargetMdl->StartVa = (PVOID)PAGE_ROUND_DOWN(VirtualAddress);
- TargetMdl->ByteOffset = (ULONG_PTR)VirtualAddress -
(ULONG_PTR)TargetMdl->StartVa;
- TargetMdl->ByteCount = Length;
- TargetMdl->Process = SourceMdl->Process;
- Delta = (ULONG_PTR)VirtualAddress - ((ULONG_PTR)SourceMdl->StartVa +
SourceMdl->ByteOffset);
- TargetMdl->MappedSystemVa = (char*)SourceMdl->MappedSystemVa + Delta;
-
- TargetMdl->MdlFlags = SourceMdl->MdlFlags &
(MDL_IO_PAGE_READ|MDL_SOURCE_IS_NONPAGED_POOL|MDL_MAPPED_TO_SYSTEM_VA);
- TargetMdl->MdlFlags |= MDL_PARTIAL;
-
- Delta = ((ULONG_PTR)TargetMdl->StartVa - (ULONG_PTR)SourceMdl->StartVa) /
PAGE_SIZE;
- Count = ADDRESS_AND_SIZE_TO_SPAN_PAGES(VirtualAddress,Length);
-
- SourcePages += Delta;
-
- DPRINT("Delta %d, Count %d\n", Delta, Count);
-
- memcpy(TargetPages, SourcePages, Count * sizeof(PFN_TYPE));
-
+ /* Return the allocated mdl */
+ return Mdl;
}
/*
* @implemented
*/
-VOID STDCALL
+VOID
+NTAPI
+IoBuildPartialMdl(IN PMDL SourceMdl,
+ IN PMDL TargetMdl,
+ IN PVOID VirtualAddress,
+ IN ULONG Length)
+{
+ PPFN_TYPE TargetPages = (PPFN_TYPE)(TargetMdl + 1);
+ PPFN_TYPE SourcePages = (PPFN_TYPE)(SourceMdl + 1);
+ ULONG Offset;
+
+ /* Calculate the offset */
+ Offset = (ULONG)((ULONG_PTR)VirtualAddress -
+ (ULONG_PTR)SourceMdl->StartVa) -
+ SourceMdl->ByteOffset;
+
+ /* Check if we don't have a length and calculate it */
+ if (!Length) Length = SourceMdl->ByteCount - Offset;
+
+ /* Write the process, start VA and byte data */
+ TargetMdl->StartVa = (PVOID)PAGE_ROUND_DOWN(VirtualAddress);
+ TargetMdl->Process = SourceMdl->Process;
+ TargetMdl->ByteCount = Length;
+ TargetMdl->ByteOffset = BYTE_OFFSET(VirtualAddress);
+
+ /* Recalculate the length in pages */
+ Length = ADDRESS_AND_SIZE_TO_SPAN_PAGES(VirtualAddress, Length);
+
+ /* Set the MDL Flags */
+ TargetMdl->MdlFlags = (MDL_ALLOCATED_FIXED_SIZE | MDL_ALLOCATED_MUST_SUCCEED);
+ TargetMdl->MdlFlags |= (MDL_IO_PAGE_READ |
+ MDL_SOURCE_IS_NONPAGED_POOL |
+ MDL_MAPPED_TO_SYSTEM_VA |
+ MDL_IO_SPACE);
+ TargetMdl->MdlFlags |= MDL_PARTIAL;
+
+ /* Set the mapped VA */
+ TargetMdl->MappedSystemVa = (PCHAR)SourceMdl->MappedSystemVa + Offset;
+
+ /* Now do the copy */
+ Offset = ((ULONG_PTR)TargetMdl->StartVa - (ULONG_PTR)SourceMdl->StartVa)
>>
+ PAGE_SHIFT;
+ SourcePages += Offset;
+ RtlMoveMemory(TargetPages, SourcePages, Length * sizeof(PFN_TYPE));
+}
+
+/*
+ * @implemented
+ */
+VOID
+NTAPI
IoFreeMdl(PMDL Mdl)
{
- /*
- * This unmaps partial mdl's from kernel space but also asserts that non-partial
- * mdl's isn't still mapped into kernel space.
- */
- ASSERT(Mdl);
- ASSERT_IRQL(DISPATCH_LEVEL);
+ /* Tell Mm to reuse the MDL */
+ MmPrepareMdlForReuse(Mdl);
- MmPrepareMdlForReuse(Mdl);
-
- ExFreePoolWithTag(Mdl, TAG_MDL);
+ /* Check if this was a pool allocation */
+ if (!(Mdl->MdlFlags & MDL_ALLOCATED_FIXED_SIZE))
+ {
+ /* Free it from the pool */
+ ExFreePoolWithTag(Mdl, TAG_MDL);
+ }
+ else
+ {
+ /* Free it from the lookaside */
+ IopFreeMdlFromLookaside(Mdl, LookasideMdlList);
+ }
}
-
/* EOF */
Propchange: trunk/reactos/ntoskrnl/io/mdl.c
------------------------------------------------------------------------------
--- svn:needs-lock (original)
+++ svn:needs-lock (removed)
@@ -1,1 +1,0 @@
-*