[ion] 22717: - Fix IRP/Completion packet lookaside allocation. We weren't setting a zone size, and I/O Completion packets were also using the wrong size. - Rewrite I/O MDl support to use lookaside lists for allocations below 23 pages (same as on NT). This is is an incredible performance optimization because MDLs are often allocated and de-allocated during I/O operations, and using the lookaside list decreases pool fragmentation and slowdown. - Rewrite IoBuildPartialMdl. It did not work like docu
Author: ion Date: Fri Jun 30 19:59:06 2006 New Revision: 22717
URL: http://svn.reactos.org/svn/reactos?rev=22717&view=rev Log: - Fix IRP/Completion packet lookaside allocation. We weren't setting a zone size, and I/O Completion packets were also using the wrong size. - Rewrite I/O MDl support to use lookaside lists for allocations below 23 pages (same as on NT). This is is an incredible performance optimization because MDLs are often allocated and de-allocated during I/O operations, and using the lookaside list decreases pool fragmentation and slowdown. - Rewrite IoBuildPartialMdl. It did not work like documented in the DDK and also had a bug documented by Microsoft as being in XP.
Modified: trunk/reactos/ntoskrnl/include/internal/io.h trunk/reactos/ntoskrnl/io/iomgr.c trunk/reactos/ntoskrnl/io/mdl.c (contents, props changed)
Modified: trunk/reactos/ntoskrnl/include/internal/io.h URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/include/internal/i... ============================================================================== --- trunk/reactos/ntoskrnl/include/internal/io.h (original) +++ trunk/reactos/ntoskrnl/include/internal/io.h Fri Jun 30 19:59:06 2006 @@ -47,6 +47,14 @@ // Number of partition tables in the Boot Record // #define PARTITION_TBL_SIZE 4 + +// +// We can call the Ob Inlined API, it's the same thing +// +#define IopAllocateMdlFromLookaside \ + ObpAllocateCapturedAttributes +#define IopFreeMdlFromLookaside \ + ObpFreeCapturedAttributes
// // Returns the size of a CM_RESOURCE_LIST @@ -830,4 +838,4 @@ extern POBJECT_TYPE IoCompletionType; extern PDEVICE_NODE IopRootDeviceNode; extern ULONG IopTraceLevel; - +extern NPAGED_LOOKASIDE_LIST IopMdlLookasideList;
Modified: trunk/reactos/ntoskrnl/io/iomgr.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/io/iomgr.c?rev=227... ============================================================================== --- trunk/reactos/ntoskrnl/io/iomgr.c (original) +++ trunk/reactos/ntoskrnl/io/iomgr.c Fri Jun 30 19:59:06 2006 @@ -41,6 +41,7 @@ extern POBJECT_TYPE IoAdapterObjectType; NPAGED_LOOKASIDE_LIST IoLargeIrpLookaside; NPAGED_LOOKASIDE_LIST IoSmallIrpLookaside; +NPAGED_LOOKASIDE_LIST IopMdlLookasideList;
VOID INIT_FUNCTION IopInitLookasideLists(VOID);
@@ -74,7 +75,7 @@ INIT_FUNCTION IopInitLookasideLists(VOID) { - ULONG LargeIrpSize, SmallIrpSize; + ULONG LargeIrpSize, SmallIrpSize, MdlSize; LONG i; PKPRCB Prcb; PNPAGED_LOOKASIDE_LIST CurrentList = NULL; @@ -82,6 +83,7 @@ /* Calculate the sizes */ LargeIrpSize = sizeof(IRP) + (8 * sizeof(IO_STACK_LOCATION)); SmallIrpSize = sizeof(IRP) + sizeof(IO_STACK_LOCATION); + MdlSize = sizeof(MDL) + (23 * sizeof(PFN_NUMBER));
/* Initialize the Lookaside List for Large IRPs */ ExInitializeNPagedLookasideList(&IoLargeIrpLookaside, @@ -90,7 +92,7 @@ 0, LargeIrpSize, IO_LARGEIRP, - 0); + 64);
/* Initialize the Lookaside List for Small IRPs */ ExInitializeNPagedLookasideList(&IoSmallIrpLookaside, @@ -99,7 +101,7 @@ 0, SmallIrpSize, IO_SMALLIRP, - 0); + 32);
/* Initialize the Lookaside List for I\O Completion */ ExInitializeNPagedLookasideList(&IoCompletionPacketLookaside, @@ -108,7 +110,16 @@ 0, sizeof(IO_COMPLETION_PACKET), IOC_TAG1, - 0); + 32); + + /* Initialize the Lookaside List for MDLs */ + ExInitializeNPagedLookasideList(&IopMdlLookasideList, + NULL, + NULL, + 0, + MdlSize, + TAG_MDL, + 128);
/* Now allocate the per-processor lists */ for (i = 0; i < KeNumberProcessors; i++) @@ -131,7 +142,7 @@ 0, LargeIrpSize, IO_LARGEIRP_CPU, - 0); + 64); } else { @@ -146,14 +157,14 @@ IO_SMALLIRP_CPU); if (CurrentList) { - /* Initialize the Lookaside List for Large IRPs */ + /* Initialize the Lookaside List for Small IRPs */ ExInitializeNPagedLookasideList(CurrentList, NULL, NULL, 0, SmallIrpSize, IO_SMALLIRP_CPU, - 0); + 32); } else { @@ -173,15 +184,37 @@ NULL, NULL, 0, + sizeof(IO_COMPLETION_PACKET), + IO_SMALLIRP_CPU, + 32); + } + else + { + CurrentList = &IoCompletionPacketLookaside; + } + Prcb->PPLookasideList[LookasideCompletionList].P = &CurrentList->L; + + /* Set the MDL Completion List */ + Prcb->PPLookasideList[LookasideMdlList].L = &IopMdlLookasideList.L; + CurrentList = ExAllocatePoolWithTag(NonPagedPool, + sizeof(NPAGED_LOOKASIDE_LIST), + TAG_MDL); + if (CurrentList) + { + /* Initialize the Lookaside List for MDLs */ + ExInitializeNPagedLookasideList(CurrentList, + NULL, + NULL, + 0, SmallIrpSize, - IOC_CPU, - 0); + TAG_MDL, + 128); } else { - CurrentList = &IoCompletionPacketLookaside; - } - Prcb->PPLookasideList[LookasideCompletionList].P = &CurrentList->L; + CurrentList = &IopMdlLookasideList; + } + Prcb->PPLookasideList[LookasideMdlList].P = &CurrentList->L; }
DPRINT("Done allocation\n");
Modified: trunk/reactos/ntoskrnl/io/mdl.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/io/mdl.c?rev=22717... ============================================================================== --- trunk/reactos/ntoskrnl/io/mdl.c (original) +++ trunk/reactos/ntoskrnl/io/mdl.c Fri Jun 30 19:59:06 2006 @@ -1,11 +1,9 @@ -/* $Id$ - * - * COPYRIGHT: See COPYING in the top level directory - * PROJECT: ReactOS kernel +/* + * PROJECT: ReactOS Kernel + * LICENSE: GPL - See COPYING in the top level directory * FILE: ntoskrnl/io/mdl.c - * PURPOSE: Io manager mdl functions - * - * PROGRAMMERS: David Welch (welch@mcmail.com) + * PURPOSE: I/O Wrappers for MDL Allocation and Deallocation + * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org) */
/* INCLUDES *****************************************************************/ @@ -20,116 +18,143 @@ * @implemented */ PMDL -STDCALL -IoAllocateMdl(PVOID VirtualAddress, - ULONG Length, - BOOLEAN SecondaryBuffer, - BOOLEAN ChargeQuota, - PIRP Irp) +NTAPI +IoAllocateMdl(IN PVOID VirtualAddress, + IN ULONG Length, + IN BOOLEAN SecondaryBuffer, + IN BOOLEAN ChargeQuota, + IN PIRP Irp) { - PMDL Mdl; + PMDL Mdl = NULL, p; + ULONG Flags = 0; + ULONG Size;
- if (ChargeQuota) - { -// Mdl = ExAllocatePoolWithQuota(NonPagedPool, -// MmSizeOfMdl(VirtualAddress,Length)); - Mdl = ExAllocatePoolWithTag(NonPagedPool, - MmSizeOfMdl(VirtualAddress,Length), - TAG_MDL); - } - else - { - Mdl = ExAllocatePoolWithTag(NonPagedPool, - MmSizeOfMdl(VirtualAddress,Length), - TAG_MDL); - } - MmInitializeMdl(Mdl, (char*)VirtualAddress, Length); + /* Fail if allocation is over 2GB */ + if (Length & 0x80000000) return NULL;
- if (Irp) - { - if (SecondaryBuffer) - { - ASSERT(Irp->MdlAddress); + /* Calculate the number of pages for the allocation */ + Size = ADDRESS_AND_SIZE_TO_SPAN_PAGES(VirtualAddress, Length); + if (Size > 23) + { + /* This is bigger then our fixed-size MDLs. Calculate real size */ + Size *= sizeof(PFN_NUMBER); + Size += sizeof(MDL); + if (Size > MAXUSHORT) return NULL; + } + else + { + /* Use an internal fixed MDL size */ + Size = (23 * sizeof(PFN_NUMBER)) + sizeof(MDL); + Flags |= MDL_ALLOCATED_FIXED_SIZE;
- /* FIXME: add to end of list maybe?? */ - Mdl->Next = Irp->MdlAddress->Next; - Irp->MdlAddress->Next = Mdl; + /* Allocate one from the lookaside list */ + Mdl = IopAllocateMdlFromLookaside(LookasideMdlList); + } + + /* Check if we don't have an mdl yet */ + if (!Mdl) + { + /* Allocate one from pool */ + Mdl = ExAllocatePoolWithTag(NonPagedPool, Size, TAG_MDL); + if (!Mdl) return NULL; + } + + /* Initialize it */ + MmInitializeMdl(Mdl, VirtualAddress, Length); + Mdl->MdlFlags |= Flags; + + /* Check if an IRP was given too */ + if (Irp) + { + /* Check if it came with a secondary buffer */ + if (SecondaryBuffer) + { + /* Insert the MDL at the end */ + p = Irp->MdlAddress; + while (p->Next) p = p->Next; + p->Next = Mdl; } else { - /* - * What if there's allready an mdl at Irp->MdlAddress? - * Is that bad and should we do something about it? - */ - Irp->MdlAddress = Mdl; + /* Otherwise, insert it directly */ + Irp->MdlAddress = Mdl; } }
- return(Mdl); -} - -/* - * @implemented - * - * You must IoFreeMdl the slave before freeing the master. - * - * IoBuildPartialMdl is more similar to MmBuildMdlForNonPagedPool, the difference - * is that the former takes the physical addresses from the master MDL, while the - * latter - from the known location of the NPP. - */ -VOID -STDCALL -IoBuildPartialMdl(PMDL SourceMdl, - PMDL TargetMdl, - PVOID VirtualAddress, - ULONG Length) -{ - PPFN_TYPE TargetPages = (PPFN_TYPE)(TargetMdl + 1); - PPFN_TYPE SourcePages = (PPFN_TYPE)(SourceMdl + 1); - ULONG Count; - ULONG Delta; - - DPRINT("VirtualAddress 0x%p, SourceMdl->StartVa 0x%p, SourceMdl->MappedSystemVa 0x%p\n", - VirtualAddress, SourceMdl->StartVa, SourceMdl->MappedSystemVa); - - TargetMdl->StartVa = (PVOID)PAGE_ROUND_DOWN(VirtualAddress); - TargetMdl->ByteOffset = (ULONG_PTR)VirtualAddress - (ULONG_PTR)TargetMdl->StartVa; - TargetMdl->ByteCount = Length; - TargetMdl->Process = SourceMdl->Process; - Delta = (ULONG_PTR)VirtualAddress - ((ULONG_PTR)SourceMdl->StartVa + SourceMdl->ByteOffset); - TargetMdl->MappedSystemVa = (char*)SourceMdl->MappedSystemVa + Delta; - - TargetMdl->MdlFlags = SourceMdl->MdlFlags & (MDL_IO_PAGE_READ|MDL_SOURCE_IS_NONPAGED_POOL|MDL_MAPPED_TO_SYSTEM_VA); - TargetMdl->MdlFlags |= MDL_PARTIAL; - - Delta = ((ULONG_PTR)TargetMdl->StartVa - (ULONG_PTR)SourceMdl->StartVa) / PAGE_SIZE; - Count = ADDRESS_AND_SIZE_TO_SPAN_PAGES(VirtualAddress,Length); - - SourcePages += Delta; - - DPRINT("Delta %d, Count %d\n", Delta, Count); - - memcpy(TargetPages, SourcePages, Count * sizeof(PFN_TYPE)); - + /* Return the allocated mdl */ + return Mdl; }
/* * @implemented */ -VOID STDCALL +VOID +NTAPI +IoBuildPartialMdl(IN PMDL SourceMdl, + IN PMDL TargetMdl, + IN PVOID VirtualAddress, + IN ULONG Length) +{ + PPFN_TYPE TargetPages = (PPFN_TYPE)(TargetMdl + 1); + PPFN_TYPE SourcePages = (PPFN_TYPE)(SourceMdl + 1); + ULONG Offset; + + /* Calculate the offset */ + Offset = (ULONG)((ULONG_PTR)VirtualAddress - + (ULONG_PTR)SourceMdl->StartVa) - + SourceMdl->ByteOffset; + + /* Check if we don't have a length and calculate it */ + if (!Length) Length = SourceMdl->ByteCount - Offset; + + /* Write the process, start VA and byte data */ + TargetMdl->StartVa = (PVOID)PAGE_ROUND_DOWN(VirtualAddress); + TargetMdl->Process = SourceMdl->Process; + TargetMdl->ByteCount = Length; + TargetMdl->ByteOffset = BYTE_OFFSET(VirtualAddress); + + /* Recalculate the length in pages */ + Length = ADDRESS_AND_SIZE_TO_SPAN_PAGES(VirtualAddress, Length); + + /* Set the MDL Flags */ + TargetMdl->MdlFlags = (MDL_ALLOCATED_FIXED_SIZE | MDL_ALLOCATED_MUST_SUCCEED); + TargetMdl->MdlFlags |= (MDL_IO_PAGE_READ | + MDL_SOURCE_IS_NONPAGED_POOL | + MDL_MAPPED_TO_SYSTEM_VA | + MDL_IO_SPACE); + TargetMdl->MdlFlags |= MDL_PARTIAL; + + /* Set the mapped VA */ + TargetMdl->MappedSystemVa = (PCHAR)SourceMdl->MappedSystemVa + Offset; + + /* Now do the copy */ + Offset = ((ULONG_PTR)TargetMdl->StartVa - (ULONG_PTR)SourceMdl->StartVa) >> + PAGE_SHIFT; + SourcePages += Offset; + RtlMoveMemory(TargetPages, SourcePages, Length * sizeof(PFN_TYPE)); +} + +/* + * @implemented + */ +VOID +NTAPI IoFreeMdl(PMDL Mdl) { - /* - * This unmaps partial mdl's from kernel space but also asserts that non-partial - * mdl's isn't still mapped into kernel space. - */ - ASSERT(Mdl); - ASSERT_IRQL(DISPATCH_LEVEL); + /* Tell Mm to reuse the MDL */ + MmPrepareMdlForReuse(Mdl);
- MmPrepareMdlForReuse(Mdl); - - ExFreePoolWithTag(Mdl, TAG_MDL); + /* Check if this was a pool allocation */ + if (!(Mdl->MdlFlags & MDL_ALLOCATED_FIXED_SIZE)) + { + /* Free it from the pool */ + ExFreePoolWithTag(Mdl, TAG_MDL); + } + else + { + /* Free it from the lookaside */ + IopFreeMdlFromLookaside(Mdl, LookasideMdlList); + } }
- /* EOF */
Propchange: trunk/reactos/ntoskrnl/io/mdl.c ------------------------------------------------------------------------------ --- svn:needs-lock (original) +++ svn:needs-lock (removed) @@ -1,1 +1,0 @@ -*
-
ion＠svn.reactos.org