[fireball] 37191: - Use ExInitializeSystemLookasideList instead of ExInitializeNPagedLookasideList for the internal I/O lookaside lists (just as it was done for the Ob lists a couple of months ago). - Optimize lookaside allocation by using one large contiguous buffer instead of fragmented buffers for each CPU. - Use NT structure names instead of ReactOS-only structures. - Fixes some memory corruption issues when doing I/O completion (found by Stefan and winetests).
Author: fireball Date: Tue Nov 4 15:55:55 2008 New Revision: 37191
URL: http://svn.reactos.org/svn/reactos?rev=37191&view=rev Log: - Use ExInitializeSystemLookasideList instead of ExInitializeNPagedLookasideList for the internal I/O lookaside lists (just as it was done for the Ob lists a couple of months ago). - Optimize lookaside allocation by using one large contiguous buffer instead of fragmented buffers for each CPU. - Use NT structure names instead of ReactOS-only structures. - Fixes some memory corruption issues when doing I/O completion (found by Stefan and winetests).
Modified: trunk/reactos/ntoskrnl/include/internal/io.h trunk/reactos/ntoskrnl/io/iomgr/iocomp.c trunk/reactos/ntoskrnl/io/iomgr/iomgr.c trunk/reactos/ntoskrnl/io/iomgr/irp.c
Modified: trunk/reactos/ntoskrnl/include/internal/io.h URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/include/internal/i... ============================================================================== --- trunk/reactos/ntoskrnl/include/internal/io.h [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/include/internal/io.h [iso-8859-1] Tue Nov 4 15:55:55 2008 @@ -81,12 +81,6 @@ #define RD_SYMLINK_CREATE_FAILED 5
// -// Packet Types when piggybacking on the IRP Overlay -// -#define IrpCompletionPacket 0x1 -#define IrpMiniCompletionPacket 0x2 - -// // We can call the Ob Inlined API, it's the same thing // #define IopAllocateMdlFromLookaside \ @@ -249,10 +243,20 @@ } IOP_TRANSFER_TYPE, *PIOP_TRANSFER_TYPE;
// +// Packet Types when piggybacking on the IRP Overlay +// +typedef enum _COMPLETION_PACKET_TYPE + { + IopCompletionPacketIrp, + IopCompletionPacketMini, + IopCompletionPacketQuota +} COMPLETION_PACKET_TYPE, *PCOMPLETION_PACKET_TYPE; + +// // Special version of the IRP Overlay used to optimize I/O completion // by not using up a separate structure. // -typedef struct _IO_COMPLETION_PACKET +typedef struct _IOP_MINI_COMPLETION_PACKET { struct { @@ -263,10 +267,11 @@ ULONG PacketType; }; }; - PVOID Key; - PVOID Context; - IO_STATUS_BLOCK IoStatus; -} IO_COMPLETION_PACKET, *PIO_COMPLETION_PACKET; + PVOID KeyContext; + PVOID ApcContext; + NTSTATUS IoStatus; + ULONG_PTR IoStatusInformation; +} IOP_MINI_COMPLETION_PACKET, *PIOP_MINI_COMPLETION_PACKET;
// // I/O Completion Context for IoSetIoCompletionRoutineEx @@ -1036,7 +1041,7 @@ extern POBJECT_TYPE IoCompletionType; extern PDEVICE_NODE IopRootDeviceNode; extern ULONG IopTraceLevel; -extern NPAGED_LOOKASIDE_LIST IopMdlLookasideList; +extern GENERAL_LOOKASIDE IopMdlLookasideList; extern GENERIC_MAPPING IopCompletionMapping; extern GENERIC_MAPPING IopFileMapping; extern POBJECT_TYPE _IoFileObjectType;
Modified: trunk/reactos/ntoskrnl/io/iomgr/iocomp.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/io/iomgr/iocomp.c?... ============================================================================== --- trunk/reactos/ntoskrnl/io/iomgr/iocomp.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/io/iomgr/iocomp.c [iso-8859-1] Tue Nov 4 15:55:55 2008 @@ -15,7 +15,7 @@
POBJECT_TYPE IoCompletionType;
-NPAGED_LOOKASIDE_LIST IoCompletionPacketLookaside; +GENERAL_LOOKASIDE IoCompletionPacketLookaside;
GENERIC_MAPPING IopCompletionMapping = { @@ -40,8 +40,7 @@ IN PVOID Context) { NTSTATUS Status; - PIO_UNLOAD_SAFE_COMPLETION_CONTEXT UnsafeContext = - (PIO_UNLOAD_SAFE_COMPLETION_CONTEXT)Context; + PIO_UNLOAD_SAFE_COMPLETION_CONTEXT UnsafeContext = Context;
/* Reference the device object */ ObReferenceObject(UnsafeContext->DeviceObject); @@ -61,7 +60,7 @@
VOID NTAPI -IopFreeIoCompletionPacket(PIO_COMPLETION_PACKET Packet) +IopFreeMiniPacket(PIOP_MINI_COMPLETION_PACKET Packet) { PKPRCB Prcb = KeGetCurrentPrcb(); PNPAGED_LOOKASIDE_LIST List; @@ -104,7 +103,7 @@ PLIST_ENTRY FirstEntry; PLIST_ENTRY CurrentEntry; PIRP Irp; - PIO_COMPLETION_PACKET Packet; + PIOP_MINI_COMPLETION_PACKET Packet;
/* Rundown the Queue */ FirstEntry = KeRundownQueue(Queue); @@ -116,14 +115,14 @@ { /* Get the Packet */ Packet = CONTAINING_RECORD(CurrentEntry, - IO_COMPLETION_PACKET, + IOP_MINI_COMPLETION_PACKET, ListEntry);
/* Go to next Entry */ CurrentEntry = CurrentEntry->Flink;
/* Check if it's part of an IRP, or a separate packet */ - if (Packet->PacketType == IrpCompletionPacket) + if (Packet->PacketType == IopCompletionPacketIrp) { /* Get the IRP and free it */ Irp = CONTAINING_RECORD(Packet, IRP, Tail.Overlay.ListEntry); @@ -132,7 +131,7 @@ else { /* Use common routine */ - IopFreeIoCompletionPacket(Packet); + IopFreeMiniPacket(Packet); } } while (FirstEntry != CurrentEntry); } @@ -155,7 +154,7 @@ PKQUEUE Queue = (PKQUEUE)IoCompletion; PNPAGED_LOOKASIDE_LIST List; PKPRCB Prcb = KeGetCurrentPrcb(); - PIO_COMPLETION_PACKET Packet; + PIOP_MINI_COMPLETION_PACKET Packet;
/* Get the P List */ List = (PNPAGED_LOOKASIDE_LIST)Prcb-> @@ -194,11 +193,11 @@ if (Packet) { /* Set up the Packet */ - Packet->PacketType = IrpMiniCompletionPacket; - Packet->Key = KeyContext; - Packet->Context = ApcContext; - Packet->IoStatus.Status = IoStatus; - Packet->IoStatus.Information = IoStatusInformation; + Packet->PacketType = IopCompletionPacketMini; + Packet->KeyContext = KeyContext; + Packet->ApcContext = ApcContext; + Packet->IoStatus = IoStatus; + Packet->IoStatusInformation = IoStatusInformation;
/* Insert the Queue */ KeInsertQueue(Queue, &Packet->ListEntry); @@ -455,7 +454,7 @@ { LARGE_INTEGER SafeTimeout; PKQUEUE Queue; - PIO_COMPLETION_PACKET Packet; + PIOP_MINI_COMPLETION_PACKET Packet; PLIST_ENTRY ListEntry; KPROCESSOR_MODE PreviousMode = ExGetPreviousMode(); NTSTATUS Status = STATUS_SUCCESS; @@ -517,11 +516,11 @@ { /* Get the Packet Data */ Packet = CONTAINING_RECORD(ListEntry, - IO_COMPLETION_PACKET, + IOP_MINI_COMPLETION_PACKET, ListEntry);
/* Check if this is piggybacked on an IRP */ - if (Packet->PacketType == IrpCompletionPacket) + if (Packet->PacketType == IopCompletionPacketIrp) { /* Get the IRP */ Irp = CONTAINING_RECORD(ListEntry, @@ -539,12 +538,13 @@ else { /* Save values */ - Key = Packet->Key; - Apc = Packet->Context; - IoStatus = Packet->IoStatus; + Key = Packet->KeyContext; + Apc = Packet->ApcContext; + IoStatus.Status = Packet->IoStatus; + IoStatus.Information = Packet->IoStatusInformation;
/* Free the packet */ - IopFreeIoCompletionPacket(Packet); + IopFreeMiniPacket(Packet); }
/* Enter SEH to write back the values */
Modified: trunk/reactos/ntoskrnl/io/iomgr/iomgr.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/io/iomgr/iomgr.c?r... ============================================================================== --- trunk/reactos/ntoskrnl/io/iomgr/iomgr.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/io/iomgr/iomgr.c [iso-8859-1] Tue Nov 4 15:55:55 2008 @@ -49,7 +49,6 @@ extern LIST_ENTRY ShutdownListHead; extern LIST_ENTRY LastChanceShutdownListHead; extern KSPIN_LOCK ShutdownListLock; -extern NPAGED_LOOKASIDE_LIST IoCompletionPacketLookaside; extern POBJECT_TYPE IoAdapterObjectType; ERESOURCE IopDatabaseResource; extern ERESOURCE FileSystemListLock; @@ -76,9 +75,10 @@
extern PDEVICE_OBJECT IopErrorLogObject;
-NPAGED_LOOKASIDE_LIST IoLargeIrpLookaside; -NPAGED_LOOKASIDE_LIST IoSmallIrpLookaside; -NPAGED_LOOKASIDE_LIST IopMdlLookasideList; +GENERAL_LOOKASIDE IoLargeIrpLookaside; +GENERAL_LOOKASIDE IoSmallIrpLookaside; +GENERAL_LOOKASIDE IopMdlLookasideList; +extern GENERAL_LOOKASIDE IoCompletionPacketLookaside;
#if defined (ALLOC_PRAGMA) #pragma alloc_text(INIT, IoInitSystem) @@ -94,143 +94,142 @@ ULONG LargeIrpSize, SmallIrpSize, MdlSize; LONG i; PKPRCB Prcb; - PNPAGED_LOOKASIDE_LIST CurrentList = NULL; + PGENERAL_LOOKASIDE CurrentList = NULL;
/* Calculate the sizes */ LargeIrpSize = sizeof(IRP) + (8 * sizeof(IO_STACK_LOCATION)); SmallIrpSize = sizeof(IRP) + sizeof(IO_STACK_LOCATION); MdlSize = sizeof(MDL) + (23 * sizeof(PFN_NUMBER));
+ /* Initialize the Lookaside List for I\O Completion */ + ExInitializeSystemLookasideList(&IoCompletionPacketLookaside, + NonPagedPool, + sizeof(IOP_MINI_COMPLETION_PACKET), + IOC_TAG1, + 32, + &ExSystemLookasideListHead); + /* Initialize the Lookaside List for Large IRPs */ - ExInitializeNPagedLookasideList(&IoLargeIrpLookaside, - NULL, - NULL, - 0, + ExInitializeSystemLookasideList(&IoLargeIrpLookaside, + NonPagedPool, LargeIrpSize, IO_LARGEIRP, - 64); + 64, + &ExSystemLookasideListHead); +
/* Initialize the Lookaside List for Small IRPs */ - ExInitializeNPagedLookasideList(&IoSmallIrpLookaside, - NULL, - NULL, - 0, + ExInitializeSystemLookasideList(&IoSmallIrpLookaside, + NonPagedPool, SmallIrpSize, IO_SMALLIRP, - 32); - - /* Initialize the Lookaside List for I\O Completion */ - ExInitializeNPagedLookasideList(&IoCompletionPacketLookaside, - NULL, - NULL, - 0, - sizeof(IO_COMPLETION_PACKET), - IOC_TAG1, - 32); + 32, + &ExSystemLookasideListHead);
/* Initialize the Lookaside List for MDLs */ - ExInitializeNPagedLookasideList(&IopMdlLookasideList, - NULL, - NULL, - 0, + ExInitializeSystemLookasideList(&IopMdlLookasideList, + NonPagedPool, MdlSize, TAG_MDL, - 128); - - /* Now allocate the per-processor lists */ + 128, + &ExSystemLookasideListHead); + + /* Allocate the global lookaside list buffer */ + CurrentList = ExAllocatePoolWithTag(NonPagedPool, + 4 * KeNumberProcessors * + sizeof(GENERAL_LOOKASIDE), + TAG_IO); + + /* Loop all processors */ for (i = 0; i < KeNumberProcessors; i++) { /* Get the PRCB for this CPU */ Prcb = KiProcessorBlock[i]; DPRINT("Setting up lookaside for CPU: %x, PRCB: %p\n", i, Prcb);
+ /* Write IRP credit limit */ + Prcb->LookasideIrpFloat = 512 / KeNumberProcessors; + + /* Set the I/O Completion List */ + Prcb->PPLookasideList[LookasideCompletionList].L = &IoCompletionPacketLookaside; + if (CurrentList) + { + /* Initialize the Lookaside List for mini-packets */ + ExInitializeSystemLookasideList(CurrentList, + NonPagedPool, + sizeof(IOP_MINI_COMPLETION_PACKET), + IO_SMALLIRP_CPU, + 32, + &ExSystemLookasideListHead); + Prcb->PPLookasideList[LookasideCompletionList].P = CurrentList; + CurrentList++; + + } + else + { + Prcb->PPLookasideList[LookasideCompletionList].P = &IoCompletionPacketLookaside; + } + /* Set the Large IRP List */ - Prcb->PPLookasideList[LookasideLargeIrpList].L = &IoLargeIrpLookaside.L; - CurrentList = ExAllocatePoolWithTag(NonPagedPool, - sizeof(NPAGED_LOOKASIDE_LIST), - IO_LARGEIRP_CPU); + Prcb->PPLookasideList[LookasideLargeIrpList].L = &IoLargeIrpLookaside; if (CurrentList) { /* Initialize the Lookaside List for Large IRPs */ - ExInitializeNPagedLookasideList(CurrentList, - NULL, - NULL, - 0, + ExInitializeSystemLookasideList(CurrentList, + NonPagedPool, LargeIrpSize, IO_LARGEIRP_CPU, - 64); + 64, + &ExSystemLookasideListHead); + Prcb->PPLookasideList[LookasideLargeIrpList].P = CurrentList; + CurrentList++; + } else { - CurrentList = &IoLargeIrpLookaside; - } - Prcb->PPLookasideList[LookasideLargeIrpList].P = &CurrentList->L; + Prcb->PPLookasideList[LookasideLargeIrpList].P = &IoLargeIrpLookaside; + }
/* Set the Small IRP List */ - Prcb->PPLookasideList[LookasideSmallIrpList].L = &IoSmallIrpLookaside.L; - CurrentList = ExAllocatePoolWithTag(NonPagedPool, - sizeof(NPAGED_LOOKASIDE_LIST), - IO_SMALLIRP_CPU); + Prcb->PPLookasideList[LookasideSmallIrpList].L = &IoSmallIrpLookaside; if (CurrentList) { /* Initialize the Lookaside List for Small IRPs */ - ExInitializeNPagedLookasideList(CurrentList, - NULL, - NULL, - 0, + ExInitializeSystemLookasideList(CurrentList, + NonPagedPool, SmallIrpSize, IO_SMALLIRP_CPU, - 32); + 32, + &ExSystemLookasideListHead); + Prcb->PPLookasideList[LookasideSmallIrpList].P = CurrentList; + CurrentList++; + } else { - CurrentList = &IoSmallIrpLookaside; - } - Prcb->PPLookasideList[LookasideSmallIrpList].P = &CurrentList->L; - - /* Set the I/O Completion List */ - Prcb->PPLookasideList[LookasideCompletionList].L = &IoCompletionPacketLookaside.L; - CurrentList = ExAllocatePoolWithTag(NonPagedPool, - sizeof(NPAGED_LOOKASIDE_LIST), - IO_SMALLIRP_CPU); + Prcb->PPLookasideList[LookasideSmallIrpList].P = &IoSmallIrpLookaside; + } + + /* Set the MDL Completion List */ + Prcb->PPLookasideList[LookasideMdlList].L = &IopMdlLookasideList; if (CurrentList) { - /* Initialize the Lookaside List for Large IRPs */ - ExInitializeNPagedLookasideList(CurrentList, - NULL, - NULL, - 0, - sizeof(IO_COMPLETION_PACKET), - IO_SMALLIRP_CPU, - 32); - } - else - { - CurrentList = &IoCompletionPacketLookaside; - } - Prcb->PPLookasideList[LookasideCompletionList].P = &CurrentList->L; - - /* Set the MDL Completion List */ - Prcb->PPLookasideList[LookasideMdlList].L = &IopMdlLookasideList.L; - CurrentList = ExAllocatePoolWithTag(NonPagedPool, - sizeof(NPAGED_LOOKASIDE_LIST), - TAG_MDL); - if (CurrentList) - { /* Initialize the Lookaside List for MDLs */ - ExInitializeNPagedLookasideList(CurrentList, - NULL, - NULL, - 0, + ExInitializeSystemLookasideList(CurrentList, + NonPagedPool, SmallIrpSize, TAG_MDL, - 128); + 128, + &ExSystemLookasideListHead); + + Prcb->PPLookasideList[LookasideMdlList].P = CurrentList; + CurrentList++; + } else { - CurrentList = &IopMdlLookasideList; - } - Prcb->PPLookasideList[LookasideMdlList].P = &CurrentList->L; + Prcb->PPLookasideList[LookasideMdlList].P = &IopMdlLookasideList; + } } }
Modified: trunk/reactos/ntoskrnl/io/iomgr/irp.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/io/iomgr/irp.c?rev... ============================================================================== --- trunk/reactos/ntoskrnl/io/iomgr/irp.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/io/iomgr/irp.c [iso-8859-1] Tue Nov 4 15:55:55 2008 @@ -419,7 +419,7 @@ { /* We have an I/O Completion setup... create the special Overlay */ Irp->Tail.CompletionKey = Key; - Irp->Tail.Overlay.PacketType = IrpCompletionPacket; + Irp->Tail.Overlay.PacketType = IopCompletionPacketIrp; KeInsertQueue(Port, &Irp->Tail.Overlay.ListEntry); } else
-
fireball＠svn.reactos.org