[ion] 15256: Fix IRP structure (add missing member) and optimize io completion by allowing the io completion packet to be piggybacked on the IRP. Also make io completion use the cpu lookaside lists now as well as the paged pool as a last-chance attempt
Fix IRP structure (add missing member) and optimize io completion by allowing the io completion packet to be piggybacked on the IRP. Also make io completion use the cpu lookaside lists now as well as the paged pool as a last-chance attempt Modified: trunk/reactos/include/ddk/iotypes.h Modified: trunk/reactos/ntoskrnl/include/internal/io.h Modified: trunk/reactos/ntoskrnl/io/iocomp.c Modified: trunk/reactos/ntoskrnl/io/irp.c _____
Modified: trunk/reactos/include/ddk/iotypes.h --- trunk/reactos/include/ddk/iotypes.h 2005-05-13 04:49:54 UTC (rev 15255) +++ trunk/reactos/include/ddk/iotypes.h 2005-05-13 05:41:24 UTC (rev 15256) @@ -794,12 +794,17 @@
}; struct _ETHREAD* Thread; PCHAR AuxiliaryBuffer; - LIST_ENTRY ListEntry; - struct _IO_STACK_LOCATION* CurrentStackLocation; + struct { + LIST_ENTRY ListEntry; + union { + struct _IO_STACK_LOCATION* CurrentStackLocation; + ULONG PacketType; + }; + }; PFILE_OBJECT OriginalFileObject; } Overlay; KAPC Apc; - ULONG CompletionKey; + PVOID CompletionKey; } Tail; } IRP, *PIRP;
_____
Modified: trunk/reactos/ntoskrnl/include/internal/io.h --- trunk/reactos/ntoskrnl/include/internal/io.h 2005-05-13 04:49:54 UTC (rev 15255) +++ trunk/reactos/ntoskrnl/include/internal/io.h 2005-05-13 05:41:24 UTC (rev 15256) @@ -43,13 +43,25 @@
struct _DEVICE_OBJECT_POWER_EXTENSION;
-typedef struct _IO_COMPLETION_PACKET{ +/* This is like the IRP Overlay so we can optimize its insertion */ +typedef struct _IO_COMPLETION_PACKET +{ + struct { + LIST_ENTRY ListEntry; + union { + struct _IO_STACK_LOCATION *CurrentStackLocation; + ULONG PacketType; + }; + }; PVOID Key; PVOID Context; IO_STATUS_BLOCK IoStatus; - LIST_ENTRY ListEntry; } IO_COMPLETION_PACKET, *PIO_COMPLETION_PACKET;
+/* Packet Types */ +#define IrpCompletionPacket 0x1 +#define IrpMiniCompletionPacket 0x2 + typedef struct _DEVOBJ_EXTENSION { CSHORT Type; USHORT Size; _____
Modified: trunk/reactos/ntoskrnl/io/iocomp.c --- trunk/reactos/ntoskrnl/io/iocomp.c 2005-05-13 04:49:54 UTC (rev 15255) +++ trunk/reactos/ntoskrnl/io/iocomp.c 2005-05-13 05:41:24 UTC (rev 15256) @@ -37,11 +37,46 @@
VOID STDCALL +IopFreeIoCompletionPacket(PIO_COMPLETION_PACKET Packet) +{ + PKPRCB Prcb = KeGetCurrentPrcb(); + PNPAGED_LOOKASIDE_LIST List; + + /* Use the P List */ + List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[LookasideCompletionList].P ; + List->L.TotalFrees++; + + /* Check if the Free was within the Depth or not */ + if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth) + { + /* Let the balancer know */ + List->L.FreeMisses++; + + /* Use the L List */ + List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[LookasideCompletionList].L ; + List->L.TotalFrees++; + + /* Check if the Free was within the Depth or not */ + if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth) + { + /* All lists failed, use the pool */ + List->L.FreeMisses++; + ExFreePool(Packet); + } + } + + /* The free was within dhe Depth */ + InterlockedPushEntrySList(&List->L.ListHead, (PSINGLE_LIST_ENTRY)Packet); +} + +VOID +STDCALL IopDeleteIoCompletion(PVOID ObjectBody) { PKQUEUE Queue = ObjectBody; PLIST_ENTRY FirstEntry; PLIST_ENTRY CurrentEntry; + PIRP Irp; PIO_COMPLETION_PACKET Packet;
DPRINT("IopDeleteIoCompletion()\n"); @@ -61,8 +96,18 @@ /* Go to next Entry */ CurrentEntry = CurrentEntry->Flink;
- /* Free it */ - ExFreeToNPagedLookasideList(&IoCompletionPacketLookaside, Packet); + /* Check if it's part of an IRP, or a separate packet */ + if (Packet->PacketType == IrpCompletionPacket) + { + /* Get the IRP and free it */ + Irp = CONTAINING_RECORD(Packet, IRP, Tail.Overlay.ListEntry); + IoFreeIrp(Irp); + } + else + { + /* Use common routine */ + IopFreeIoCompletionPacket(Packet); + } } while (FirstEntry != CurrentEntry); } } @@ -80,21 +125,59 @@ IN BOOLEAN Quota) { PKQUEUE Queue = (PKQUEUE)IoCompletion; + PNPAGED_LOOKASIDE_LIST List; + PKPRCB Prcb = KeGetCurrentPrcb(); PIO_COMPLETION_PACKET Packet;
- /* Allocate the Packet */ - Packet = ExAllocateFromNPagedLookasideList(&IoCompletionPacketLookaside); - if (NULL == Packet) return STATUS_NO_MEMORY; + /* Get the P List */ + List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[LookasideCompletionList].P ; + + /* Try to allocate the Packet */ + List->L.TotalAllocates++; + Packet = (PVOID)InterlockedPopEntrySList(&List->L.ListHead); + + /* Check if that failed, use the L list if it did */ + if (!Packet) + { + /* Let the balancer know */ + List->L.AllocateMisses++; + + /* Get L List */ + List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[LookasideCompletionList].L ; + + /* Try to allocate the Packet */ + List->L.TotalAllocates++; + Packet = (PVOID)InterlockedPopEntrySList(&List->L.ListHead); + } + + /* Still failed, use pool */ + if (!Packet) + { + /* Let the balancer know */ + List->L.AllocateMisses++; + + /* Allocate from Nonpaged Pool */ + Packet = ExAllocatePoolWithTag(NonPagedPool, sizeof(*Packet), IOC_TAG); + } + + /* Make sure we have one by now... */ + if (Packet) + { + /* Set up the Packet */ + Packet->PacketType = IrpMiniCompletionPacket; + Packet->Key = KeyContext; + Packet->Context = ApcContext; + Packet->IoStatus.Status = IoStatus; + Packet->IoStatus.Information = IoStatusInformation;
- /* Set up the Packet */ - Packet->Key = KeyContext; - Packet->Context = ApcContext; - Packet->IoStatus.Status = IoStatus; - Packet->IoStatus.Information = IoStatusInformation; + /* Insert the Queue */ + KeInsertQueue(Queue, &Packet->ListEntry); + } + else + { + return STATUS_INSUFFICIENT_RESOURCES; + }
- /* Insert the Queue */ - KeInsertQueue(Queue, &Packet->ListEntry); - /* Return Success */ return STATUS_SUCCESS; } @@ -410,10 +493,27 @@
_SEH_TRY {
- /* Return it */ - *CompletionKey = Packet->Key; - *CompletionContext = Packet->Context; - *IoStatusBlock = Packet->IoStatus; + /* Check if this is piggybacked on an IRP */ + if (Packet->PacketType == IrpCompletionPacket) + { + /* Get the IRP */ + PIRP Irp = NULL; + Irp = CONTAINING_RECORD(ListEntry, IRP, Tail.Overlay.ListEntry); + + /* Return values to user */ + *CompletionKey = Irp->Tail.CompletionKey; + *CompletionContext = Irp->Overlay.AsynchronousParameters.UserApcContext; + *IoStatusBlock = Packet->IoStatus; + IoFreeIrp(Irp); + } + else + { + /* This is a user-mode generated or API generated mini-packet */ + *CompletionKey = Packet->Key; + *CompletionContext = Packet->Context; + *IoStatusBlock = Packet->IoStatus; + IopFreeIoCompletionPacket(Packet); + }
} _SEH_HANDLE {
_____
Modified: trunk/reactos/ntoskrnl/io/irp.c --- trunk/reactos/ntoskrnl/io/irp.c 2005-05-13 04:49:54 UTC (rev 15255) +++ trunk/reactos/ntoskrnl/io/irp.c 2005-05-13 05:41:24 UTC (rev 15256) @@ -187,12 +187,10 @@
else if (FileObject && FileObject->CompletionContext) { /* Call the IO Completion Port if we have one, instead */ - IoSetIoCompletion(FileObject->CompletionContext->Port, - FileObject->CompletionContext->Key, - Irp->Overlay.AsynchronousParameters.UserApcContext, - Irp->IoStatus.Status, - Irp->IoStatus.Information, - FALSE); + Irp->Tail.CompletionKey = FileObject->CompletionContext->Key; + Irp->Tail.Overlay.PacketType = IrpCompletionPacket; + KeInsertQueue(FileObject->CompletionContext->Port, + &Irp->Tail.Overlay.ListEntry); Irp = NULL; } } @@ -369,7 +367,7 @@ PIRP Irp = NULL; USHORT Size = IoSizeOfIrp(StackSize); PKPRCB Prcb; - ULONG Flags = 0; + UCHAR Flags = 0; PNPAGED_LOOKASIDE_LIST List; PP_NPAGED_LOOKASIDE_NUMBER ListType = LookasideSmallIrpList;
-
ion＠svn.reactos.com