[ion] 15255: Implement Lookaside List allocation and release for IRPs on Per-CPU Lists, or global system lists if the per-cpu has been exhausted. This should tremendously reduce memory fragmentation and speed up I/o noticeably. Also, don't allocate IRPs with quota charge when they shouldn't.
Implement Lookaside List allocation and release for IRPs on Per-CPU Lists, or global system lists if the per-cpu has been exhausted. This should tremendously reduce memory fragmentation and speed up I/o noticeably. Also, don't allocate IRPs with quota charge when they shouldn't. Modified: trunk/reactos/include/ddk/iodef.h Modified: trunk/reactos/ntoskrnl/io/file.c Modified: trunk/reactos/ntoskrnl/io/irp.c _____
Modified: trunk/reactos/include/ddk/iodef.h --- trunk/reactos/include/ddk/iodef.h 2005-05-13 03:34:13 UTC (rev 15254) +++ trunk/reactos/include/ddk/iodef.h 2005-05-13 04:49:54 UTC (rev 15255) @@ -134,6 +134,11 @@
IRP_RETRY_IO_COMPLETION = 0x4000 };
+#define IRP_QUOTA_CHARGED 0x01 +#define IRP_ALLOCATED_MUST_SUCCEED 0x02 +#define IRP_ALLOCATED_FIXED_SIZE 0x04 +#define IRP_LOOKASIDE_ALLOCATION 0x08 + #define SL_FORCE_ACCESS_CHECK (0x1) #define SL_OPEN_PAGING_FILE (0x2) #define SL_OPEN_TARGET_DIRECTORY (0x4) _____
Modified: trunk/reactos/ntoskrnl/io/file.c --- trunk/reactos/ntoskrnl/io/file.c 2005-05-13 03:34:13 UTC (rev 15254) +++ trunk/reactos/ntoskrnl/io/file.c 2005-05-13 04:49:54 UTC (rev 15255) @@ -210,7 +210,7 @@
KeInitializeEvent(&Event, SynchronizationEvent, FALSE);
/* Allocate an IRP */ - Irp = IoAllocateIrp(DeviceObject->StackSize, TRUE); + Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
/* Set it up */ Irp->UserEvent = &Event; @@ -329,7 +329,7 @@ }
/* Allocate the IRP */ - Irp = IoAllocateIrp(DeviceObject->StackSize, TRUE); + Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
/* Set the IRP */ Irp->Tail.Overlay.OriginalFileObject = FileObject; @@ -498,7 +498,7 @@ KeInitializeEvent(&Event, SynchronizationEvent, FALSE);
/* Allocate an IRP */ - Irp = IoAllocateIrp(DeviceObject->StackSize, TRUE); + Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
/* Set it up */ Irp->UserEvent = &Event; @@ -1149,7 +1149,7 @@ }
/* Allocate the IRP */ - Irp = IoAllocateIrp(DeviceObject->StackSize, TRUE); + Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
/* Set the IRP */ Irp->Tail.Overlay.OriginalFileObject = FileObject; @@ -1586,7 +1586,7 @@ }
/* Allocate the IRP */ - if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, TRUE))) + if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE))) { ObDereferenceObject(FileObject); return STATUS_INSUFFICIENT_RESOURCES; @@ -1699,7 +1699,7 @@ DeviceObject = FileObject->DeviceObject;
- Irp = IoAllocateIrp(DeviceObject->StackSize, TRUE); + Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE); if (Irp==NULL) { ObDereferenceObject(FileObject); @@ -1795,7 +1795,7 @@ DeviceObject = IoGetRelatedDeviceObject(FileObject);
Irp = IoAllocateIrp(DeviceObject->StackSize, - TRUE); + FALSE); if (Irp == NULL) { Status = STATUS_INSUFFICIENT_RESOURCES; @@ -2123,7 +2123,7 @@ }
/* Allocate the IRP */ - if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, TRUE))) + if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE))) { ObDereferenceObject(FileObject); return STATUS_INSUFFICIENT_RESOURCES; @@ -2321,7 +2321,7 @@ }
/* Allocate the IRP */ - if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, TRUE))) + if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE))) { ObDereferenceObject(FileObject); return STATUS_INSUFFICIENT_RESOURCES; @@ -2775,7 +2775,7 @@ }
/* Allocate the IRP */ - if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, TRUE))) + if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE))) { ObDereferenceObject(FileObject); return STATUS_INSUFFICIENT_RESOURCES; @@ -2900,7 +2900,7 @@ DeviceObject = IoGetRelatedDeviceObject(FileObject);
Irp = IoAllocateIrp(DeviceObject->StackSize, - TRUE); + FALSE); if (Irp == NULL) { Status = STATUS_INSUFFICIENT_RESOURCES; _____
Modified: trunk/reactos/ntoskrnl/io/irp.c --- trunk/reactos/ntoskrnl/io/irp.c 2005-05-13 03:34:13 UTC (rev 15254) +++ trunk/reactos/ntoskrnl/io/irp.c 2005-05-13 04:49:54 UTC (rev 15255) @@ -366,31 +366,93 @@
IoAllocateIrp(CCHAR StackSize, BOOLEAN ChargeQuota) { - PIRP Irp; + PIRP Irp = NULL; USHORT Size = IoSizeOfIrp(StackSize); - - /* Check if we shoudl charge quota */ - if (ChargeQuota) + PKPRCB Prcb; + ULONG Flags = 0; + PNPAGED_LOOKASIDE_LIST List; + PP_NPAGED_LOOKASIDE_NUMBER ListType = LookasideSmallIrpList; + + /* Figure out which Lookaside List to use */ + if ((StackSize <= 8) && (ChargeQuota == FALSE)) { - /* Irp = ExAllocatePoolWithQuotaTag(NonPagedPool,IoSizeOfIrp(StackSize), TAG_IRP); */ - /* FIXME */ - Irp = ExAllocatePoolWithTag(NonPagedPool, - Size, - TAG_IRP); + DPRINT("Using lookaside, %d\n", StackSize); + /* Set Fixed Size Flag */ + Flags = IRP_ALLOCATED_FIXED_SIZE; + + /* See if we should use big list */ + if (StackSize != 1) + { + DPRINT("Using large lookaside\n"); + Size = IoSizeOfIrp(8); + ListType = LookasideLargeIrpList; + } + + /* Get the PRCB */ + Prcb = KeGetCurrentPrcb(); + + /* Get the P List First */ + List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].P; + + /* Attempt allocation */ + List->L.TotalAllocates++; + DPRINT("Total allocates: %d\n", List->L.TotalAllocates); + Irp = (PIRP)InterlockedPopEntrySList(&List->L.ListHead); + DPRINT("Alloc attempt on CPU list: %p\n", Irp); + + /* Check if the P List failed */ + if (!Irp) + { + /* Let the balancer know */ + List->L.AllocateMisses++; + DPRINT("Total misses: %d\n", List->L.AllocateMisses); + + /* Try the L List */ + List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].L; + List->L.TotalAllocates++; + Irp = (PIRP)InterlockedPopEntrySList(&List->L.ListHead); + DPRINT("Alloc attempt on SYS list: %p\n", Irp); + } } + + /* Check if we have to use the pool */ + if (!Irp) + { + DPRINT("Using pool\n"); + /* Did we try lookaside and fail? */ + if (Flags & IRP_ALLOCATED_FIXED_SIZE) List->L.AllocateMisses++; + + /* Check if we shoudl charge quota */ + if (ChargeQuota) + { + /* Irp = ExAllocatePoolWithQuotaTag(NonPagedPool, Size, TAG_IRP); */ + /* FIXME */ + Irp = ExAllocatePoolWithTag(NonPagedPool, Size, TAG_IRP); + } + else + { + /* Allocate the IRP With no Quota charge */ + Irp = ExAllocatePoolWithTag(NonPagedPool, Size, TAG_IRP); + } + + /* Make sure it was sucessful */ + if (!Irp) return(NULL); + } else { - /* Allocate the IRP With no Quota charge */ - Irp = ExAllocatePoolWithTag(NonPagedPool, - Size, - TAG_IRP); + /* We have an IRP from Lookaside */ + Flags |= IRP_LOOKASIDE_ALLOCATION; } - - /* Make sure it was sucessful */ - if (Irp==NULL) return(NULL); - + + /* Set Flag */ + if (ChargeQuota) Flags |= IRP_QUOTA_CHARGED; + /* Now Initialize it */ + DPRINT("irp allocated\n"); IoInitializeIrp(Irp, Size, StackSize); + + /* Set the Allocation Flags */ + Irp->AllocationFlags = Flags;
/* Return it */ return Irp; @@ -430,7 +492,7 @@ StartingOffset,IoStatusBlock);
/* Allocate IRP */ - if (!(Irp = IoAllocateIrp(DeviceObject->StackSize,TRUE))) return Irp; + if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE))) return Irp;
/* Get the Stack */ StackPtr = IoGetNextIrpStackLocation(Irp); @@ -580,7 +642,7 @@ InternalDeviceIoControl,Event,IoStatusBlock);
/* Allocate IRP */ - if (!(Irp = IoAllocateIrp(DeviceObject->StackSize,TRUE))) return Irp; + if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE))) return Irp;
/* Get the Stack */ StackPtr = IoGetNextIrpStackLocation(Irp); @@ -1177,8 +1239,59 @@ STDCALL IoFreeIrp(PIRP Irp) { - /* Free the pool memory associated with it */ - ExFreePoolWithTag(Irp, TAG_IRP); + PNPAGED_LOOKASIDE_LIST List; + PP_NPAGED_LOOKASIDE_NUMBER ListType = LookasideSmallIrpList; + PKPRCB Prcb; + + /* If this was a pool alloc, free it with the pool */ + if (!(Irp->AllocationFlags & IRP_ALLOCATED_FIXED_SIZE)) + { + /* Free it */ + DPRINT("Freeing pool IRP\n"); + ExFreePool(Irp); + } + else + { + DPRINT("Freeing Lookaside IRP\n"); + + /* Check if this was a Big IRP */ + if (Irp->StackCount != 1) + { + DPRINT("Large IRP\n"); + ListType = LookasideLargeIrpList; + } + + /* Get the PRCB */ + Prcb = KeGetCurrentPrcb(); + + /* Use the P List */ + List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].P; + List->L.TotalFrees++; + + /* Check if the Free was within the Depth or not */ + if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth) + { + /* Let the balancer know */ + List->L.FreeMisses++; + + /* Use the L List */ + List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].L; + List->L.TotalFrees++; + + /* Check if the Free was within the Depth or not */ + if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth) + { + /* All lists failed, use the pool */ + List->L.FreeMisses++; + ExFreePool(Irp); + } + } + + /* The free was within dhe Depth */ + InterlockedPushEntrySList(&List->L.ListHead, (PSINGLE_LIST_ENTRY)Irp); + } + + DPRINT("Free done\n"); }
/* @@ -1324,7 +1437,7 @@ PIRP AssocIrp;
/* Allocate the IRP */ - AssocIrp = IoAllocateIrp(StackSize,FALSE); + AssocIrp = IoAllocateIrp(StackSize, FALSE); if (AssocIrp == NULL) return NULL;
/* Set the Flags */
-
ion＠svn.reactos.com