[ros-diffs] [ion] 15255: Implement Lookaside List allocation and release for IRPs on Per-CPU Lists, or global system lists if the per-cpu has been exhausted. This should tremendously reduce memory fragmentation and speed up I/o noticeably. Also, don't allocate IRPs with quota charge when they shouldn't.

Implement Lookaside List allocation and release for IRPs on Per-CPU
Lists, or global system lists if the per-cpu has been exhausted. This
should tremendously reduce memory fragmentation and speed up I/o
noticeably. Also, don't allocate IRPs with quota charge when they
shouldn't.
Modified: trunk/reactos/include/ddk/iodef.h
Modified: trunk/reactos/ntoskrnl/io/file.c
Modified: trunk/reactos/ntoskrnl/io/irp.c
  _____  

Modified: trunk/reactos/include/ddk/iodef.h
--- trunk/reactos/include/ddk/iodef.h	2005-05-13 03:34:13 UTC (rev
15254)
+++ trunk/reactos/include/ddk/iodef.h	2005-05-13 04:49:54 UTC (rev
15255)
@@ -134,6 +134,11 @@

    IRP_RETRY_IO_COMPLETION = 0x4000
 };
 
+#define IRP_QUOTA_CHARGED           0x01
+#define IRP_ALLOCATED_MUST_SUCCEED  0x02
+#define IRP_ALLOCATED_FIXED_SIZE    0x04
+#define IRP_LOOKASIDE_ALLOCATION    0x08
+
 #define SL_FORCE_ACCESS_CHECK      (0x1)
 #define SL_OPEN_PAGING_FILE        (0x2)
 #define SL_OPEN_TARGET_DIRECTORY   (0x4)
  _____  

Modified: trunk/reactos/ntoskrnl/io/file.c
--- trunk/reactos/ntoskrnl/io/file.c	2005-05-13 03:34:13 UTC (rev
15254)
+++ trunk/reactos/ntoskrnl/io/file.c	2005-05-13 04:49:54 UTC (rev
15255)
@@ -210,7 +210,7 @@

         KeInitializeEvent(&Event, SynchronizationEvent, FALSE);
 
         /* Allocate an IRP */
-        Irp = IoAllocateIrp(DeviceObject->StackSize, TRUE);
+        Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
 
         /* Set it up */
         Irp->UserEvent = &Event;
@@ -329,7 +329,7 @@
     }
 
     /* Allocate the IRP */
-    Irp = IoAllocateIrp(DeviceObject->StackSize, TRUE);
+    Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
 
     /* Set the IRP */
     Irp->Tail.Overlay.OriginalFileObject = FileObject;
@@ -498,7 +498,7 @@
     KeInitializeEvent(&Event, SynchronizationEvent, FALSE);
 
     /* Allocate an IRP */
-    Irp = IoAllocateIrp(DeviceObject->StackSize, TRUE);
+    Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
 
     /* Set it up */
     Irp->UserEvent = &Event;
@@ -1149,7 +1149,7 @@
     }
 
     /* Allocate the IRP */
-    Irp = IoAllocateIrp(DeviceObject->StackSize, TRUE);
+    Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
 
     /* Set the IRP */
     Irp->Tail.Overlay.OriginalFileObject = FileObject;
@@ -1586,7 +1586,7 @@
     }
 
     /* Allocate the IRP */
-    if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, TRUE)))
+    if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE)))
     {
         ObDereferenceObject(FileObject);
         return STATUS_INSUFFICIENT_RESOURCES;
@@ -1699,7 +1699,7 @@
    DeviceObject = FileObject->DeviceObject;
 
 
-   Irp = IoAllocateIrp(DeviceObject->StackSize, TRUE);
+   Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
    if (Irp==NULL)
      {
  ObDereferenceObject(FileObject);
@@ -1795,7 +1795,7 @@
   DeviceObject = IoGetRelatedDeviceObject(FileObject);
 
   Irp = IoAllocateIrp(DeviceObject->StackSize,
-        TRUE);
+        FALSE);
   if (Irp == NULL)
   {
     Status = STATUS_INSUFFICIENT_RESOURCES;
@@ -2123,7 +2123,7 @@
     }
 
     /* Allocate the IRP */
-    if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, TRUE)))
+    if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE)))
     {
         ObDereferenceObject(FileObject);
         return STATUS_INSUFFICIENT_RESOURCES;
@@ -2321,7 +2321,7 @@
     }
 
     /* Allocate the IRP */
-    if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, TRUE)))
+    if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE)))
     {
         ObDereferenceObject(FileObject);
         return STATUS_INSUFFICIENT_RESOURCES;
@@ -2775,7 +2775,7 @@
     }
 
     /* Allocate the IRP */
-    if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, TRUE)))
+    if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE)))
     {
         ObDereferenceObject(FileObject);
         return STATUS_INSUFFICIENT_RESOURCES;
@@ -2900,7 +2900,7 @@
   DeviceObject = IoGetRelatedDeviceObject(FileObject);
 
   Irp = IoAllocateIrp(DeviceObject->StackSize,
-        TRUE);
+        FALSE);
   if (Irp == NULL)
   {
     Status = STATUS_INSUFFICIENT_RESOURCES;
  _____  

Modified: trunk/reactos/ntoskrnl/io/irp.c
--- trunk/reactos/ntoskrnl/io/irp.c	2005-05-13 03:34:13 UTC (rev
15254)
+++ trunk/reactos/ntoskrnl/io/irp.c	2005-05-13 04:49:54 UTC (rev
15255)
@@ -366,31 +366,93 @@

 IoAllocateIrp(CCHAR StackSize,
               BOOLEAN ChargeQuota)
 {
-    PIRP Irp;
+    PIRP Irp = NULL;
     USHORT Size = IoSizeOfIrp(StackSize);
-
-    /* Check if we shoudl charge quota */
-    if (ChargeQuota)
+    PKPRCB Prcb;
+    ULONG Flags = 0;
+    PNPAGED_LOOKASIDE_LIST List;
+    PP_NPAGED_LOOKASIDE_NUMBER ListType = LookasideSmallIrpList;
+    
+    /* Figure out which Lookaside List to use */
+    if ((StackSize <= 8) && (ChargeQuota == FALSE))
     {
-        /* Irp =
ExAllocatePoolWithQuotaTag(NonPagedPool,IoSizeOfIrp(StackSize),
TAG_IRP); */
-        /* FIXME */
-        Irp = ExAllocatePoolWithTag(NonPagedPool,
-                                    Size,
-                                    TAG_IRP);
+        DPRINT("Using lookaside, %d\n", StackSize);
+        /* Set Fixed Size Flag */
+        Flags = IRP_ALLOCATED_FIXED_SIZE;
+        
+        /* See if we should use big list */
+        if (StackSize != 1)
+        {
+            DPRINT("Using large lookaside\n");
+            Size = IoSizeOfIrp(8);
+            ListType = LookasideLargeIrpList;
+        }
+        
+        /* Get the PRCB */
+        Prcb = KeGetCurrentPrcb();
+        
+        /* Get the P List First */
+        List =
(PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].P;
+        
+        /* Attempt allocation */
+        List->L.TotalAllocates++;
+        DPRINT("Total allocates: %d\n", List->L.TotalAllocates);
+        Irp = (PIRP)InterlockedPopEntrySList(&List->L.ListHead);
+        DPRINT("Alloc attempt on CPU list: %p\n", Irp);
+        
+        /* Check if the P List failed */
+        if (!Irp)
+        {
+            /* Let the balancer know */
+            List->L.AllocateMisses++;
+            DPRINT("Total misses: %d\n", List->L.AllocateMisses);
+            
+            /* Try the L List */
+            List =
(PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].L;
+            List->L.TotalAllocates++;
+            Irp = (PIRP)InterlockedPopEntrySList(&List->L.ListHead);
+            DPRINT("Alloc attempt on SYS list: %p\n", Irp);
+        }
     }
+    
+    /* Check if we have to use the pool */
+    if (!Irp)
+    {
+        DPRINT("Using pool\n");
+        /* Did we try lookaside and fail? */
+        if (Flags & IRP_ALLOCATED_FIXED_SIZE) List->L.AllocateMisses++;
+        
+        /* Check if we shoudl charge quota */
+        if (ChargeQuota)
+        {
+            /* Irp = ExAllocatePoolWithQuotaTag(NonPagedPool, Size,
TAG_IRP); */
+            /* FIXME */
+            Irp = ExAllocatePoolWithTag(NonPagedPool, Size, TAG_IRP);
+        }
+        else
+        {
+            /* Allocate the IRP With no Quota charge */
+            Irp = ExAllocatePoolWithTag(NonPagedPool, Size, TAG_IRP);
+        }
+        
+        /* Make sure it was sucessful */
+        if (!Irp) return(NULL);    
+    }
     else
     {
-        /* Allocate the IRP With no Quota charge */
-        Irp = ExAllocatePoolWithTag(NonPagedPool,
-                                    Size,
-                                    TAG_IRP);
+        /* We have an IRP from Lookaside */
+        Flags |= IRP_LOOKASIDE_ALLOCATION;
     }
-
-    /* Make sure it was sucessful */
-    if (Irp==NULL) return(NULL);
-
+   
+    /* Set Flag */
+    if (ChargeQuota) Flags |= IRP_QUOTA_CHARGED;
+    
     /* Now Initialize it */
+    DPRINT("irp allocated\n");
     IoInitializeIrp(Irp, Size, StackSize);
+    
+    /* Set the Allocation Flags */
+    Irp->AllocationFlags = Flags;
 
     /* Return it */
     return Irp;
@@ -430,7 +492,7 @@
             StartingOffset,IoStatusBlock);
 
     /* Allocate IRP */
-    if (!(Irp = IoAllocateIrp(DeviceObject->StackSize,TRUE))) return
Irp;
+    if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE))) return
Irp;
 
     /* Get the Stack */
     StackPtr = IoGetNextIrpStackLocation(Irp);
@@ -580,7 +642,7 @@
            InternalDeviceIoControl,Event,IoStatusBlock);
 
     /* Allocate IRP */
-    if (!(Irp = IoAllocateIrp(DeviceObject->StackSize,TRUE))) return
Irp;
+    if (!(Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE))) return
Irp;
 
     /* Get the Stack */
     StackPtr = IoGetNextIrpStackLocation(Irp);
@@ -1177,8 +1239,59 @@
 STDCALL
 IoFreeIrp(PIRP Irp)
 {
-    /* Free the pool memory associated with it */
-    ExFreePoolWithTag(Irp, TAG_IRP);
+    PNPAGED_LOOKASIDE_LIST List;
+    PP_NPAGED_LOOKASIDE_NUMBER ListType =  LookasideSmallIrpList;
+    PKPRCB Prcb;
+    
+    /* If this was a pool alloc, free it with the pool */
+    if (!(Irp->AllocationFlags & IRP_ALLOCATED_FIXED_SIZE))
+    {
+        /* Free it */
+        DPRINT("Freeing pool IRP\n");
+        ExFreePool(Irp);
+    }
+    else
+    {
+        DPRINT("Freeing Lookaside IRP\n");
+        
+        /* Check if this was a Big IRP */
+        if (Irp->StackCount != 1)
+        {
+            DPRINT("Large IRP\n");
+            ListType = LookasideLargeIrpList;
+        }
+        
+        /* Get the PRCB */
+        Prcb = KeGetCurrentPrcb();
+        
+        /* Use the P List */
+        List =
(PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].P;
+        List->L.TotalFrees++;
+        
+        /* Check if the Free was within the Depth or not */
+        if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth)
+        {
+            /* Let the balancer know */
+            List->L.FreeMisses++;
+            
+            /* Use the L List */
+            List =
(PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].L;
+            List->L.TotalFrees++;
+
+            /* Check if the Free was within the Depth or not */
+            if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth)
+            {            
+                /* All lists failed, use the pool */
+                List->L.FreeMisses++;
+                ExFreePool(Irp);
+            }
+        }
+        
+        /* The free was within dhe Depth */
+        InterlockedPushEntrySList(&List->L.ListHead,
(PSINGLE_LIST_ENTRY)Irp);
+    }
+    
+    DPRINT("Free done\n");
 }
 
 /*
@@ -1324,7 +1437,7 @@
    PIRP AssocIrp;
 
    /* Allocate the IRP */
-   AssocIrp = IoAllocateIrp(StackSize,FALSE);
+   AssocIrp = IoAllocateIrp(StackSize, FALSE);
    if (AssocIrp == NULL) return NULL;
 
    /* Set the Flags */