[ion] 22717: - Fix IRP/Completion packet lookaside allocation. We weren't setting a zone size, and I/O Completion packets were also using the wrong size. - Rewrite I/O MDl support to use lookaside lists for allocations below 23 pages (same as on NT). This is is an incredible performance optimization because MDLs are often allocated and de-allocated during I/O operations, and using the lookaside list decreases pool fragmentation and slowdown. - Rewrite IoBuildPartialMdl. It did not work like docu

Author: ion
Date: Fri Jun 30 19:59:06 2006
New Revision: 22717

URL: http://svn.reactos.org/svn/reactos?rev=22717&view=rev
Log:
- Fix IRP/Completion packet lookaside allocation. We weren't setting a zone size, and
I/O Completion packets were also using the wrong size.
- Rewrite I/O MDl support to use lookaside lists for allocations below 23 pages (same as
on NT). This is is an incredible performance optimization because MDLs are often allocated
and de-allocated during I/O operations, and using the lookaside list decreases pool
fragmentation and slowdown.
- Rewrite IoBuildPartialMdl. It did not work like documented in the DDK and also had a bug
documented by Microsoft as being in XP.

Modified:
    trunk/reactos/ntoskrnl/include/internal/io.h
    trunk/reactos/ntoskrnl/io/iomgr.c
    trunk/reactos/ntoskrnl/io/mdl.c   (contents, props changed)

Modified: trunk/reactos/ntoskrnl/include/internal/io.h
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/include/internal/…
==============================================================================
--- trunk/reactos/ntoskrnl/include/internal/io.h (original)
+++ trunk/reactos/ntoskrnl/include/internal/io.h Fri Jun 30 19:59:06 2006
@@ -47,6 +47,14 @@
 // Number of partition tables in the Boot Record
 //
 #define PARTITION_TBL_SIZE                              4
+
+//
+// We can call the Ob Inlined API, it's the same thing
+//
+#define IopAllocateMdlFromLookaside                     \
+    ObpAllocateCapturedAttributes
+#define IopFreeMdlFromLookaside                         \
+    ObpFreeCapturedAttributes
 
 //
 // Returns the size of a CM_RESOURCE_LIST
@@ -830,4 +838,4 @@
 extern POBJECT_TYPE IoCompletionType;
 extern PDEVICE_NODE IopRootDeviceNode;
 extern ULONG IopTraceLevel;
-
+extern NPAGED_LOOKASIDE_LIST IopMdlLookasideList;

Modified: trunk/reactos/ntoskrnl/io/iomgr.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/io/iomgr.c?rev=22…
==============================================================================
--- trunk/reactos/ntoskrnl/io/iomgr.c (original)
+++ trunk/reactos/ntoskrnl/io/iomgr.c Fri Jun 30 19:59:06 2006
@@ -41,6 +41,7 @@
 extern POBJECT_TYPE IoAdapterObjectType;
 NPAGED_LOOKASIDE_LIST IoLargeIrpLookaside;
 NPAGED_LOOKASIDE_LIST IoSmallIrpLookaside;
+NPAGED_LOOKASIDE_LIST IopMdlLookasideList;
 
 VOID INIT_FUNCTION IopInitLookasideLists(VOID);
 
@@ -74,7 +75,7 @@
 INIT_FUNCTION
 IopInitLookasideLists(VOID)
 {
-    ULONG LargeIrpSize, SmallIrpSize;
+    ULONG LargeIrpSize, SmallIrpSize, MdlSize;
     LONG i;
     PKPRCB Prcb;
     PNPAGED_LOOKASIDE_LIST CurrentList = NULL;
@@ -82,6 +83,7 @@
     /* Calculate the sizes */
     LargeIrpSize = sizeof(IRP) + (8 * sizeof(IO_STACK_LOCATION));
     SmallIrpSize = sizeof(IRP) + sizeof(IO_STACK_LOCATION);
+    MdlSize = sizeof(MDL) + (23 * sizeof(PFN_NUMBER));
 
     /* Initialize the Lookaside List for Large IRPs */
     ExInitializeNPagedLookasideList(&IoLargeIrpLookaside,
@@ -90,7 +92,7 @@
                                     0,
                                     LargeIrpSize,
                                     IO_LARGEIRP,
-                                    0);
+                                    64);
 
     /* Initialize the Lookaside List for Small IRPs */
     ExInitializeNPagedLookasideList(&IoSmallIrpLookaside,
@@ -99,7 +101,7 @@
                                     0,
                                     SmallIrpSize,
                                     IO_SMALLIRP,
-                                    0);
+                                    32);
 
     /* Initialize the Lookaside List for I\O Completion */
     ExInitializeNPagedLookasideList(&IoCompletionPacketLookaside,
@@ -108,7 +110,16 @@
                                     0,
                                     sizeof(IO_COMPLETION_PACKET),
                                     IOC_TAG1,
-                                    0);
+                                    32);
+
+    /* Initialize the Lookaside List for MDLs */
+    ExInitializeNPagedLookasideList(&IopMdlLookasideList,
+                                    NULL,
+                                    NULL,
+                                    0,
+                                    MdlSize,
+                                    TAG_MDL,
+                                    128);
 
     /* Now allocate the per-processor lists */
     for (i = 0; i < KeNumberProcessors; i++)
@@ -131,7 +142,7 @@
                                             0,
                                             LargeIrpSize,
                                             IO_LARGEIRP_CPU,
-                                            0);
+                                            64);
         }
         else
         {
@@ -146,14 +157,14 @@
                                             IO_SMALLIRP_CPU);
         if (CurrentList)
         {
-            /* Initialize the Lookaside List for Large IRPs */
+            /* Initialize the Lookaside List for Small IRPs */
             ExInitializeNPagedLookasideList(CurrentList,
                                             NULL,
                                             NULL,
                                             0,
                                             SmallIrpSize,
                                             IO_SMALLIRP_CPU,
-                                            0);
+                                            32);
         }
         else
         {
@@ -173,15 +184,37 @@
                                             NULL,
                                             NULL,
                                             0,
+                                            sizeof(IO_COMPLETION_PACKET),
+                                            IO_SMALLIRP_CPU,
+                                            32);
+        }
+        else
+        {
+            CurrentList = &IoCompletionPacketLookaside;
+        }
+        Prcb->PPLookasideList[LookasideCompletionList].P = &CurrentList->L;
+
+        /* Set the MDL Completion List */
+        Prcb->PPLookasideList[LookasideMdlList].L = &IopMdlLookasideList.L;
+        CurrentList = ExAllocatePoolWithTag(NonPagedPool,
+                                            sizeof(NPAGED_LOOKASIDE_LIST),
+                                            TAG_MDL);
+        if (CurrentList)
+        {
+            /* Initialize the Lookaside List for MDLs */
+            ExInitializeNPagedLookasideList(CurrentList,
+                                            NULL,
+                                            NULL,
+                                            0,
                                             SmallIrpSize,
-                                            IOC_CPU,
-                                            0);
+                                            TAG_MDL,
+                                            128);
         }
         else
         {
-            CurrentList = &IoCompletionPacketLookaside;
-        }
-        Prcb->PPLookasideList[LookasideCompletionList].P = &CurrentList->L;
+            CurrentList = &IopMdlLookasideList;
+        }
+        Prcb->PPLookasideList[LookasideMdlList].P = &CurrentList->L;
     }
 
     DPRINT("Done allocation\n");

Modified: trunk/reactos/ntoskrnl/io/mdl.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/io/mdl.c?rev=2271…
==============================================================================
--- trunk/reactos/ntoskrnl/io/mdl.c (original)
+++ trunk/reactos/ntoskrnl/io/mdl.c Fri Jun 30 19:59:06 2006
@@ -1,11 +1,9 @@
-/* $Id$
- *
- * COPYRIGHT:       See COPYING in the top level directory
- * PROJECT:         ReactOS kernel
+/*
+ * PROJECT:         ReactOS Kernel
+ * LICENSE:         GPL - See COPYING in the top level directory
  * FILE:            ntoskrnl/io/mdl.c
- * PURPOSE:         Io manager mdl functions
- *
- * PROGRAMMERS:     David Welch (welch(a)mcmail.com)
+ * PURPOSE:         I/O Wrappers for MDL Allocation and Deallocation
+ * PROGRAMMERS:     Alex Ionescu (alex.ionescu(a)reactos.org)
  */
 
 /* INCLUDES *****************************************************************/
@@ -20,116 +18,143 @@
  * @implemented
  */
 PMDL
-STDCALL
-IoAllocateMdl(PVOID VirtualAddress,
-		   ULONG Length,
-		   BOOLEAN SecondaryBuffer,
-		   BOOLEAN ChargeQuota,
-		   PIRP Irp)
+NTAPI
+IoAllocateMdl(IN PVOID VirtualAddress,
+              IN ULONG Length,
+              IN BOOLEAN SecondaryBuffer,
+              IN BOOLEAN ChargeQuota,
+              IN PIRP Irp)
 {
-   PMDL Mdl;
+    PMDL Mdl = NULL, p;
+    ULONG Flags = 0;
+    ULONG Size;
 
-   if (ChargeQuota)
-     {
-//	Mdl = ExAllocatePoolWithQuota(NonPagedPool,
-//				      MmSizeOfMdl(VirtualAddress,Length));
-	Mdl = ExAllocatePoolWithTag(NonPagedPool,
-				    MmSizeOfMdl(VirtualAddress,Length),
-				    TAG_MDL);
-     }
-   else
-     {
-	Mdl = ExAllocatePoolWithTag(NonPagedPool,
-				    MmSizeOfMdl(VirtualAddress,Length),
-				    TAG_MDL);
-     }
-   MmInitializeMdl(Mdl, (char*)VirtualAddress, Length);
+    /* Fail if allocation is over 2GB */
+    if (Length & 0x80000000) return NULL;
 
-   if (Irp)
-   {
-      if (SecondaryBuffer)
-      {
-         ASSERT(Irp->MdlAddress);
+    /* Calculate the number of pages for the allocation */
+    Size = ADDRESS_AND_SIZE_TO_SPAN_PAGES(VirtualAddress, Length);
+    if (Size > 23)
+    {
+        /* This is bigger then our fixed-size MDLs. Calculate real size */
+        Size *= sizeof(PFN_NUMBER);
+        Size += sizeof(MDL);
+        if (Size > MAXUSHORT) return NULL;
+    }
+    else
+    {
+        /* Use an internal fixed MDL size */
+        Size = (23 * sizeof(PFN_NUMBER)) + sizeof(MDL);
+        Flags |= MDL_ALLOCATED_FIXED_SIZE;
 
-         /* FIXME: add to end of list maybe?? */
-         Mdl->Next = Irp->MdlAddress->Next;
-         Irp->MdlAddress->Next = Mdl;
+        /* Allocate one from the lookaside list */
+        Mdl = IopAllocateMdlFromLookaside(LookasideMdlList);
+    }
+
+    /* Check if we don't have an mdl yet */
+    if (!Mdl)
+    {
+        /* Allocate one from pool */
+        Mdl = ExAllocatePoolWithTag(NonPagedPool, Size, TAG_MDL);
+        if (!Mdl) return NULL;
+    }
+
+    /* Initialize it */
+    MmInitializeMdl(Mdl, VirtualAddress, Length);
+    Mdl->MdlFlags |= Flags;
+
+    /* Check if an IRP was given too */
+    if (Irp)
+    {
+        /* Check if it came with a secondary buffer */
+        if (SecondaryBuffer)
+        {
+            /* Insert the MDL at the end */
+            p = Irp->MdlAddress;
+            while (p->Next) p = p->Next;
+            p->Next = Mdl;
       }
       else
       {
-         /*
-          * What if there's allready an mdl at Irp->MdlAddress?
-          * Is that bad and should we do something about it?
-          */
-         Irp->MdlAddress = Mdl;
+            /* Otherwise, insert it directly */
+            Irp->MdlAddress = Mdl;
       }
    }
 
-   return(Mdl);
-}
-
-/*
- * @implemented
- *
- * You must IoFreeMdl the slave before freeing the master.
- *
- * IoBuildPartialMdl is more similar to MmBuildMdlForNonPagedPool, the difference
- * is that the former takes the physical addresses from the master MDL, while the
- * latter - from the known location of the NPP.
- */
-VOID
-STDCALL
-IoBuildPartialMdl(PMDL SourceMdl,
-		       PMDL TargetMdl,
-		       PVOID VirtualAddress,
-		       ULONG Length)
-{
-   PPFN_TYPE TargetPages = (PPFN_TYPE)(TargetMdl + 1);
-   PPFN_TYPE SourcePages = (PPFN_TYPE)(SourceMdl + 1);
-   ULONG Count;
-   ULONG Delta;
-
-   DPRINT("VirtualAddress 0x%p, SourceMdl->StartVa 0x%p,
SourceMdl->MappedSystemVa 0x%p\n",
-          VirtualAddress, SourceMdl->StartVa, SourceMdl->MappedSystemVa);
-
-   TargetMdl->StartVa = (PVOID)PAGE_ROUND_DOWN(VirtualAddress);
-   TargetMdl->ByteOffset = (ULONG_PTR)VirtualAddress -
(ULONG_PTR)TargetMdl->StartVa;
-   TargetMdl->ByteCount = Length;
-   TargetMdl->Process = SourceMdl->Process;
-   Delta = (ULONG_PTR)VirtualAddress - ((ULONG_PTR)SourceMdl->StartVa +
SourceMdl->ByteOffset);
-   TargetMdl->MappedSystemVa = (char*)SourceMdl->MappedSystemVa + Delta;
-
-   TargetMdl->MdlFlags = SourceMdl->MdlFlags &
(MDL_IO_PAGE_READ|MDL_SOURCE_IS_NONPAGED_POOL|MDL_MAPPED_TO_SYSTEM_VA);
-   TargetMdl->MdlFlags |= MDL_PARTIAL;
-
-   Delta = ((ULONG_PTR)TargetMdl->StartVa - (ULONG_PTR)SourceMdl->StartVa) /
PAGE_SIZE;
-   Count = ADDRESS_AND_SIZE_TO_SPAN_PAGES(VirtualAddress,Length);
-
-   SourcePages += Delta;
-
-   DPRINT("Delta %d, Count %d\n", Delta, Count);
-
-   memcpy(TargetPages, SourcePages, Count * sizeof(PFN_TYPE));
-
+    /* Return the allocated mdl */
+    return Mdl;
 }
 
 /*
  * @implemented
  */
-VOID STDCALL
+VOID
+NTAPI
+IoBuildPartialMdl(IN PMDL SourceMdl,
+                  IN PMDL TargetMdl,
+                  IN PVOID VirtualAddress,
+                  IN ULONG Length)
+{
+    PPFN_TYPE TargetPages = (PPFN_TYPE)(TargetMdl + 1);
+    PPFN_TYPE SourcePages = (PPFN_TYPE)(SourceMdl + 1);
+    ULONG Offset;
+
+    /* Calculate the offset */
+    Offset = (ULONG)((ULONG_PTR)VirtualAddress -
+                     (ULONG_PTR)SourceMdl->StartVa) -
+                     SourceMdl->ByteOffset;
+
+    /* Check if we don't have a length and calculate it */
+    if (!Length) Length = SourceMdl->ByteCount - Offset;
+
+    /* Write the process, start VA and byte data */
+    TargetMdl->StartVa = (PVOID)PAGE_ROUND_DOWN(VirtualAddress);
+    TargetMdl->Process = SourceMdl->Process;
+    TargetMdl->ByteCount = Length;
+    TargetMdl->ByteOffset = BYTE_OFFSET(VirtualAddress);
+
+    /* Recalculate the length in pages */
+    Length = ADDRESS_AND_SIZE_TO_SPAN_PAGES(VirtualAddress, Length);
+
+    /* Set the MDL Flags */
+    TargetMdl->MdlFlags = (MDL_ALLOCATED_FIXED_SIZE | MDL_ALLOCATED_MUST_SUCCEED);
+    TargetMdl->MdlFlags |= (MDL_IO_PAGE_READ |
+                            MDL_SOURCE_IS_NONPAGED_POOL |
+                            MDL_MAPPED_TO_SYSTEM_VA |
+                            MDL_IO_SPACE);
+    TargetMdl->MdlFlags |= MDL_PARTIAL;
+
+    /* Set the mapped VA */
+    TargetMdl->MappedSystemVa = (PCHAR)SourceMdl->MappedSystemVa + Offset;
+
+    /* Now do the copy */
+    Offset = ((ULONG_PTR)TargetMdl->StartVa - (ULONG_PTR)SourceMdl->StartVa)
>>
+             PAGE_SHIFT;
+    SourcePages += Offset;
+    RtlMoveMemory(TargetPages, SourcePages, Length * sizeof(PFN_TYPE));
+}
+
+/*
+ * @implemented
+ */
+VOID
+NTAPI
 IoFreeMdl(PMDL Mdl)
 {
-   /*
-    * This unmaps partial mdl's from kernel space but also asserts that non-partial
-    * mdl's isn't still mapped into kernel space.
-    */
-   ASSERT(Mdl);
-   ASSERT_IRQL(DISPATCH_LEVEL);
+    /* Tell Mm to reuse the MDL */
+    MmPrepareMdlForReuse(Mdl);
 
-   MmPrepareMdlForReuse(Mdl);
-
-   ExFreePoolWithTag(Mdl, TAG_MDL);
+    /* Check if this was a pool allocation */
+    if (!(Mdl->MdlFlags & MDL_ALLOCATED_FIXED_SIZE))
+    {
+        /* Free it from the pool */
+        ExFreePoolWithTag(Mdl, TAG_MDL);
+    }
+    else
+    {
+        /* Free it from the lookaside */
+        IopFreeMdlFromLookaside(Mdl, LookasideMdlList);
+    }
 }
 
-
 /* EOF */

Propchange: trunk/reactos/ntoskrnl/io/mdl.c
------------------------------------------------------------------------------
--- svn:needs-lock (original)
+++ svn:needs-lock (removed)
@@ -1,1 +1,0 @@
-*