[ros-arm-bringup] 39925: - Implement a new Hyperspace Mapping Interface: - The new interface is portable and much faster than before. For example, unmapping a hyperpage is almost a one-line operation. - The new interface is also thread-safe and uses the EPROCESS hyperspace spinlock. - However, in order to isolate from React Mm internals, the Hyper IRQL and Process are stored as globals, so this will not work on SMP. - For now, mapping vs. zero PTEs are not treated differently, but the two interf

Author: ros-arm-bringup
Date: Tue Mar 10 03:31:14 2009
New Revision: 39925

URL: http://svn.reactos.org/svn/reactos?rev=39925&view=rev
Log:
- Implement a new Hyperspace Mapping Interface:
    - The new interface is portable and much faster than before. For example, unmapping a
hyperpage is almost a one-line operation.
    - The new interface is also thread-safe and uses the EPROCESS hyperspace spinlock.
    - However, in order to isolate from React Mm internals, the Hyper IRQL and Process are
stored as globals, so this will not work on SMP.
    - For now, mapping vs. zero PTEs are not treated differently, but the two interfaces
have been separated pending future work.
    - Performance tests with _rdtsc resulted in an improvement of over 300% compared to
the old interface.
    - Hyperspace mappings are frequent, so the improvement is noticeable during startup
(3/10ths of a second).
    - This also fixes incorrect initializtion of hyperspace -- pages were zeroed out
(which requires hyperspace) before hyperspace was created.


Added:
    trunk/reactos/ntoskrnl/mm/hypermap.c   (with props)
Modified:
    trunk/reactos/ntoskrnl/include/internal/i386/mm.h
    trunk/reactos/ntoskrnl/include/internal/mm.h
    trunk/reactos/ntoskrnl/mm/i386/page.c
    trunk/reactos/ntoskrnl/mm/kmap.c
    trunk/reactos/ntoskrnl/mm/mminit.c
    trunk/reactos/ntoskrnl/ntoskrnl-generic.rbuild

Modified: trunk/reactos/ntoskrnl/include/internal/i386/mm.h
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/include/internal/…
==============================================================================
--- trunk/reactos/ntoskrnl/include/internal/i386/mm.h [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/include/internal/i386/mm.h [iso-8859-1] Tue Mar 10 03:31:14
2009
@@ -20,6 +20,8 @@
     ((PMMPTE)(((((ULONG)(x)) >> 22) << 2) + PAGEDIRECTORY_MAP))
 #define MiAddressToPte(x) \
     ((PMMPTE)(((((ULONG)(x)) >> 12) << 2) + PAGETABLE_MAP))
+#define MiAddressToPteOffset(x) \
+    ((((ULONG)(x)) << 10) >> 22)
 
 #define ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (1024 * PAGE_SIZE))
 #define ADDR_TO_PDE_OFFSET(v) ((((ULONG)(v)) / (1024 * PAGE_SIZE)))

Modified: trunk/reactos/ntoskrnl/include/internal/mm.h
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/include/internal/…
==============================================================================
--- trunk/reactos/ntoskrnl/include/internal/mm.h [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/include/internal/mm.h [iso-8859-1] Tue Mar 10 03:31:14 2009
@@ -58,6 +58,13 @@
 
 /* Number of list heads to use */
 #define MI_FREE_POOL_LISTS 4
+
+#define HYPER_SPACE		                    (0xC0400000)
+
+#define MI_HYPERSPACE_PTES                  (256 - 1)
+#define MI_MAPPING_RANGE_START              (ULONG)HYPER_SPACE
+#define MI_MAPPING_RANGE_END                (MI_MAPPING_RANGE_START + \
+                                             MI_HYPERSPACE_PTES * PAGE_SIZE)
 
 /* Signature of free pool blocks */
 #define MM_FREE_POOL_TAG    TAG('F', 'r', 'p', 'l')
@@ -1060,15 +1067,45 @@
     PVOID Context
 );
 
+/* hypermap.c *****************************************************************/
+
+extern PEPROCESS HyperProcess;
+extern KIRQL HyperIrql;
+
+PVOID
+NTAPI
+MiMapPageInHyperSpace(IN PEPROCESS Process,
+                      IN PFN_NUMBER Page,
+                      IN PKIRQL OldIrql);
+
+VOID
+NTAPI
+MiUnmapPageInHyperSpace(IN PEPROCESS Process,
+                        IN PVOID Address,
+                        IN KIRQL OldIrql);
+
+PVOID
+NTAPI
+MiMapPagesToZeroInHyperSpace(IN PFN_NUMBER Page);
+
+VOID
+NTAPI
+MiUnmapPagesInZeroSpace(IN PVOID Address);
+
+//
+// ReactOS Compatibility Layer
+//
+PVOID
+FORCEINLINE
+MmCreateHyperspaceMapping(IN PFN_NUMBER Page)
+{
+    HyperProcess = (PEPROCESS)KeGetCurrentThread()->ApcState.Process;
+    return MiMapPageInHyperSpace(HyperProcess, Page, &HyperIrql);
+}
+
+#define MmDeleteHyperspaceMapping(x) MiUnmapPageInHyperSpace(HyperProcess, x,
HyperIrql);
+
 /* i386/page.c *********************************************************/
-
-PVOID
-NTAPI
-MmCreateHyperspaceMapping(PFN_TYPE Page);
-
-PFN_TYPE
-NTAPI
-MmDeleteHyperspaceMapping(PVOID Address);
 
 NTSTATUS
 NTAPI

Added: trunk/reactos/ntoskrnl/mm/hypermap.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/hypermap.c?rev…
==============================================================================
--- trunk/reactos/ntoskrnl/mm/hypermap.c (added)
+++ trunk/reactos/ntoskrnl/mm/hypermap.c [iso-8859-1] Tue Mar 10 03:31:14 2009
@@ -1,0 +1,197 @@
+/*
+ * PROJECT:         ReactOS Kernel
+ * LICENSE:         BSD - See COPYING.ARM in the top level directory
+ * FILE:            ntoskrnl/mm/hypermap.c
+ * PURPOSE:         Hyperspace Mapping Functionality
+ * PROGRAMMERS:     ReactOS Portable Systems Group
+ */
+
+/* INCLUDES *******************************************************************/
+
+#include <ntoskrnl.h>
+#define NDEBUG
+#include <debug.h>
+
+/* GLOBALS ********************************************************************/
+
+PMMPTE MmFirstReservedMappingPte;
+PMMPTE MmLastReservedMappingPte;
+MMPTE HyperTemplatePte;
+PEPROCESS HyperProcess;
+KIRQL HyperIrql;
+
+/* PRIVATE FUNCTIONS **********************************************************/
+
+VOID
+NTAPI
+MiInitHyperSpace(VOID)
+{
+    PMMPTE PointerPte;
+    
+    //
+    // Get the hyperspace PTE and zero out the page table
+    //
+    PointerPte = MiAddressToPte(HYPER_SPACE);
+    RtlZeroMemory(PointerPte, PAGE_SIZE);
+    
+    //
+    // Setup mapping PTEs
+    //
+    MmFirstReservedMappingPte = MiAddressToPte(MI_MAPPING_RANGE_START);
+    MmLastReservedMappingPte =  MiAddressToPte(MI_MAPPING_RANGE_END);
+    MmFirstReservedMappingPte->u.Hard.PageFrameNumber = MI_HYPERSPACE_PTES;
+}
+
+PVOID
+NTAPI
+MiMapPageInHyperSpace(IN PEPROCESS Process,
+                      IN PFN_NUMBER Page,
+                      IN PKIRQL OldIrql)
+{
+    MMPTE TempPte;
+    PMMPTE PointerPte;
+    PFN_NUMBER Offset;
+    PVOID Address; 
+    
+    //
+    // Never accept page 0
+    //
+    ASSERT(Page != 0);
+    
+    //
+    // Build the PTE
+    //
+    TempPte = HyperTemplatePte;
+    TempPte.u.Hard.PageFrameNumber = Page;
+    
+    //
+    // Pick the first hyperspace PTE
+    //
+    PointerPte = MmFirstReservedMappingPte;
+
+    //
+    // Acquire the hyperlock
+    //
+    ASSERT(Process == PsGetCurrentProcess());
+    KeAcquireSpinLock(&Process->HyperSpaceLock, OldIrql);
+    
+    //
+    // Now get the first free PTE
+    //
+    Offset = PFN_FROM_PTE(PointerPte);
+    if (!Offset)
+    {
+        //
+        // Reset the PTEs
+        //
+        Offset = MI_HYPERSPACE_PTES;
+        KeFlushProcessTb();
+    }
+    
+    //
+    // Prepare the next PTE
+    //
+    PointerPte->u.Hard.PageFrameNumber = Offset - 1;
+    
+    //
+    // Write the current PTE
+    //
+    PointerPte += Offset;
+    ASSERT(PointerPte->u.Hard.Valid == 0);
+    ASSERT(TempPte.u.Hard.Valid == 1);
+    *PointerPte = TempPte;
+    
+    //
+    // Return the address
+    //
+    Address = (PVOID)((ULONG_PTR)PointerPte << 10);
+    return Address;
+}
+
+VOID
+NTAPI
+MiUnmapPageInHyperSpace(IN PEPROCESS Process,
+                        IN PVOID Address,
+                        IN KIRQL OldIrql)
+{
+    ASSERT(Process == PsGetCurrentProcess());
+
+    //
+    // Blow away the mapping
+    //
+    MiAddressToPte(Address)->u.Long = 0;
+    
+    //
+    // Release the hyperlock
+    //    
+    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
+    KeReleaseSpinLock(&Process->HyperSpaceLock, OldIrql);
+}
+
+PVOID
+NTAPI
+MiMapPagesToZeroInHyperSpace(IN PFN_NUMBER Page)
+{
+    MMPTE TempPte;
+    PMMPTE PointerPte;
+    PFN_NUMBER Offset;
+    PVOID Address; 
+    
+    //
+    // Never accept page 0
+    //
+    ASSERT(Page != 0);
+    
+    //
+    // Build the PTE
+    //
+    TempPte = HyperTemplatePte;
+    TempPte.u.Hard.PageFrameNumber = Page;
+    
+    //
+    // Pick the first hyperspace PTE
+    //
+    PointerPte = MmFirstReservedMappingPte;
+
+    //
+    // Now get the first free PTE
+    //
+    Offset = PFN_FROM_PTE(PointerPte);
+    if (!Offset)
+    {
+        //
+        // Reset the PTEs
+        //
+        Offset = MI_HYPERSPACE_PTES;
+        KeFlushProcessTb();
+    }
+    
+    //
+    // Prepare the next PTE
+    //
+    PointerPte->u.Hard.PageFrameNumber = Offset - 1;
+    
+    //
+    // Write the current PTE
+    //
+    PointerPte += Offset;
+    ASSERT(PointerPte->u.Hard.Valid == 0);
+    ASSERT(TempPte.u.Hard.Valid == 1);
+    *PointerPte = TempPte;
+    
+    //
+    // Return the address
+    //
+    Address = (PVOID)((ULONG_PTR)PointerPte << 10);
+    return Address;
+}
+
+VOID
+NTAPI
+MiUnmapPagesInZeroSpace(IN PVOID Address)
+{
+    //
+    // Blow away the mapping
+    //
+    MiAddressToPte(Address)->u.Long = 0;
+}

Propchange: trunk/reactos/ntoskrnl/mm/hypermap.c
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: trunk/reactos/ntoskrnl/mm/hypermap.c
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Modified: trunk/reactos/ntoskrnl/mm/i386/page.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/i386/page.c?re…
==============================================================================
--- trunk/reactos/ntoskrnl/mm/i386/page.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/mm/i386/page.c [iso-8859-1] Tue Mar 10 03:31:14 2009
@@ -1069,88 +1069,6 @@
     return p;
 }
 
-PVOID
-NTAPI
-MmCreateHyperspaceMapping(PFN_TYPE Page)
-{
-    PVOID Address;
-    ULONG i;
-    
-    ULONG Entry;
-    PULONG Pte;
-    Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE;
-    Pte = (PULONG)MiAddressToPte(HYPERSPACE) + Page % 1024;
-    if (Page & 1024)
-    {
-        for (i = Page % 1024; i < 1024; i++, Pte++)
-        {
-            if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
-            {
-                break;
-            }
-        }
-        if (i >= 1024)
-        {
-            Pte = (PULONG)MiAddressToPte(HYPERSPACE);
-            for (i = 0; i < Page % 1024; i++, Pte++)
-            {
-                if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
-                {
-                    break;
-                }
-            }
-            if (i >= Page % 1024)
-            {
-                KeBugCheck(MEMORY_MANAGEMENT);
-            }
-        }
-    }
-    else
-    {
-        for (i = Page % 1024; (LONG)i >= 0; i--, Pte--)
-        {
-            if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
-            {
-                break;
-            }
-        }
-        if ((LONG)i < 0)
-        {
-            Pte = (PULONG)MiAddressToPte(HYPERSPACE) + 1023;
-            for (i = 1023; i > Page % 1024; i--, Pte--)
-            {
-                if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
-                {
-                    break;
-                }
-            }
-            if (i <= Page % 1024)
-            {
-                KeBugCheck(MEMORY_MANAGEMENT);
-            }
-        }
-    }
-    Address = (PVOID)((ULONG_PTR)HYPERSPACE + i * PAGE_SIZE);
-    __invlpg(Address);
-    return Address;
-}
-
-PFN_TYPE
-NTAPI
-MmDeleteHyperspaceMapping(PVOID Address)
-{
-    PFN_TYPE Pfn;
-    ULONG Entry;
-    
-    ASSERT (IS_HYPERSPACE(Address));
-    
-    Entry = InterlockedExchangePte(MiAddressToPte(Address), 0);
-    Pfn = PTE_TO_PFN(Entry);
-    
-    __invlpg(Address);
-    return Pfn;
-}
-
 VOID
 NTAPI
 MmUpdatePageDir(PEPROCESS Process, PVOID Address, ULONG Size)
@@ -1187,6 +1105,8 @@
     }
 }
 
+extern MMPTE HyperTemplatePte;
+
 VOID
 INIT_FUNCTION
 NTAPI
@@ -1196,6 +1116,12 @@
     PULONG CurrentPageDirectory = (PULONG)PAGEDIRECTORY_MAP;
     
     DPRINT("MmInitGlobalKernelPageDirectory()\n");
+    
+    //
+    // Setup template
+    //
+    HyperTemplatePte.u.Long = (PA_PRESENT | PA_READWRITE | PA_DIRTY | PA_ACCESSED);
+    if (Ke386GlobalPagesEnabled) HyperTemplatePte.u.Long |= PA_BIT_GLOBAL;
     
     for (i = ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 1024; i++)
     {

Modified: trunk/reactos/ntoskrnl/mm/kmap.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/kmap.c?rev=399…
==============================================================================
--- trunk/reactos/ntoskrnl/mm/kmap.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/mm/kmap.c [iso-8859-1] Tue Mar 10 03:31:14 2009
@@ -22,13 +22,13 @@
 {
    PVOID TempAddress;
 
-   TempAddress = MmCreateHyperspaceMapping(Page);
+   TempAddress = MiMapPagesToZeroInHyperSpace(Page);
    if (TempAddress == NULL)
    {
       return(STATUS_NO_MEMORY);
    }
    memset(TempAddress, 0, PAGE_SIZE);
-   MmDeleteHyperspaceMapping(TempAddress);
+   MiUnmapPagesInZeroSpace(TempAddress);
    return(STATUS_SUCCESS);
 }
 

Modified: trunk/reactos/ntoskrnl/mm/mminit.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/mminit.c?rev=3…
==============================================================================
--- trunk/reactos/ntoskrnl/mm/mminit.c [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/mm/mminit.c [iso-8859-1] Tue Mar 10 03:31:14 2009
@@ -351,6 +351,11 @@
     return LastKrnlPhysAddr << PAGE_SHIFT;
 }
 
+
+VOID
+NTAPI
+MiInitHyperSpace(VOID);
+
 VOID
 INIT_FUNCTION
 NTAPI
@@ -434,6 +439,9 @@
     
     /* Unmap low memory */
     MmDeletePageTable(NULL, 0);
+    
+    /* Initialize hyperspace */
+    MiInitHyperSpace();
 
     /* Intialize memory areas */
     MmInitVirtualMemory();

Modified: trunk/reactos/ntoskrnl/ntoskrnl-generic.rbuild
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/ntoskrnl-generic.…
==============================================================================
--- trunk/reactos/ntoskrnl/ntoskrnl-generic.rbuild [iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/ntoskrnl-generic.rbuild [iso-8859-1] Tue Mar 10 03:31:14 2009
@@ -365,6 +365,7 @@
 		<file>cont.c</file>
 		<file>drvlck.c</file>
 		<file>freelist.c</file>
+		<file>hypermap.c</file>
 		<file>iospace.c</file>
 		<file>kmap.c</file>
 		<file>marea.c</file>