Author: ros-arm-bringup Date: Tue Mar 10 03:31:14 2009 New Revision: 39925
URL: http://svn.reactos.org/svn/reactos?rev=39925&view=rev Log: - Implement a new Hyperspace Mapping Interface: - The new interface is portable and much faster than before. For example, unmapping a hyperpage is almost a one-line operation. - The new interface is also thread-safe and uses the EPROCESS hyperspace spinlock. - However, in order to isolate from React Mm internals, the Hyper IRQL and Process are stored as globals, so this will not work on SMP. - For now, mapping vs. zero PTEs are not treated differently, but the two interfaces have been separated pending future work. - Performance tests with _rdtsc resulted in an improvement of over 300% compared to the old interface. - Hyperspace mappings are frequent, so the improvement is noticeable during startup (3/10ths of a second). - This also fixes incorrect initializtion of hyperspace -- pages were zeroed out (which requires hyperspace) before hyperspace was created.
Added: trunk/reactos/ntoskrnl/mm/hypermap.c (with props) Modified: trunk/reactos/ntoskrnl/include/internal/i386/mm.h trunk/reactos/ntoskrnl/include/internal/mm.h trunk/reactos/ntoskrnl/mm/i386/page.c trunk/reactos/ntoskrnl/mm/kmap.c trunk/reactos/ntoskrnl/mm/mminit.c trunk/reactos/ntoskrnl/ntoskrnl-generic.rbuild
Modified: trunk/reactos/ntoskrnl/include/internal/i386/mm.h URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/include/internal/i... ============================================================================== --- trunk/reactos/ntoskrnl/include/internal/i386/mm.h [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/include/internal/i386/mm.h [iso-8859-1] Tue Mar 10 03:31:14 2009 @@ -20,6 +20,8 @@ ((PMMPTE)(((((ULONG)(x)) >> 22) << 2) + PAGEDIRECTORY_MAP)) #define MiAddressToPte(x) \ ((PMMPTE)(((((ULONG)(x)) >> 12) << 2) + PAGETABLE_MAP)) +#define MiAddressToPteOffset(x) \ + ((((ULONG)(x)) << 10) >> 22)
#define ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (1024 * PAGE_SIZE)) #define ADDR_TO_PDE_OFFSET(v) ((((ULONG)(v)) / (1024 * PAGE_SIZE)))
Modified: trunk/reactos/ntoskrnl/include/internal/mm.h URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/include/internal/m... ============================================================================== --- trunk/reactos/ntoskrnl/include/internal/mm.h [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/include/internal/mm.h [iso-8859-1] Tue Mar 10 03:31:14 2009 @@ -58,6 +58,13 @@
/* Number of list heads to use */ #define MI_FREE_POOL_LISTS 4 + +#define HYPER_SPACE (0xC0400000) + +#define MI_HYPERSPACE_PTES (256 - 1) +#define MI_MAPPING_RANGE_START (ULONG)HYPER_SPACE +#define MI_MAPPING_RANGE_END (MI_MAPPING_RANGE_START + \ + MI_HYPERSPACE_PTES * PAGE_SIZE)
/* Signature of free pool blocks */ #define MM_FREE_POOL_TAG TAG('F', 'r', 'p', 'l') @@ -1060,15 +1067,45 @@ PVOID Context );
+/* hypermap.c *****************************************************************/ + +extern PEPROCESS HyperProcess; +extern KIRQL HyperIrql; + +PVOID +NTAPI +MiMapPageInHyperSpace(IN PEPROCESS Process, + IN PFN_NUMBER Page, + IN PKIRQL OldIrql); + +VOID +NTAPI +MiUnmapPageInHyperSpace(IN PEPROCESS Process, + IN PVOID Address, + IN KIRQL OldIrql); + +PVOID +NTAPI +MiMapPagesToZeroInHyperSpace(IN PFN_NUMBER Page); + +VOID +NTAPI +MiUnmapPagesInZeroSpace(IN PVOID Address); + +// +// ReactOS Compatibility Layer +// +PVOID +FORCEINLINE +MmCreateHyperspaceMapping(IN PFN_NUMBER Page) +{ + HyperProcess = (PEPROCESS)KeGetCurrentThread()->ApcState.Process; + return MiMapPageInHyperSpace(HyperProcess, Page, &HyperIrql); +} + +#define MmDeleteHyperspaceMapping(x) MiUnmapPageInHyperSpace(HyperProcess, x, HyperIrql); + /* i386/page.c *********************************************************/ - -PVOID -NTAPI -MmCreateHyperspaceMapping(PFN_TYPE Page); - -PFN_TYPE -NTAPI -MmDeleteHyperspaceMapping(PVOID Address);
NTSTATUS NTAPI
Added: trunk/reactos/ntoskrnl/mm/hypermap.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/hypermap.c?rev=... ============================================================================== --- trunk/reactos/ntoskrnl/mm/hypermap.c (added) +++ trunk/reactos/ntoskrnl/mm/hypermap.c [iso-8859-1] Tue Mar 10 03:31:14 2009 @@ -1,0 +1,197 @@ +/* + * PROJECT: ReactOS Kernel + * LICENSE: BSD - See COPYING.ARM in the top level directory + * FILE: ntoskrnl/mm/hypermap.c + * PURPOSE: Hyperspace Mapping Functionality + * PROGRAMMERS: ReactOS Portable Systems Group + */ + +/* INCLUDES *******************************************************************/ + +#include <ntoskrnl.h> +#define NDEBUG +#include <debug.h> + +/* GLOBALS ********************************************************************/ + +PMMPTE MmFirstReservedMappingPte; +PMMPTE MmLastReservedMappingPte; +MMPTE HyperTemplatePte; +PEPROCESS HyperProcess; +KIRQL HyperIrql; + +/* PRIVATE FUNCTIONS **********************************************************/ + +VOID +NTAPI +MiInitHyperSpace(VOID) +{ + PMMPTE PointerPte; + + // + // Get the hyperspace PTE and zero out the page table + // + PointerPte = MiAddressToPte(HYPER_SPACE); + RtlZeroMemory(PointerPte, PAGE_SIZE); + + // + // Setup mapping PTEs + // + MmFirstReservedMappingPte = MiAddressToPte(MI_MAPPING_RANGE_START); + MmLastReservedMappingPte = MiAddressToPte(MI_MAPPING_RANGE_END); + MmFirstReservedMappingPte->u.Hard.PageFrameNumber = MI_HYPERSPACE_PTES; +} + +PVOID +NTAPI +MiMapPageInHyperSpace(IN PEPROCESS Process, + IN PFN_NUMBER Page, + IN PKIRQL OldIrql) +{ + MMPTE TempPte; + PMMPTE PointerPte; + PFN_NUMBER Offset; + PVOID Address; + + // + // Never accept page 0 + // + ASSERT(Page != 0); + + // + // Build the PTE + // + TempPte = HyperTemplatePte; + TempPte.u.Hard.PageFrameNumber = Page; + + // + // Pick the first hyperspace PTE + // + PointerPte = MmFirstReservedMappingPte; + + // + // Acquire the hyperlock + // + ASSERT(Process == PsGetCurrentProcess()); + KeAcquireSpinLock(&Process->HyperSpaceLock, OldIrql); + + // + // Now get the first free PTE + // + Offset = PFN_FROM_PTE(PointerPte); + if (!Offset) + { + // + // Reset the PTEs + // + Offset = MI_HYPERSPACE_PTES; + KeFlushProcessTb(); + } + + // + // Prepare the next PTE + // + PointerPte->u.Hard.PageFrameNumber = Offset - 1; + + // + // Write the current PTE + // + PointerPte += Offset; + ASSERT(PointerPte->u.Hard.Valid == 0); + ASSERT(TempPte.u.Hard.Valid == 1); + *PointerPte = TempPte; + + // + // Return the address + // + Address = (PVOID)((ULONG_PTR)PointerPte << 10); + return Address; +} + +VOID +NTAPI +MiUnmapPageInHyperSpace(IN PEPROCESS Process, + IN PVOID Address, + IN KIRQL OldIrql) +{ + ASSERT(Process == PsGetCurrentProcess()); + + // + // Blow away the mapping + // + MiAddressToPte(Address)->u.Long = 0; + + // + // Release the hyperlock + // + ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); + KeReleaseSpinLock(&Process->HyperSpaceLock, OldIrql); +} + +PVOID +NTAPI +MiMapPagesToZeroInHyperSpace(IN PFN_NUMBER Page) +{ + MMPTE TempPte; + PMMPTE PointerPte; + PFN_NUMBER Offset; + PVOID Address; + + // + // Never accept page 0 + // + ASSERT(Page != 0); + + // + // Build the PTE + // + TempPte = HyperTemplatePte; + TempPte.u.Hard.PageFrameNumber = Page; + + // + // Pick the first hyperspace PTE + // + PointerPte = MmFirstReservedMappingPte; + + // + // Now get the first free PTE + // + Offset = PFN_FROM_PTE(PointerPte); + if (!Offset) + { + // + // Reset the PTEs + // + Offset = MI_HYPERSPACE_PTES; + KeFlushProcessTb(); + } + + // + // Prepare the next PTE + // + PointerPte->u.Hard.PageFrameNumber = Offset - 1; + + // + // Write the current PTE + // + PointerPte += Offset; + ASSERT(PointerPte->u.Hard.Valid == 0); + ASSERT(TempPte.u.Hard.Valid == 1); + *PointerPte = TempPte; + + // + // Return the address + // + Address = (PVOID)((ULONG_PTR)PointerPte << 10); + return Address; +} + +VOID +NTAPI +MiUnmapPagesInZeroSpace(IN PVOID Address) +{ + // + // Blow away the mapping + // + MiAddressToPte(Address)->u.Long = 0; +}
Propchange: trunk/reactos/ntoskrnl/mm/hypermap.c ------------------------------------------------------------------------------ svn:eol-style = native
Propchange: trunk/reactos/ntoskrnl/mm/hypermap.c ------------------------------------------------------------------------------ svn:mime-type = text/plain
Modified: trunk/reactos/ntoskrnl/mm/i386/page.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/i386/page.c?rev... ============================================================================== --- trunk/reactos/ntoskrnl/mm/i386/page.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/mm/i386/page.c [iso-8859-1] Tue Mar 10 03:31:14 2009 @@ -1069,88 +1069,6 @@ return p; }
-PVOID -NTAPI -MmCreateHyperspaceMapping(PFN_TYPE Page) -{ - PVOID Address; - ULONG i; - - ULONG Entry; - PULONG Pte; - Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE; - Pte = (PULONG)MiAddressToPte(HYPERSPACE) + Page % 1024; - if (Page & 1024) - { - for (i = Page % 1024; i < 1024; i++, Pte++) - { - if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0)) - { - break; - } - } - if (i >= 1024) - { - Pte = (PULONG)MiAddressToPte(HYPERSPACE); - for (i = 0; i < Page % 1024; i++, Pte++) - { - if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0)) - { - break; - } - } - if (i >= Page % 1024) - { - KeBugCheck(MEMORY_MANAGEMENT); - } - } - } - else - { - for (i = Page % 1024; (LONG)i >= 0; i--, Pte--) - { - if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0)) - { - break; - } - } - if ((LONG)i < 0) - { - Pte = (PULONG)MiAddressToPte(HYPERSPACE) + 1023; - for (i = 1023; i > Page % 1024; i--, Pte--) - { - if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0)) - { - break; - } - } - if (i <= Page % 1024) - { - KeBugCheck(MEMORY_MANAGEMENT); - } - } - } - Address = (PVOID)((ULONG_PTR)HYPERSPACE + i * PAGE_SIZE); - __invlpg(Address); - return Address; -} - -PFN_TYPE -NTAPI -MmDeleteHyperspaceMapping(PVOID Address) -{ - PFN_TYPE Pfn; - ULONG Entry; - - ASSERT (IS_HYPERSPACE(Address)); - - Entry = InterlockedExchangePte(MiAddressToPte(Address), 0); - Pfn = PTE_TO_PFN(Entry); - - __invlpg(Address); - return Pfn; -} - VOID NTAPI MmUpdatePageDir(PEPROCESS Process, PVOID Address, ULONG Size) @@ -1187,6 +1105,8 @@ } }
+extern MMPTE HyperTemplatePte; + VOID INIT_FUNCTION NTAPI @@ -1196,6 +1116,12 @@ PULONG CurrentPageDirectory = (PULONG)PAGEDIRECTORY_MAP;
DPRINT("MmInitGlobalKernelPageDirectory()\n"); + + // + // Setup template + // + HyperTemplatePte.u.Long = (PA_PRESENT | PA_READWRITE | PA_DIRTY | PA_ACCESSED); + if (Ke386GlobalPagesEnabled) HyperTemplatePte.u.Long |= PA_BIT_GLOBAL;
for (i = ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 1024; i++) {
Modified: trunk/reactos/ntoskrnl/mm/kmap.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/kmap.c?rev=3992... ============================================================================== --- trunk/reactos/ntoskrnl/mm/kmap.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/mm/kmap.c [iso-8859-1] Tue Mar 10 03:31:14 2009 @@ -22,13 +22,13 @@ { PVOID TempAddress;
- TempAddress = MmCreateHyperspaceMapping(Page); + TempAddress = MiMapPagesToZeroInHyperSpace(Page); if (TempAddress == NULL) { return(STATUS_NO_MEMORY); } memset(TempAddress, 0, PAGE_SIZE); - MmDeleteHyperspaceMapping(TempAddress); + MiUnmapPagesInZeroSpace(TempAddress); return(STATUS_SUCCESS); }
Modified: trunk/reactos/ntoskrnl/mm/mminit.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/mminit.c?rev=39... ============================================================================== --- trunk/reactos/ntoskrnl/mm/mminit.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/mm/mminit.c [iso-8859-1] Tue Mar 10 03:31:14 2009 @@ -351,6 +351,11 @@ return LastKrnlPhysAddr << PAGE_SHIFT; }
+ +VOID +NTAPI +MiInitHyperSpace(VOID); + VOID INIT_FUNCTION NTAPI @@ -434,6 +439,9 @@
/* Unmap low memory */ MmDeletePageTable(NULL, 0); + + /* Initialize hyperspace */ + MiInitHyperSpace();
/* Intialize memory areas */ MmInitVirtualMemory();
Modified: trunk/reactos/ntoskrnl/ntoskrnl-generic.rbuild URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/ntoskrnl-generic.r... ============================================================================== --- trunk/reactos/ntoskrnl/ntoskrnl-generic.rbuild [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/ntoskrnl-generic.rbuild [iso-8859-1] Tue Mar 10 03:31:14 2009 @@ -365,6 +365,7 @@ <file>cont.c</file> <file>drvlck.c</file> <file>freelist.c</file> + <file>hypermap.c</file> <file>iospace.c</file> <file>kmap.c</file> <file>marea.c</file>