[fireball] 59491: [NTOS] - Implement enabling Large Pages support if it is detected. The Page Size Extension (PSE) bit in CR4 is enabled when paging is disabled. To achieve that, a temporary virtua...

Author: fireball
Date: Tue Jul 16 13:49:03 2013
New Revision: 59491

URL: http://svn.reactos.org/svn/reactos?rev=59491&view=rev
Log:
[NTOS]
- Implement enabling Large Pages support if it is detected. The Page Size Extension (PSE)
bit in CR4 is enabled when paging is disabled. To achieve that, a temporary virtual and
physical identity mapping is created, which is discarded after PSE is enabled.

Modified:
    trunk/reactos/ntoskrnl/include/internal/i386/ke.h
    trunk/reactos/ntoskrnl/ke/i386/ctxswitch.S
    trunk/reactos/ntoskrnl/ke/i386/kiinit.c
    trunk/reactos/ntoskrnl/ke/i386/patpge.c

Modified: trunk/reactos/ntoskrnl/include/internal/i386/ke.h
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/include/internal/…
==============================================================================
--- trunk/reactos/ntoskrnl/include/internal/i386/ke.h	[iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/include/internal/i386/ke.h	[iso-8859-1] Tue Jul 16 13:49:03
2013
@@ -163,6 +163,18 @@
     KV86_FRAME V86Frame;
 } KV8086_STACK_FRAME, *PKV8086_STACK_FRAME;
 
+//
+// Large Pages Support
+//
+typedef struct _LARGE_IDENTITY_MAP
+{
+    PHARDWARE_PTE TopLevelDirectory;
+    ULONG Cr3;
+    ULONG_PTR StartAddress;
+    ULONG PagesCount;
+    PVOID PagesList[30];
+} LARGE_IDENTITY_MAP, *PLARGE_IDENTITY_MAP;
+
 /* Diable interrupts and return whether they were enabled before */
 FORCEINLINE
 BOOLEAN
@@ -372,6 +384,33 @@
 NTAPI
 Ki386EnableGlobalPage(
     IN volatile ULONG_PTR Context
+);
+
+ULONG_PTR
+NTAPI
+Ki386EnableTargetLargePage(
+    IN volatile ULONG_PTR Context
+);
+
+BOOLEAN
+NTAPI
+Ki386CreateIdentityMap(
+    IN PLARGE_IDENTITY_MAP IdentityMap,
+    IN PVOID StartPtr,
+    IN ULONG Length
+);
+
+VOID
+NTAPI
+Ki386FreeIdentityMap(
+    IN PLARGE_IDENTITY_MAP IdentityMap
+);
+
+VOID
+NTAPI
+Ki386EnableCurrentLargePage(
+    IN ULONG_PTR StartAddress,
+    IN ULONG Cr3
 );
 
 VOID

Modified: trunk/reactos/ntoskrnl/ke/i386/ctxswitch.S
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/ke/i386/ctxswitch…
==============================================================================
--- trunk/reactos/ntoskrnl/ke/i386/ctxswitch.S	[iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/ke/i386/ctxswitch.S	[iso-8859-1] Tue Jul 16 13:49:03 2013
@@ -81,6 +81,61 @@
     pop esp
     ret
 
+PUBLIC _Ki386EnableCurrentLargePage@8
+_Ki386EnableCurrentLargePage@8:
+    /* Save StartAddress in eax */
+    mov eax, [esp + 4]
+
+    /* Save new CR3 value in ecx */
+    mov ecx, [esp + 8]
+
+    /* Save flags value */
+    pushfd
+
+    /* Disable interrupts */
+    cli
+
+    /* Compute linear address */
+    sub eax, offset _Ki386EnableCurrentLargePage@8
+    add eax, offset _Ki386LargePageIdentityLabel
+
+    /* Save old CR3 in edx and replace with a new one */
+    mov edx, cr3
+    mov cr3, ecx
+
+    /* Jump to the next instruction but in linear mapping */
+    jmp eax
+
+_Ki386LargePageIdentityLabel:
+    /* Disable paging */
+    mov eax, cr0
+    and eax, NOT CR0_PG
+    mov cr0, eax
+
+    /* Jump to the next instruction */
+    jmp $+2
+
+    /* Enable Page Size Extension in CR4 */
+    mov ecx, cr4
+    or ecx, CR4_PSE
+    mov cr4, ecx
+
+    /* Done, now re-enable paging */
+    or eax, CR0_PG
+    mov cr0, eax
+
+    /* Jump to virtual address */
+    mov eax, offset VirtualSpace
+    jmp eax
+
+VirtualSpace:
+    /* Restore CR3 contents */
+    mov cr3, edx
+
+    /* Restore flags */
+    popfd
+
+    ret 8
 
 /* FIXFIX: Move to C code ****/
 PUBLIC _Ki386SetupAndExitToV86Mode@4

Modified: trunk/reactos/ntoskrnl/ke/i386/kiinit.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/ke/i386/kiinit.c?…
==============================================================================
--- trunk/reactos/ntoskrnl/ke/i386/kiinit.c	[iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/ke/i386/kiinit.c	[iso-8859-1] Tue Jul 16 13:49:03 2013
@@ -46,12 +46,17 @@
     ULONG Dummy;
     KI_SAMPLE_MAP Samples[4];
     PKI_SAMPLE_MAP CurrentSample = Samples;
+    LARGE_IDENTITY_MAP IdentityMap;
 
     /* Check for large page support */
     if (KeFeatureBits & KF_LARGE_PAGE)
     {
-        /* FIXME: Support this */
-        DPRINT("Large Page support detected but not yet taken advantage
of\n");
+        /* Do an IPI to enable it on all CPUs */
+        if (Ki386CreateIdentityMap(&IdentityMap, Ki386EnableCurrentLargePage, 2))
+            KeIpiGenericCall(Ki386EnableTargetLargePage, (ULONG_PTR)&IdentityMap);
+
+        /* Free the pages allocated for identity map */
+        Ki386FreeIdentityMap(&IdentityMap);
     }
 
     /* Check for global page support */

Modified: trunk/reactos/ntoskrnl/ke/i386/patpge.c
URL:
http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/ke/i386/patpge.c?…
==============================================================================
--- trunk/reactos/ntoskrnl/ke/i386/patpge.c	[iso-8859-1] (original)
+++ trunk/reactos/ntoskrnl/ke/i386/patpge.c	[iso-8859-1] Tue Jul 16 13:49:03 2013
@@ -12,6 +12,9 @@
 #define NDEBUG
 #include <debug.h>
 
+#define PDE_BITS 10
+#define PTE_BITS 10
+
 /* FUNCTIONS *****************************************************************/
 
 ULONG_PTR
@@ -59,3 +62,194 @@
     /* FIXME: Support this */
     DPRINT("PAT support detected but not yet taken advantage of\n");
 }
+
+ULONG_PTR
+NTAPI
+INIT_FUNCTION
+Ki386EnableTargetLargePage(IN ULONG_PTR Context)
+{
+    PLARGE_IDENTITY_MAP IdentityMap = (PLARGE_IDENTITY_MAP)Context;
+
+    /* Call helper function with the start address and temporary page table pointer */
+    Ki386EnableCurrentLargePage(IdentityMap->StartAddress, IdentityMap->Cr3);
+
+    return 0;
+}
+
+PVOID
+NTAPI
+Ki386AllocateContiguousMemory(PLARGE_IDENTITY_MAP IdentityMap,
+                              ULONG PagesCount,
+                              BOOLEAN InLower4Gb)
+{
+    PHYSICAL_ADDRESS AddressMask;
+    PVOID Result;
+    ULONG SizeInBytes = PagesCount * PAGE_SIZE;
+
+    /* Initialize acceptable address mask to any possible address */
+    AddressMask.LowPart = 0xFFFFFFFF;
+    AddressMask.HighPart = 0xFFFFFFFF;
+
+    /* Mark out higher 4Gb if caller asked so */
+    if (InLower4Gb) AddressMask.HighPart = 0;
+
+    /* Try to allocate the memory */
+    Result = MmAllocateContiguousMemory(SizeInBytes, AddressMask);
+    if (!Result) return NULL;
+
+    /* Zero it out */
+    RtlZeroMemory(Result, SizeInBytes);
+
+    /* Track allocated pages in the IdentityMap structure */
+    IdentityMap->PagesList[IdentityMap->PagesCount] = Result;
+    IdentityMap->PagesCount++;
+
+    return Result;
+}
+
+PHYSICAL_ADDRESS
+NTAPI
+Ki386BuildIdentityBuffer(PLARGE_IDENTITY_MAP IdentityMap,
+                         PVOID StartPtr,
+                         ULONG Length)
+{
+    // TODO: Check whether all pages are below 4GB boundary
+    return MmGetPhysicalAddress(StartPtr);
+}
+
+BOOLEAN
+NTAPI
+Ki386IdentityMapMakeValid(PLARGE_IDENTITY_MAP IdentityMap,
+                          PHARDWARE_PTE Pde,
+                          PHARDWARE_PTE *PageTable)
+{
+    ULONG NewPage;
+
+    if (Pde->Valid == 0)
+    {
+        /* Invalid, so allocate a new page for it */
+        NewPage = (ULONG)Ki386AllocateContiguousMemory(IdentityMap, 1, FALSE);
+        if (!NewPage) return FALSE;
+
+        /* Set PFN to its virtual address and mark it as valid */
+        Pde->PageFrameNumber = NewPage >> PAGE_SHIFT;
+        Pde->Valid = 1;
+
+        /* Pass page table address to the caller */
+        if (PageTable) *PageTable = (PHARDWARE_PTE)NewPage;
+    }
+    else
+    {
+        /* Valid entry, just pass the page table address to the caller */
+        if (PageTable)
+            *PageTable = (PHARDWARE_PTE)(Pde->PageFrameNumber << PAGE_SHIFT);
+    }
+    
+    return TRUE;
+}
+
+BOOLEAN
+NTAPI
+Ki386MapAddress(PLARGE_IDENTITY_MAP IdentityMap,
+                ULONG_PTR VirtualPtr,
+                PHYSICAL_ADDRESS PhysicalPtr)
+{
+    PHARDWARE_PTE Pde, Pte;
+    PHARDWARE_PTE PageTable;
+
+    /* Allocate page directory on demand */
+    if (!IdentityMap->TopLevelDirectory)
+    {
+        IdentityMap->TopLevelDirectory = Ki386AllocateContiguousMemory(IdentityMap, 1,
1);
+        if (!IdentityMap->TopLevelDirectory) return FALSE;
+    }
+
+    /* Get PDE of VirtualPtr and make it writable and valid */
+    Pde = &IdentityMap->TopLevelDirectory[(VirtualPtr >> 22) & ((1
<< PDE_BITS) - 1)];
+    if (!Ki386IdentityMapMakeValid(IdentityMap, Pde, &PageTable)) return FALSE;
+    Pde->Write = 1;
+
+    /* Get PTE of VirtualPtr, make it valid, and map PhysicalPtr there */
+    Pte = &PageTable[(VirtualPtr >> 12) & ((1 << PTE_BITS) - 1)];
+    Pte->Valid = 1;
+    Pte->PageFrameNumber = PhysicalPtr.QuadPart >> PAGE_SHIFT;
+
+    return TRUE;
+}
+
+VOID
+NTAPI
+Ki386ConvertPte(PHARDWARE_PTE Pte)
+{
+    PVOID VirtualPtr;
+    PHYSICAL_ADDRESS PhysicalPtr;
+
+    /* Get virtual and physical addresses */
+    VirtualPtr = (PVOID)(Pte->PageFrameNumber << PAGE_SHIFT);
+    PhysicalPtr = MmGetPhysicalAddress(VirtualPtr);
+
+    /* Map its physical address in the page table provided by the caller */
+    Pte->PageFrameNumber = PhysicalPtr.QuadPart >> PAGE_SHIFT;
+}
+
+BOOLEAN
+NTAPI
+Ki386CreateIdentityMap(PLARGE_IDENTITY_MAP IdentityMap, PVOID StartPtr, ULONG
PagesCount)
+{
+    ULONG_PTR Ptr;
+    ULONG PteIndex;
+    PHYSICAL_ADDRESS IdentityPtr;
+
+    /* Zero out the IdentityMap contents */
+    RtlZeroMemory(IdentityMap, sizeof(LARGE_IDENTITY_MAP));
+
+    /* Get the pointer to the physical address and save it in the struct */
+    IdentityPtr = Ki386BuildIdentityBuffer(IdentityMap, StartPtr, PagesCount);
+    IdentityMap->StartAddress = IdentityPtr.LowPart;
+    if (IdentityMap->StartAddress == 0)
+    {
+        DPRINT1("Failed to get buffer for large pages identity mapping\n");
+        return FALSE;
+    }
+    DPRINT("IdentityMap->StartAddress %p\n", IdentityMap->StartAddress);
+
+    /* Map all pages */
+    for (Ptr = (ULONG_PTR)StartPtr;
+         Ptr < (ULONG_PTR)StartPtr + PagesCount * PAGE_SIZE;
+         Ptr += PAGE_SIZE, IdentityPtr.QuadPart += PAGE_SIZE)
+    {
+        /* Map virtual address */
+        if (!Ki386MapAddress(IdentityMap, Ptr, IdentityPtr)) return FALSE;
+
+        /* Map physical address */
+        if (!Ki386MapAddress(IdentityMap, IdentityPtr.LowPart, IdentityPtr)) return
FALSE;
+    }
+
+    /* Convert all PTEs in the page directory from virtual to physical,
+       because Ki386IdentityMapMakeValid mapped only virtual addresses */
+    for (PteIndex = 0; PteIndex < (PAGE_SIZE / sizeof(HARDWARE_PTE)); PteIndex++)
+    {
+        if (IdentityMap->TopLevelDirectory[PteIndex].Valid != 0)
+            Ki386ConvertPte(&IdentityMap->TopLevelDirectory[PteIndex]);
+    }
+
+    /* Save the page directory address (allocated by Ki386MapAddress) */
+    IdentityMap->Cr3 =
MmGetPhysicalAddress(IdentityMap->TopLevelDirectory).LowPart;
+
+    DPRINT("IdentityMap->Cr3 0x%x\n", IdentityMap->Cr3);
+
+    return TRUE;
+}
+
+VOID
+NTAPI
+Ki386FreeIdentityMap(PLARGE_IDENTITY_MAP IdentityMap)
+{
+    ULONG Page;
+
+    DPRINT("Freeing %d pages allocated for identity mapping\n",
IdentityMap->PagesCount);
+
+    /* Free all allocated pages, if any */
+    for (Page = 0; Page < IdentityMap->PagesCount; Page++)
+        MmFreeContiguousMemory(IdentityMap->PagesList[Page]);
+}