[sir_richard] 55874: [NTOS]: Implement pool tagging for large allocations too. Once again, no expansion is implemented.
Author: sir_richard Date: Sun Feb 26 05:53:53 2012 New Revision: 55874
URL: http://svn.reactos.org/svn/reactos?rev=55874&view=rev Log: [NTOS]: Implement pool tagging for large allocations too. Once again, no expansion is implemented.
Modified: trunk/reactos/ntoskrnl/mm/ARM3/expool.c trunk/reactos/ntoskrnl/mm/ARM3/miarm.h trunk/reactos/ntoskrnl/mm/ARM3/pool.c
Modified: trunk/reactos/ntoskrnl/mm/ARM3/expool.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/ARM3/expool.c?r... ============================================================================== --- trunk/reactos/ntoskrnl/mm/ARM3/expool.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/mm/ARM3/expool.c [iso-8859-1] Sun Feb 26 05:53:53 2012 @@ -19,6 +19,8 @@ #undef ExAllocatePoolWithQuotaTag
/* GLOBALS ********************************************************************/ + +#define POOL_BIG_TABLE_ENTRY_FREE 0x1
typedef struct _POOL_DPC_CONTEXT { @@ -40,6 +42,8 @@ KSPIN_LOCK ExpTaggedPoolLock; ULONG PoolHitTag; BOOLEAN ExStopBadTags; +KSPIN_LOCK ExpLargePoolTableLock; +LONG ExpPoolBigEntriesInUse;
/* Pool block/header/list access macros */ #define POOL_ENTRY(x) (PPOOL_HEADER)((ULONG_PTR)(x) - sizeof(POOL_HEADER)) @@ -311,6 +315,24 @@ // ULONGLONG Result = 40543 * Tag; return BucketMask & (Result ^ (Result >> 32)); +} + +FORCEINLINE +ULONG +ExpComputePartialHashForAddress(IN PVOID BaseAddress) +{ + ULONG Result; + // + // Compute the hash by converting the address into a page number, and then + // XORing each nibble with the next one. + // + // We do *NOT* AND with the bucket mask at this point because big table expansion + // might happen. Therefore, the final step of the hash must be performed + // while holding the expansion pushlock, and this is why we call this a + // "partial" hash only. + // + Result = (ULONG_PTR)BaseAddress >> PAGE_SHIFT; + return (Result >> 24) ^ (Result >> 16) ^ (Result >> 8) ^ Result; }
/* PRIVATE FUNCTIONS **********************************************************/ @@ -1120,6 +1142,169 @@ return Status; }
+BOOLEAN +NTAPI +ExpAddTagForBigPages(IN PVOID Va, + IN ULONG Key, + IN ULONG NumberOfPages, + IN POOL_TYPE PoolType) +{ + ULONG Hash, i = 0; + PVOID OldVa; + KIRQL OldIrql; + SIZE_T TableSize; + PPOOL_TRACKER_BIG_PAGES Entry, EntryEnd, EntryStart; + ASSERT(((ULONG_PTR)Va & POOL_BIG_TABLE_ENTRY_FREE) == 0); + ASSERT(!(PoolType & SESSION_POOL_MASK)); + + // + // As the table is expandable, these values must only be read after acquiring + // the lock to avoid a teared access during an expansion + // + Hash = ExpComputePartialHashForAddress(Va); + KeAcquireSpinLock(&ExpLargePoolTableLock, &OldIrql); + Hash &= PoolBigPageTableHash; + TableSize = PoolBigPageTableSize; + + // + // We loop from the current hash bucket to the end of the table, and then + // rollover to hash bucket 0 and keep going from there. If we return back + // to the beginning, then we attempt expansion at the bottom of the loop + // + EntryStart = Entry = &PoolBigPageTable[Hash]; + EntryEnd = &PoolBigPageTable[TableSize]; + do + { + // + // Make sure that this is a free entry and attempt to atomically make the + // entry busy now + // + OldVa = Entry->Va; + if (((ULONG_PTR)OldVa & POOL_BIG_TABLE_ENTRY_FREE) && + (InterlockedCompareExchangePointer(&Entry->Va, Va, OldVa) == OldVa)) + { + // + // We now own this entry, write down the size and the pool tag + // + Entry->Key = Key; + Entry->NumberOfPages = NumberOfPages; + + // + // Add one more entry to the count, and see if we're getting within + // 25% of the table size, at which point we'll do an expansion now + // to avoid blocking too hard later on. + // + // Note that we only do this if it's also been the 16th time that we + // keep losing the race or that we are not finding a free entry anymore, + // which implies a massive number of concurrent big pool allocations. + // + InterlockedIncrement(&ExpPoolBigEntriesInUse); + if ((i >= 16) && (ExpPoolBigEntriesInUse > (TableSize / 4))) + { + DPRINT1("Should attempt expansion since we now have %d entries\n", + ExpPoolBigEntriesInUse); + } + + // + // We have our entry, return + // + KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql); + return TRUE; + } + + // + // We don't have our entry yet, so keep trying, making the entry list + // circular if we reach the last entry. We'll eventually break out of + // the loop once we've rolled over and returned back to our original + // hash bucket + // + i++; + if (++Entry >= EntryEnd) Entry = &PoolBigPageTable[0]; + } while (Entry != EntryStart); + + // + // This means there's no free hash buckets whatsoever, so we would now have + // to attempt expanding the table + // + DPRINT1("Big pool expansion needed, not implemented!"); + KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql); + return FALSE; +} + +ULONG +NTAPI +ExpFindAndRemoveTagBigPages(IN PVOID Va, + OUT PULONG BigPages, + IN POOL_TYPE PoolType) +{ + BOOLEAN FirstTry = TRUE; + SIZE_T TableSize; + KIRQL OldIrql; + ULONG PoolTag, Hash; + PPOOL_TRACKER_BIG_PAGES Entry; + ASSERT(((ULONG_PTR)Va & POOL_BIG_TABLE_ENTRY_FREE) == 0); + ASSERT(!(PoolType & SESSION_POOL_MASK)); + + // + // As the table is expandable, these values must only be read after acquiring + // the lock to avoid a teared access during an expansion + // + Hash = ExpComputePartialHashForAddress(Va); + KeAcquireSpinLock(&ExpLargePoolTableLock, &OldIrql); + Hash &= PoolBigPageTableHash; + TableSize = PoolBigPageTableSize; + + // + // Loop while trying to find this big page allocation + // + while (PoolBigPageTable[Hash].Va != Va) + { + // + // Increment the size until we go past the end of the table + // + if (++Hash >= TableSize) + { + // + // Is this the second time we've tried? + // + if (!FirstTry) + { + // + // This means it was never inserted into the pool table and it + // received the special "BIG" tag -- return that and return 0 + // so that the code can ask Mm for the page count instead + // + KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql); + *BigPages = 0; + return ' GIB'; + } + + // + // The first time this happens, reset the hash index and try again + // + Hash = 0; + FirstTry = FALSE; + } + } + + // + // Now capture all the information we need from the entry, since after we + // release the lock, the data can change + // + Entry = &PoolBigPageTable[Hash]; + *BigPages = Entry->NumberOfPages; + PoolTag = Entry->Key; + + // + // Set the free bit, and decrement the number of allocations. Finally, release + // the lock and return the tag that was located + // + InterlockedIncrement((PLONG)&Entry->Va); + InterlockedDecrement(&ExpPoolBigEntriesInUse); + KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql); + return PoolTag; +} + /* PUBLIC FUNCTIONS ***********************************************************/
/* @@ -1169,9 +1354,18 @@ if (NumberOfBytes > POOL_MAX_ALLOC) { // - // Then just return the number of pages requested - // - return MiAllocatePoolPages(PoolType, NumberOfBytes); + // Allocate pages for it + // + Entry = MiAllocatePoolPages(PoolType, NumberOfBytes); + if (!Entry) return NULL; + + // + // Add a tag for the big page allocation and switch to the generic "BIG" + // tag if we failed to do so, then insert a tracker for this alloation. + // + if (!ExpAddTagForBigPages(Entry, Tag, BYTES_TO_PAGES(NumberOfBytes), PoolType)) Tag = ' GIB'; + ExpInsertPoolTracker(Tag, ROUND_TO_PAGES(NumberOfBytes), PoolType); + return Entry; }
// @@ -1465,6 +1659,7 @@ PPOOL_DESCRIPTOR PoolDesc; ULONG Tag; BOOLEAN Combined = FALSE; + PFN_NUMBER PageCount;
// // Check if it was allocated from a special pool @@ -1479,10 +1674,40 @@ }
// - // Quickly deal with big page allocations + // Check if this is a big page allocation // if (PAGE_ALIGN(P) == P) { + // + // We need to find the tag for it, so first we need to find out what + // kind of allocation this was (paged or nonpaged), then we can go + // ahead and try finding the tag for it. Remember to get rid of the + // PROTECTED_POOL tag if it's found. + // + // Note that if at insertion time, we failed to add the tag for a big + // pool allocation, we used a special tag called 'BIG' to identify the + // allocation, and we may get this tag back. In this scenario, we must + // manually get the size of the allocation by actually counting through + // the PFN database. + // + PoolType = MmDeterminePoolType(P); + Tag = ExpFindAndRemoveTagBigPages(P, &PageCount, PoolType); + if (!Tag) + { + DPRINT1("We do not know the size of this allocation. This is not yet supported\n"); + ASSERT(Tag == ' GIB'); + PageCount = 1; // We are going to lie! This might screw up accounting? + } + else if (Tag & PROTECTED_POOL) + { + Tag &= ~PROTECTED_POOL; + } + + // + // We have our tag and our page count, so we can go ahead and remove this + // tracker now + // + ExpRemovePoolTracker(Tag, PageCount << PAGE_SHIFT, PoolType); MiFreePoolPages(P); return; }
Modified: trunk/reactos/ntoskrnl/mm/ARM3/miarm.h URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/ARM3/miarm.h?re... ============================================================================== --- trunk/reactos/ntoskrnl/mm/ARM3/miarm.h [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/mm/ARM3/miarm.h [iso-8859-1] Sun Feb 26 05:53:53 2012 @@ -1443,6 +1443,12 @@ IN ULONG_PTR Vpn );
+POOL_TYPE +NTAPI +MmDeterminePoolType( + IN PVOID PoolAddress +); + // // MiRemoveZeroPage will use inline code to zero out the page manually if only // free pages are available. In some scenarios, we don't/can't run that piece of
Modified: trunk/reactos/ntoskrnl/mm/ARM3/pool.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/mm/ARM3/pool.c?rev... ============================================================================== --- trunk/reactos/ntoskrnl/mm/ARM3/pool.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/mm/ARM3/pool.c [iso-8859-1] Sun Feb 26 05:53:53 2012 @@ -367,6 +367,17 @@ MiInitializeSystemPtes(PointerPte + 1, MiExpansionPoolPagesInitialCharge, NonPagedPoolExpansion); +} + +POOL_TYPE +NTAPI +MmDeterminePoolType(IN PVOID PoolAddress) +{ + // + // Use a simple bounds check + // + return (PoolAddress >= MmPagedPoolStart) && (PoolAddress <= MmPagedPoolEnd) ? + PagedPool : NonPagedPool; }
PVOID
-
sir_richard＠svn.reactos.org