https://git.reactos.org/?p=reactos.git;a=commitdiff;h=4ff5d39edb2b9bf59ed78…
commit 4ff5d39edb2b9bf59ed78789b736cb276a6a249b
Author: Jérôme Gardou <jerome.gardou(a)reactos.org>
AuthorDate: Thu Mar 18 09:15:48 2021 +0100
Commit: Jérôme Gardou <zefklop(a)users.noreply.github.com>
CommitDate: Thu Mar 18 12:24:21 2021 +0100
[NTOS:MM] Use a define for the big pool table occupation rate
---
ntoskrnl/mm/ARM3/expool.c | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
diff --git a/ntoskrnl/mm/ARM3/expool.c b/ntoskrnl/mm/ARM3/expool.c
index 95bc21ad822..25ad64c62ae 100644
--- a/ntoskrnl/mm/ARM3/expool.c
+++ b/ntoskrnl/mm/ARM3/expool.c
@@ -22,6 +22,16 @@
#define POOL_BIG_TABLE_ENTRY_FREE 0x1
+/*
+ * This defines when we shrink or expand the table.
+ * 3 --> keep the number of used entries in the 33%-66% of the table capacity.
+ * 4 --> 25% - 75%
+ * etc.
+ */
+#define POOL_BIG_TABLE_USE_RATE 4
+#define POOL_BIG_TABLE_LOW_THRESHOLD (POOL_BIG_TABLE_USE_RATE * 2)
+#define POOL_BIG_TABLE_HIGH_THRESHOLD ((POOL_BIG_TABLE_USE_RATE - 1) / POOL_BIG_TABLE_USE_RATE)
+
typedef struct _POOL_DPC_CONTEXT
{
PPOOL_TRACKER_TABLE PoolTrackTable;
@@ -1626,7 +1636,7 @@ Retry:
// which implies a massive number of concurrent big pool allocations.
//
ExpPoolBigEntriesInUse++;
- if ((i >= 16) && (ExpPoolBigEntriesInUse > (TableSize * 3 / 4)))
+ if ((i >= 16) && (ExpPoolBigEntriesInUse > (TableSize * POOL_BIG_TABLE_HIGH_THRESHOLD)))
{
DPRINT("Attempting expansion since we now have %lu entries\n",
ExpPoolBigEntriesInUse);
@@ -1740,7 +1750,7 @@ ExpFindAndRemoveTagBigPages(IN PVOID Va,
/* If reaching 12.5% of the size (or whatever integer rounding gets us to),
* halve the allocation size, which will get us to 25% of space used. */
- if (ExpPoolBigEntriesInUse < (PoolBigPageTableSize / 8))
+ if (ExpPoolBigEntriesInUse < (PoolBigPageTableSize / POOL_BIG_TABLE_LOW_THRESHOLD))
{
/* Shrink the table. */
ExpReallocateBigPageTable(OldIrql, TRUE);
https://git.reactos.org/?p=reactos.git;a=commitdiff;h=8494688eebecb28797356…
commit 8494688eebecb287973569dc1261fd0741706324
Author: Jérôme Gardou <jerome.gardou(a)reactos.org>
AuthorDate: Thu Mar 18 09:05:00 2021 +0100
Commit: Jérôme Gardou <zefklop(a)users.noreply.github.com>
CommitDate: Thu Mar 18 12:24:21 2021 +0100
[NTOS:MM] Do not use atomic operations to set a bit while holding a lock.
Saving your fingers from typing it doesn't make it better code.
---
ntoskrnl/mm/ARM3/expool.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ntoskrnl/mm/ARM3/expool.c b/ntoskrnl/mm/ARM3/expool.c
index fb55f9159ba..95bc21ad822 100644
--- a/ntoskrnl/mm/ARM3/expool.c
+++ b/ntoskrnl/mm/ARM3/expool.c
@@ -1734,7 +1734,7 @@ ExpFindAndRemoveTagBigPages(IN PVOID Va,
// Set the free bit, and decrement the number of allocations. Finally, release
// the lock and return the tag that was located
//
- InterlockedIncrement((PLONG)&Entry->Va);
+ Entry->Va = (PVOID)((ULONG_PTR)Entry->Va | POOL_BIG_TABLE_ENTRY_FREE);
ExpPoolBigEntriesInUse--;
https://git.reactos.org/?p=reactos.git;a=commitdiff;h=f06b58925d9819c08bbf1…
commit f06b58925d9819c08bbf1482ca9d0435182bf656
Author: Jérôme Gardou <jerome.gardou(a)reactos.org>
AuthorDate: Mon Feb 22 17:30:24 2021 +0100
Commit: Jérôme Gardou <zefklop(a)users.noreply.github.com>
CommitDate: Thu Mar 18 12:24:21 2021 +0100
[NTOS:MM] Implement shrinking big pool allocation table
Shrink when using 1/8 of its allocated capacity (thus use 25% of it at the end of the process)
Expand when using 3/4 of its allocated capacity (thus use ~40% of it at the end of the process)
---
ntoskrnl/mm/ARM3/expool.c | 85 +++++++++++++++++++++++++++++++++++------------
1 file changed, 64 insertions(+), 21 deletions(-)
diff --git a/ntoskrnl/mm/ARM3/expool.c b/ntoskrnl/mm/ARM3/expool.c
index 1ea7ee6cc46..fb55f9159ba 100644
--- a/ntoskrnl/mm/ARM3/expool.c
+++ b/ntoskrnl/mm/ARM3/expool.c
@@ -1446,14 +1446,14 @@ ExGetPoolTagInfo(IN PSYSTEM_POOLTAG_INFORMATION SystemInformation,
}
_IRQL_requires_(DISPATCH_LEVEL)
+static
BOOLEAN
-NTAPI
-ExpExpandBigPageTable(
- _In_ _IRQL_restores_ KIRQL OldIrql)
+ExpReallocateBigPageTable(
+ _In_ _IRQL_restores_ KIRQL OldIrql,
+ _In_ BOOLEAN Shrink)
{
- ULONG OldSize = PoolBigPageTableSize;
- ULONG NewSize = 2 * OldSize;
- ULONG NewSizeInBytes;
+ SIZE_T OldSize = PoolBigPageTableSize;
+ SIZE_T NewSize, NewSizeInBytes;
PPOOL_TRACKER_BIG_PAGES NewTable;
PPOOL_TRACKER_BIG_PAGES OldTable;
ULONG i;
@@ -1465,11 +1465,41 @@ ExpExpandBigPageTable(
ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
/* Make sure we don't overflow */
- if (!NT_SUCCESS(RtlULongMult(2,
- OldSize * sizeof(POOL_TRACKER_BIG_PAGES),
- &NewSizeInBytes)))
+ if (Shrink)
+ {
+ NewSize = OldSize / 2;
+
+ /* Make sure we don't shrink too much. */
+ ASSERT(NewSize >= ExpPoolBigEntriesInUse);
+
+ NewSize = ALIGN_UP_BY(NewSize, PAGE_SIZE / sizeof(POOL_TRACKER_BIG_PAGES));
+ ASSERT(NewSize <= OldSize);
+
+ /* If there is only one page left, then keep it around. Not a failure either. */
+ if (NewSize == OldSize)
+ {
+ ASSERT(NewSize == (PAGE_SIZE / sizeof(POOL_TRACKER_BIG_PAGES)));
+ KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql);
+ return TRUE;
+ }
+ }
+ else
{
- DPRINT1("Overflow expanding big page table. Size=%lu\n", OldSize);
+ if (!NT_SUCCESS(RtlSIZETMult(2, OldSize, &NewSize)))
+ {
+ DPRINT1("Overflow expanding big page table. Size=%lu\n", OldSize);
+ KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql);
+ return FALSE;
+ }
+
+ /* Make sure we don't stupidly waste pages */
+ NewSize = ALIGN_DOWN_BY(NewSize, PAGE_SIZE / sizeof(POOL_TRACKER_BIG_PAGES));
+ ASSERT(NewSize > OldSize);
+ }
+
+ if (!NT_SUCCESS(RtlSIZETMult(sizeof(POOL_TRACKER_BIG_PAGES), NewSize, &NewSizeInBytes)))
+ {
+ DPRINT1("Overflow while calculating big page table size. Size=%lu\n", OldSize);
KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql);
return FALSE;
}
@@ -1482,7 +1512,7 @@ ExpExpandBigPageTable(
return FALSE;
}
- DPRINT("Expanding big pool tracker table to %lu entries\n", NewSize);
+ DPRINT("%s big pool tracker table to %lu entries\n", Shrink ? "Shrinking" : "Expanding", NewSize);
/* Initialize the new table */
RtlZeroMemory(NewTable, NewSizeInBytes);
@@ -1503,15 +1533,16 @@ ExpExpandBigPageTable(
}
/* Recalculate the hash due to the new table size */
- Hash = ExpComputePartialHashForAddress(OldTable[i].Va) & HashMask;
+ Hash = ExpComputePartialHashForAddress(OldTable[i].Va) % HashMask;
/* Find the location in the new table */
while (!((ULONG_PTR)NewTable[Hash].Va & POOL_BIG_TABLE_ENTRY_FREE))
{
- Hash = (Hash + 1) & HashMask;
+ if (++Hash == NewSize)
+ Hash = 0;
}
- /* We just enlarged the table, so we must have space */
+ /* We must have space */
ASSERT((ULONG_PTR)NewTable[Hash].Va & POOL_BIG_TABLE_ENTRY_FREE);
/* Finally, copy the item */
@@ -1587,20 +1618,20 @@ Retry:
//
// Add one more entry to the count, and see if we're getting within
- // 25% of the table size, at which point we'll do an expansion now
+ // 75% of the table size, at which point we'll do an expansion now
// to avoid blocking too hard later on.
//
// Note that we only do this if it's also been the 16th time that we
// keep losing the race or that we are not finding a free entry anymore,
// which implies a massive number of concurrent big pool allocations.
//
- InterlockedIncrementUL(&ExpPoolBigEntriesInUse);
- if ((i >= 16) && (ExpPoolBigEntriesInUse > (TableSize / 4)))
+ ExpPoolBigEntriesInUse++;
+ if ((i >= 16) && (ExpPoolBigEntriesInUse > (TableSize * 3 / 4)))
{
DPRINT("Attempting expansion since we now have %lu entries\n",
ExpPoolBigEntriesInUse);
ASSERT(TableSize == PoolBigPageTableSize);
- ExpExpandBigPageTable(OldIrql);
+ ExpReallocateBigPageTable(OldIrql, FALSE);
return TRUE;
}
@@ -1626,7 +1657,7 @@ Retry:
// to attempt expanding the table
//
ASSERT(TableSize == PoolBigPageTableSize);
- if (ExpExpandBigPageTable(OldIrql))
+ if (ExpReallocateBigPageTable(OldIrql, FALSE))
{
goto Retry;
}
@@ -1704,8 +1735,20 @@ ExpFindAndRemoveTagBigPages(IN PVOID Va,
// the lock and return the tag that was located
//
InterlockedIncrement((PLONG)&Entry->Va);
- InterlockedDecrementUL(&ExpPoolBigEntriesInUse);
- KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql);
+
+ ExpPoolBigEntriesInUse--;
+
+ /* If reaching 12.5% of the size (or whatever integer rounding gets us to),
+ * halve the allocation size, which will get us to 25% of space used. */
+ if (ExpPoolBigEntriesInUse < (PoolBigPageTableSize / 8))
+ {
+ /* Shrink the table. */
+ ExpReallocateBigPageTable(OldIrql, TRUE);
+ }
+ else
+ {
+ KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql);
+ }
return PoolTag;
}
https://git.reactos.org/?p=reactos.git;a=commitdiff;h=34630a06ca7416dcd98f4…
commit 34630a06ca7416dcd98f436f245be2132278754e
Author: Jérôme Gardou <jerome.gardou(a)reactos.org>
AuthorDate: Mon Feb 22 14:38:36 2021 +0100
Commit: Jérôme Gardou <zefklop(a)users.noreply.github.com>
CommitDate: Thu Mar 18 12:24:21 2021 +0100
[NTOS:MM] Do not bugcheck when we are freeing a big allocation for which we didn't manage to insert the tag
---
ntoskrnl/mm/ARM3/expool.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/ntoskrnl/mm/ARM3/expool.c b/ntoskrnl/mm/ARM3/expool.c
index 2cda0ac5b06..1ea7ee6cc46 100644
--- a/ntoskrnl/mm/ARM3/expool.c
+++ b/ntoskrnl/mm/ARM3/expool.c
@@ -2551,7 +2551,9 @@ ExFreePoolWithTag(IN PVOID P,
{
DPRINT1("Freeing pool - invalid tag specified: %.4s != %.4s\n", (char*)&TagToFree, (char*)&Tag);
#if DBG
- KeBugCheckEx(BAD_POOL_CALLER, 0x0A, (ULONG_PTR)P, Tag, TagToFree);
+ /* Do not bugcheck in case this is a big allocation for which we didn't manage to insert the tag */
+ if (Tag != ' GIB')
+ KeBugCheckEx(BAD_POOL_CALLER, 0x0A, (ULONG_PTR)P, Tag, TagToFree);
#endif
}