[sginsberg] 39214: - Inline Guarded Mutex and Fast Mutex implementations when called from within the kernel
Author: sginsberg Date: Fri Jan 30 10:45:17 2009 New Revision: 39214
URL: http://svn.reactos.org/svn/reactos?rev=39214&view=rev Log: - Inline Guarded Mutex and Fast Mutex implementations when called from within the kernel
Modified: trunk/reactos/ntoskrnl/ex/fmutex.c trunk/reactos/ntoskrnl/include/internal/ex.h trunk/reactos/ntoskrnl/include/internal/ke.h trunk/reactos/ntoskrnl/include/internal/ke_x.h trunk/reactos/ntoskrnl/include/internal/ntoskrnl.h trunk/reactos/ntoskrnl/ke/gmutex.c trunk/reactos/ntoskrnl/ke/wait.c
Modified: trunk/reactos/ntoskrnl/ex/fmutex.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/ex/fmutex.c?rev=39... ============================================================================== --- trunk/reactos/ntoskrnl/ex/fmutex.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/ex/fmutex.c [iso-8859-1] Fri Jan 30 10:45:17 2009 @@ -12,61 +12,14 @@ #define NDEBUG #include <debug.h>
-VOID -FASTCALL -KiAcquireFastMutex( - IN PFAST_MUTEX FastMutex -); - -/* PRIVATE FUNCTIONS *********************************************************/ - -FORCEINLINE -VOID -ExiAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex) -{ - PKTHREAD Thread = KeGetCurrentThread(); - - DPRINT("Sanity print: %d %d %p\n", - KeGetCurrentIrql(), Thread->CombinedApcDisable, Thread->Teb); - - /* Sanity check */ - ASSERT((KeGetCurrentIrql() == APC_LEVEL) || - (Thread->CombinedApcDisable != 0) || - (Thread->Teb == NULL) || - (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); - ASSERT(FastMutex->Owner != Thread); - - /* Decrease the count */ - if (InterlockedDecrement(&FastMutex->Count)) - { - /* Someone is still holding it, use slow path */ - KiAcquireFastMutex(FastMutex); - } - - /* Set the owner */ - FastMutex->Owner = Thread; -} - -FORCEINLINE -VOID -ExiReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) -{ - ASSERT((KeGetCurrentIrql() == APC_LEVEL) || - (KeGetCurrentThread()->CombinedApcDisable != 0) || - (KeGetCurrentThread()->Teb == NULL) || - (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); - ASSERT(FastMutex->Owner == KeGetCurrentThread()); - - /* Erase the owner */ - FastMutex->Owner = NULL; - - /* Increase the count */ - if (InterlockedIncrement(&FastMutex->Count) <= 0) - { - /* Someone was waiting for it, signal the waiter */ - KeSetEventBoostPriority(&FastMutex->Gate, NULL); - } -} +/* Undefine some macros we implement here */ +#undef ExEnterCriticalRegionAndAcquireFastMutexUnsafe +#undef ExReleaseFastMutexUnsafeAndLeaveCriticalRegion +#undef ExAcquireFastMutex +#undef ExReleaseFastMutex +#undef ExAcquireFastMutexUnsafe +#undef ExReleaseFastMutexUnsafe +#undef ExTryToAcquireFastMutex
/* PUBLIC FUNCTIONS **********************************************************/
@@ -77,11 +30,8 @@ FASTCALL ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) { - /* Enter the Critical Region */ - KeEnterCriticalRegion(); - - /* Acquire the mutex unsafely */ - ExiAcquireFastMutexUnsafe(FastMutex); + /* Call the inline */ + _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(FastMutex); }
/* @@ -91,11 +41,8 @@ FASTCALL ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex) { - /* Release the mutex unsafely */ - ExiReleaseFastMutexUnsafe(FastMutex); - - /* Leave the critical region */ - KeLeaveCriticalRegion(); + /* Call the inline */ + _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(FastMutex); }
/* @@ -105,22 +52,8 @@ FASTCALL ExAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex) { - KIRQL OldIrql; - ASSERT(KeGetCurrentIrql() <= APC_LEVEL); - - /* Raise IRQL to APC */ - KeRaiseIrql(APC_LEVEL, &OldIrql); - - /* Decrease the count */ - if (InterlockedDecrement(&FastMutex->Count) != 0) - { - /* Someone is still holding it, use slow path */ - KiAcquireFastMutex(FastMutex); - } - - /* Set the owner and IRQL */ - FastMutex->Owner = KeGetCurrentThread(); - FastMutex->OldIrql = OldIrql; + /* Call the inline */ + _ExAcquireFastMutex(FastMutex); }
/* @@ -130,22 +63,8 @@ FASTCALL ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex) { - KIRQL OldIrql; - ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); - - /* Erase the owner */ - FastMutex->Owner = NULL; - OldIrql = (KIRQL)FastMutex->OldIrql; - - /* Increase the count */ - if (InterlockedIncrement(&FastMutex->Count) <= 0) - { - /* Someone was waiting for it, signal the waiter */ - KeSetEventBoostPriority(&FastMutex->Gate, IO_NO_INCREMENT); - } - - /* Lower IRQL back */ - KeLowerIrql(OldIrql); + /* Call the inline */ + _ExReleaseFastMutex(FastMutex); }
/* @@ -156,7 +75,7 @@ ExAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) { /* Acquire the mutex unsafely */ - ExiAcquireFastMutexUnsafe(FastMutex); + _ExAcquireFastMutexUnsafe(FastMutex); }
/* @@ -167,7 +86,7 @@ ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) { /* Release the mutex unsafely */ - ExiReleaseFastMutexUnsafe(FastMutex); + _ExReleaseFastMutexUnsafe(FastMutex); }
/* @@ -177,26 +96,8 @@ FASTCALL ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex) { - KIRQL OldIrql; - ASSERT(KeGetCurrentIrql() <= APC_LEVEL); - - /* Raise to APC_LEVEL */ - KeRaiseIrql(APC_LEVEL, &OldIrql); - - /* Check if we can quickly acquire it */ - if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1) - { - /* We have, set us as owners */ - FastMutex->Owner = KeGetCurrentThread(); - FastMutex->OldIrql = OldIrql; - return TRUE; - } - else - { - /* Acquire attempt failed */ - KeLowerIrql(OldIrql); - return FALSE; - } + /* Call the inline */ + return _ExTryToAcquireFastMutex(FastMutex); }
/* EOF */
Modified: trunk/reactos/ntoskrnl/include/internal/ex.h URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/include/internal/e... ============================================================================== --- trunk/reactos/ntoskrnl/include/internal/ex.h [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/include/internal/ex.h [iso-8859-1] Fri Jan 30 10:45:17 2009 @@ -994,6 +994,146 @@ } }
+/* FAST MUTEX INLINES *********************************************************/ + +FORCEINLINE +VOID +_ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex) +{ + PKTHREAD Thread = KeGetCurrentThread(); + + /* Sanity check */ + ASSERT((KeGetCurrentIrql() == APC_LEVEL) || + (Thread->CombinedApcDisable != 0) || + (Thread->Teb == NULL) || + (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); + ASSERT(FastMutex->Owner != Thread); + + /* Decrease the count */ + if (InterlockedDecrement(&FastMutex->Count)) + { + /* Someone is still holding it, use slow path */ + KiAcquireFastMutex(FastMutex); + } + + /* Set the owner */ + FastMutex->Owner = Thread; +} + +FORCEINLINE +VOID +_ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) +{ + ASSERT((KeGetCurrentIrql() == APC_LEVEL) || + (KeGetCurrentThread()->CombinedApcDisable != 0) || + (KeGetCurrentThread()->Teb == NULL) || + (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); + ASSERT(FastMutex->Owner == KeGetCurrentThread()); + + /* Erase the owner */ + FastMutex->Owner = NULL; + + /* Increase the count */ + if (InterlockedIncrement(&FastMutex->Count) <= 0) + { + /* Someone was waiting for it, signal the waiter */ + KeSetEventBoostPriority(&FastMutex->Gate, NULL); + } +} + +FORCEINLINE +VOID +_ExAcquireFastMutex(IN PFAST_MUTEX FastMutex) +{ + KIRQL OldIrql; + ASSERT(KeGetCurrentIrql() <= APC_LEVEL); + + /* Raise IRQL to APC */ + KeRaiseIrql(APC_LEVEL, &OldIrql); + + /* Decrease the count */ + if (InterlockedDecrement(&FastMutex->Count)) + { + /* Someone is still holding it, use slow path */ + KiAcquireFastMutex(FastMutex); + } + + /* Set the owner and IRQL */ + FastMutex->Owner = KeGetCurrentThread(); + FastMutex->OldIrql = OldIrql; +} + +FORCEINLINE +VOID +_ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex) +{ + KIRQL OldIrql; + ASSERT(KeGetCurrentIrql() == APC_LEVEL); + + /* Erase the owner */ + FastMutex->Owner = NULL; + OldIrql = (KIRQL)FastMutex->OldIrql; + + /* Increase the count */ + if (InterlockedIncrement(&FastMutex->Count) <= 0) + { + /* Someone was waiting for it, signal the waiter */ + KeSetEventBoostPriority(&FastMutex->Gate, NULL); + } + + /* Lower IRQL back */ + KeLowerIrql(OldIrql); +} + +FORCEINLINE +BOOLEAN +_ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex) +{ + KIRQL OldIrql; + ASSERT(KeGetCurrentIrql() <= APC_LEVEL); + + /* Raise to APC_LEVEL */ + KeRaiseIrql(APC_LEVEL, &OldIrql); + + /* Check if we can quickly acquire it */ + if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1) + { + /* We have, set us as owners */ + FastMutex->Owner = KeGetCurrentThread(); + FastMutex->OldIrql = OldIrql; + return TRUE; + } + else + { + /* Acquire attempt failed */ + KeLowerIrql(OldIrql); + YieldProcessor(); + return FALSE; + } +} + +FORCEINLINE +VOID +_ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) +{ + /* Enter the Critical Region */ + KeEnterCriticalRegion(); + + /* Acquire the mutex unsafely */ + _ExAcquireFastMutexUnsafe(FastMutex); +} + +FORCEINLINE +VOID +_ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex) +{ + /* Release the mutex unsafely */ + _ExReleaseFastMutexUnsafe(FastMutex); + + /* Leave the critical region */ + KeLeaveCriticalRegion(); +} + /* OTHER FUNCTIONS **********************************************************/
BOOLEAN
Modified: trunk/reactos/ntoskrnl/include/internal/ke.h URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/include/internal/k... ============================================================================== --- trunk/reactos/ntoskrnl/include/internal/ke.h [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/include/internal/ke.h [iso-8859-1] Fri Jan 30 10:45:17 2009 @@ -312,7 +312,15 @@
VOID FASTCALL -KiAcquireGuardedMutexContented(PKGUARDED_MUTEX GuardedMutex); +KiAcquireGuardedMutex( + IN OUT PKGUARDED_MUTEX GuardedMutex +); + +VOID +FASTCALL +KiAcquireFastMutex( + IN PFAST_MUTEX FastMutex +);
/* gate.c **********************************************************************/
Modified: trunk/reactos/ntoskrnl/include/internal/ke_x.h URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/include/internal/k... ============================================================================== --- trunk/reactos/ntoskrnl/include/internal/ke_x.h [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/include/internal/ke_x.h [iso-8859-1] Fri Jan 30 10:45:17 2009 @@ -169,10 +169,6 @@ } \ } \ } - -// -// TODO: Guarded Mutex Routines -//
// // Enters a Critical Region @@ -1710,3 +1706,178 @@ /* Return the new priority */ return Priority; } + +// +// Guarded Mutex Routines +// +FORCEINLINE +VOID +_KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex) +{ + /* Setup the Initial Data */ + GuardedMutex->Count = GM_LOCK_BIT; + GuardedMutex->Owner = NULL; + GuardedMutex->Contention = 0; + + /* Initialize the Wait Gate */ + KeInitializeGate(&GuardedMutex->Gate); +} + +FORCEINLINE +VOID +_KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex) +{ + PKTHREAD Thread = KeGetCurrentThread(); + + /* Sanity checks */ + ASSERT((KeGetCurrentIrql() == APC_LEVEL) || + (Thread->SpecialApcDisable < 0) || + (Thread->Teb == NULL) || + (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); + ASSERT(GuardedMutex->Owner != Thread); + + /* Remove the lock */ + if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V)) + { + /* The Guarded Mutex was already locked, enter contented case */ + KiAcquireGuardedMutex(GuardedMutex); + } + + /* Set the Owner */ + GuardedMutex->Owner = Thread; +} + +FORCEINLINE +VOID +_KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex) +{ + LONG OldValue, NewValue; + + /* Sanity checks */ + ASSERT((KeGetCurrentIrql() == APC_LEVEL) || + (KeGetCurrentThread()->SpecialApcDisable < 0) || + (KeGetCurrentThread()->Teb == NULL) || + (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); + ASSERT(GuardedMutex->Owner == KeGetCurrentThread()); + + /* Destroy the Owner */ + GuardedMutex->Owner = NULL; + + /* Add the Lock Bit */ + OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT); + ASSERT((OldValue & GM_LOCK_BIT) == 0); + + /* Check if it was already locked, but not woken */ + if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN)) + { + /* Update the Oldvalue to what it should be now */ + OldValue += GM_LOCK_BIT; + + /* The mutex will be woken, minus one waiter */ + NewValue = OldValue + GM_LOCK_WAITER_WOKEN - + GM_LOCK_WAITER_INC; + + /* Remove the Woken bit */ + if (InterlockedCompareExchange(&GuardedMutex->Count, + NewValue, + OldValue) == OldValue) + { + /* Signal the Gate */ + KeSignalGateBoostPriority(&GuardedMutex->Gate); + } + } +} + +FORCEINLINE +VOID +_KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex) +{ + PKTHREAD Thread = KeGetCurrentThread(); + + /* Sanity checks */ + ASSERT(KeGetCurrentIrql() <= APC_LEVEL); + ASSERT(GuardedMutex->Owner != Thread); + + /* Disable Special APCs */ + KeEnterGuardedRegion(); + + /* Remove the lock */ + if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V)) + { + /* The Guarded Mutex was already locked, enter contented case */ + KiAcquireGuardedMutex(GuardedMutex); + } + + /* Set the Owner and Special APC Disable state */ + GuardedMutex->Owner = Thread; + GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable; +} + +FORCEINLINE +VOID +_KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) +{ + LONG OldValue, NewValue; + + /* Sanity checks */ + ASSERT(KeGetCurrentIrql() <= APC_LEVEL); + ASSERT(GuardedMutex->Owner == KeGetCurrentThread()); + ASSERT(KeGetCurrentThread()->SpecialApcDisable == + GuardedMutex->SpecialApcDisable); + + /* Destroy the Owner */ + GuardedMutex->Owner = NULL; + + /* Add the Lock Bit */ + OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT); + ASSERT((OldValue & GM_LOCK_BIT) == 0); + + /* Check if it was already locked, but not woken */ + if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN)) + { + /* Update the Oldvalue to what it should be now */ + OldValue += GM_LOCK_BIT; + + /* The mutex will be woken, minus one waiter */ + NewValue = OldValue + GM_LOCK_WAITER_WOKEN - + GM_LOCK_WAITER_INC; + + /* Remove the Woken bit */ + if (InterlockedCompareExchange(&GuardedMutex->Count, + NewValue, + OldValue) == OldValue) + { + /* Signal the Gate */ + KeSignalGateBoostPriority(&GuardedMutex->Gate); + } + } + + /* Re-enable APCs */ + KeLeaveGuardedRegion(); +} + +FORCEINLINE +BOOLEAN +_KeTryToAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) +{ + PKTHREAD Thread = KeGetCurrentThread(); + + /* Block APCs */ + KeEnterGuardedRegion(); + + /* Remove the lock */ + if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V)) + { + /* Re-enable APCs */ + KeLeaveGuardedRegion(); + YieldProcessor(); + + /* Return failure */ + return FALSE; + } + + /* Set the Owner and APC State */ + GuardedMutex->Owner = Thread; + GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable; + return TRUE; +}
Modified: trunk/reactos/ntoskrnl/include/internal/ntoskrnl.h URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/include/internal/n... ============================================================================== --- trunk/reactos/ntoskrnl/include/internal/ntoskrnl.h [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/include/internal/ntoskrnl.h [iso-8859-1] Fri Jan 30 10:45:17 2009 @@ -41,6 +41,24 @@ #define InterlockedExchangeAdd _InterlockedExchangeAdd #define InterlockedOr _InterlockedOr #define InterlockedAnd _InterlockedAnd + +// +// Use inlined versions of fast/guarded mutex routines +// +#define ExEnterCriticalRegionAndAcquireFastMutexUnsafe _ExEnterCriticalRegionAndAcquireFastMutexUnsafe +#define ExReleaseFastMutexUnsafeAndLeaveCriticalRegion _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion +#define ExAcquireFastMutex _ExAcquireFastMutex +#define ExReleaseFastMutex _ExReleaseFastMutex +#define ExAcquireFastMutexUnsafe _ExAcquireFastMutexUnsafe +#define ExReleaseFastMutexUnsafe _ExReleaseFastMutexUnsafe +#define ExTryToAcquireFastMutex _ExTryToAcquireFastMutex + +#define KeInitializeGuardedMutex _KeInitializeGuardedMutex +#define KeAcquireGuardedMutex _KeAcquireGuardedMutex +#define KeReleaseGuardedMutex _KeReleaseGuardedMutex +#define KeAcquireGuardedMutexUnsafe _KeAcquireGuardedMutexUnsafe +#define KeReleaseGuardedMutexUnsafe _KeReleaseGuardedMutexUnsafe +#define KeTryToAcquireGuardedMutex _KeTryToAcquireGuardedMutex
#include "ke.h" #include "ob.h"
Modified: trunk/reactos/ntoskrnl/ke/gmutex.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/ke/gmutex.c?rev=39... ============================================================================== --- trunk/reactos/ntoskrnl/ke/gmutex.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/ke/gmutex.c [iso-8859-1] Fri Jan 30 10:45:17 2009 @@ -13,128 +13,13 @@ #define NDEBUG #include <debug.h>
-/* PRIVATE FUNCTIONS *********************************************************/ - -VOID -FASTCALL -KiAcquireGuardedMutexContented(IN OUT PKGUARDED_MUTEX GuardedMutex) -{ - ULONG BitsToRemove, BitsToAdd; - LONG OldValue, NewValue; - - /* Increase the contention count */ - GuardedMutex->Contention++; - - /* Start by unlocking the Guarded Mutex */ - BitsToRemove = GM_LOCK_BIT; - BitsToAdd = GM_LOCK_WAITER_INC; - - /* Start change loop */ - for (;;) - { - /* Loop sanity checks */ - ASSERT((BitsToRemove == GM_LOCK_BIT) || - (BitsToRemove == (GM_LOCK_BIT | GM_LOCK_WAITER_WOKEN))); - ASSERT((BitsToAdd == GM_LOCK_WAITER_INC) || - (BitsToAdd == GM_LOCK_WAITER_WOKEN)); - - /* Get the Count Bits */ - OldValue = GuardedMutex->Count; - - /* Start internal bit change loop */ - for (;;) - { - /* Check if the Guarded Mutex is locked */ - if (OldValue & GM_LOCK_BIT) - { - /* Sanity check */ - ASSERT((BitsToRemove == GM_LOCK_BIT) || - ((OldValue & GM_LOCK_WAITER_WOKEN) != 0)); - - /* Unlock it by removing the Lock Bit */ - NewValue = OldValue ^ BitsToRemove; - NewValue = InterlockedCompareExchange(&GuardedMutex->Count, - NewValue, - OldValue); - if (NewValue == OldValue) return; - } - else - { - /* The Guarded Mutex isn't locked, so simply set the bits */ - NewValue = OldValue + BitsToAdd; - NewValue = InterlockedCompareExchange(&GuardedMutex->Count, - NewValue, - OldValue); - if (NewValue == OldValue) break; - } - - /* Old value changed, loop again */ - OldValue = NewValue; - } - - /* Now we have to wait for it */ - KeWaitForGate(&GuardedMutex->Gate, WrGuardedMutex, KernelMode); - ASSERT((GuardedMutex->Count & GM_LOCK_WAITER_WOKEN) != 0); - - /* Ok, the wait is done, so set the new bits */ - BitsToRemove = GM_LOCK_BIT | GM_LOCK_WAITER_WOKEN; - BitsToAdd = GM_LOCK_WAITER_WOKEN; - - /* We depend on these bits being just right */ - C_ASSERT((GM_LOCK_WAITER_WOKEN * 2) == GM_LOCK_WAITER_INC); - } -} - -FORCEINLINE -VOID -FASTCALL -KiAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) -{ - BOOLEAN OldBit; - - /* Remove the lock */ - OldBit = InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V); - if (!OldBit) - { - /* The Guarded Mutex was already locked, enter contented case */ - KiAcquireGuardedMutexContented(GuardedMutex); - } -} - -FORCEINLINE -VOID -FASTCALL -KiReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) -{ - LONG OldValue, NewValue; - - /* Destroy the Owner */ - GuardedMutex->Owner = NULL; - - /* Add the Lock Bit */ - OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT); - ASSERT((OldValue & GM_LOCK_BIT) == 0); - - /* Check if it was already locked, but not woken */ - if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN)) - { - /* Update the Oldvalue to what it should be now */ - OldValue += GM_LOCK_BIT; - - /* The mutex will be woken, minus one waiter */ - NewValue = OldValue + GM_LOCK_WAITER_WOKEN - - GM_LOCK_WAITER_INC; - - /* Remove the Woken bit */ - if (InterlockedCompareExchange(&GuardedMutex->Count, - NewValue, - OldValue) == OldValue) - { - /* Signal the Gate */ - KeSignalGateBoostPriority(&GuardedMutex->Gate); - } - } -} +/* Undefine some macros we implement here */ +#undef KeInitializeGuardedMutex +#undef KeAcquireGuardedMutex +#undef KeReleaseGuardedMutex +#undef KeAcquireGuardedMutexUnsafe +#undef KeReleaseGuardedMutexUnsafe +#undef KeTryToAcquireGuardedMutex
/* PUBLIC FUNCTIONS **********************************************************/
@@ -145,13 +30,30 @@ FASTCALL KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex) { - /* Setup the Initial Data */ - GuardedMutex->Count = GM_LOCK_BIT; - GuardedMutex->Owner = NULL; - GuardedMutex->Contention = 0; + /* Call the inline */ + _KeInitializeGuardedMutex(GuardedMutex); +}
- /* Initialize the Wait Gate */ - KeInitializeGate(&GuardedMutex->Gate); +/* + * @implemented + */ +VOID +FASTCALL +KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex) +{ + /* Call the inline */ + _KeAcquireGuardedMutex(GuardedMutex); +} + +/* + * @implemented + */ +VOID +FASTCALL +KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) +{ + /* Call the inline */ + _KeReleaseGuardedMutex(GuardedMutex); }
/* @@ -161,20 +63,8 @@ FASTCALL KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex) { - PKTHREAD Thread = KeGetCurrentThread(); - - /* Sanity checks */ - ASSERT((KeGetCurrentIrql() == APC_LEVEL) || - (Thread->SpecialApcDisable < 0) || - (Thread->Teb == NULL) || - (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); - ASSERT(GuardedMutex->Owner != Thread); - - /* Do the actual acquire */ - KiAcquireGuardedMutex(GuardedMutex); - - /* Set the Owner */ - GuardedMutex->Owner = Thread; + /* Call the inline */ + _KeAcquireGuardedMutexUnsafe(GuardedMutex); }
/* @@ -184,59 +74,8 @@ FASTCALL KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex) { - /* Sanity checks */ - ASSERT((KeGetCurrentIrql() == APC_LEVEL) || - (KeGetCurrentThread()->SpecialApcDisable < 0) || - (KeGetCurrentThread()->Teb == NULL) || - (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); - ASSERT(GuardedMutex->Owner == KeGetCurrentThread()); - - /* Release the mutex */ - KiReleaseGuardedMutex(GuardedMutex); -} - -/* - * @implemented - */ -VOID -FASTCALL -KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex) -{ - PKTHREAD Thread = KeGetCurrentThread(); - - /* Sanity checks */ - ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); - ASSERT(GuardedMutex->Owner != Thread); - - /* Disable Special APCs */ - KeEnterGuardedRegion(); - - /* Do the actual acquire */ - KiAcquireGuardedMutex(GuardedMutex); - - /* Set the Owner and Special APC Disable state */ - GuardedMutex->Owner = Thread; - GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable; -} - -/* - * @implemented - */ -VOID -FASTCALL -KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) -{ - /* Sanity checks */ - ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); - ASSERT(GuardedMutex->Owner == KeGetCurrentThread()); - ASSERT(GuardedMutex->SpecialApcDisable == - KeGetCurrentThread()->SpecialApcDisable); - - /* Release the mutex */ - KiReleaseGuardedMutex(GuardedMutex); - - /* Re-enable APCs */ - KeLeaveGuardedRegion(); + /* Call the inline */ + _KeReleaseGuardedMutexUnsafe(GuardedMutex); }
/* @@ -246,28 +85,8 @@ FASTCALL KeTryToAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) { - PKTHREAD Thread = KeGetCurrentThread(); - BOOLEAN OldBit; - - /* Block APCs */ - KeEnterGuardedRegion(); - - /* Remove the lock */ - OldBit = InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V); - if (!OldBit) - { - /* Re-enable APCs */ - KeLeaveGuardedRegion(); - YieldProcessor(); - - /* Return failure */ - return FALSE; - } - - /* Set the Owner and APC State */ - GuardedMutex->Owner = Thread; - GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable; - return TRUE; + /* Call the inline */ + return _KeTryToAcquireGuardedMutex(GuardedMutex); }
/**
Modified: trunk/reactos/ntoskrnl/ke/wait.c URL: http://svn.reactos.org/svn/reactos/trunk/reactos/ntoskrnl/ke/wait.c?rev=3921... ============================================================================== --- trunk/reactos/ntoskrnl/ke/wait.c [iso-8859-1] (original) +++ trunk/reactos/ntoskrnl/ke/wait.c [iso-8859-1] Fri Jan 30 10:45:17 2009 @@ -115,6 +115,76 @@ KernelMode, FALSE, NULL); +} + +VOID +FASTCALL +KiAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) +{ + ULONG BitsToRemove, BitsToAdd; + LONG OldValue, NewValue; + + /* Increase the contention count */ + GuardedMutex->Contention++; + + /* Start by unlocking the Guarded Mutex */ + BitsToRemove = GM_LOCK_BIT; + BitsToAdd = GM_LOCK_WAITER_INC; + + /* Start change loop */ + for (;;) + { + /* Loop sanity checks */ + ASSERT((BitsToRemove == GM_LOCK_BIT) || + (BitsToRemove == (GM_LOCK_BIT | GM_LOCK_WAITER_WOKEN))); + ASSERT((BitsToAdd == GM_LOCK_WAITER_INC) || + (BitsToAdd == GM_LOCK_WAITER_WOKEN)); + + /* Get the Count Bits */ + OldValue = GuardedMutex->Count; + + /* Start internal bit change loop */ + for (;;) + { + /* Check if the Guarded Mutex is locked */ + if (OldValue & GM_LOCK_BIT) + { + /* Sanity check */ + ASSERT((BitsToRemove == GM_LOCK_BIT) || + ((OldValue & GM_LOCK_WAITER_WOKEN) != 0)); + + /* Unlock it by removing the Lock Bit */ + NewValue = OldValue ^ BitsToRemove; + NewValue = InterlockedCompareExchange(&GuardedMutex->Count, + NewValue, + OldValue); + if (NewValue == OldValue) return; + } + else + { + /* The Guarded Mutex isn't locked, so simply set the bits */ + NewValue = OldValue + BitsToAdd; + NewValue = InterlockedCompareExchange(&GuardedMutex->Count, + NewValue, + OldValue); + if (NewValue == OldValue) break; + } + + /* Old value changed, loop again */ + OldValue = NewValue; + } + + /* Now we have to wait for it */ + KeWaitForGate(&GuardedMutex->Gate, WrGuardedMutex, KernelMode); + ASSERT((GuardedMutex->Count & GM_LOCK_WAITER_WOKEN) != 0); + + /* Ok, the wait is done, so set the new bits */ + BitsToRemove = GM_LOCK_BIT | GM_LOCK_WAITER_WOKEN; + BitsToAdd = GM_LOCK_WAITER_WOKEN; + + /* We depend on these bits being just right */ + C_ASSERT((GM_LOCK_WAITER_WOKEN * 2) == GM_LOCK_WAITER_INC); + } }
//
-
sginsberg＠svn.reactos.org