- Rewrite Executive Interlocked Functions to actually work as supposed
and also immensly increase their speed, most notably on non-SMP
machines:
* The functions now disable interrupts as they should. This is a core
functionality of the ExInterlocked functions which we did not do.
* The functions now preserve EFLAGS as they should. This is another
core functionality and point of ExInterlocked functions, which we did
not do.
* Use KiAcquire/ReleaseSpinlock equivalents instead of going through
HAL. Also make them macros which can be later used in other assembly
files.
* Write optimized versions for non-SMP machines. Namely, the spinlock
macros are coded to be no-ops on UP machines, since they are not needed.
* Write all functions in assembly, so we can cleanly optimize them and
properly restore/save EFLAGS and use CLI.
- Use the right interlocked macro in guarded mutex implementation.
- Fix export of Exi386InterlockedExchangeUlong
- Use documented EFLAGS definition instead of hard-coded value in some
interlocked functions.
Modified: trunk/reactos/include/ndk/asm.h
Added: trunk/reactos/ntoskrnl/ex/i386/fastinterlck_asm.S
Modified: trunk/reactos/ntoskrnl/ex/i386/interlck_asm.S
Deleted: trunk/reactos/ntoskrnl/ex/interlck.c
Deleted: trunk/reactos/ntoskrnl/ex/list.c
Deleted: trunk/reactos/ntoskrnl/ex/synch.c
Modified: trunk/reactos/ntoskrnl/ke/gmutex.c
Modified: trunk/reactos/ntoskrnl/ntoskrnl.def
Modified: trunk/reactos/ntoskrnl/ntoskrnl.xml
_____
Modified: trunk/reactos/include/ndk/asm.h
--- trunk/reactos/include/ndk/asm.h 2005-12-30 23:50:05 UTC (rev
20473)
+++ trunk/reactos/include/ndk/asm.h 2005-12-30 23:53:13 UTC (rev
20474)
@@ -289,6 +289,9 @@
#define EFLAGS_ALIGN_CHECK 0x40000
#define EFLAGS_VIF 0x80000
#define EFLAGS_VIP 0x100000
+#define EFLAG_SIGN 0x8000
+#define EFLAG_ZERO 0x4000
+#define EFLAG_SELECT (EFLAG_SIGN |
EFLAG_ZERO)
//
// CR0
_____
Added: trunk/reactos/ntoskrnl/ex/i386/fastinterlck_asm.S
--- trunk/reactos/ntoskrnl/ex/i386/fastinterlck_asm.S 2005-12-30
23:50:05 UTC (rev 20473)
+++ trunk/reactos/ntoskrnl/ex/i386/fastinterlck_asm.S 2005-12-30
23:53:13 UTC (rev 20474)
@@ -0,0 +1,701 @@
+/*
+ * COPYRIGHT: See COPYING in the top level directory
+ * PROJECT: ReactOS kernel
+ * FILE: ntoskrnl/ex/i386/fastinterlck_asm.S
+ * PURPOSE: FASTCALL Interlocked Functions
+ * PROGRAMMERS: Alex Ionescu (alex(a)relsoft.net)
+ */
+
+/* INCLUDES
******************************************************************/
+#include <ndk/asm.h>
+.intel_syntax noprefix
+
+//#define CONFIG_SMP
+
+#ifdef CONFIG_SMP
+#define LOCK lock
+#define ACQUIRE_SPINLOCK(x, y) \
+ lock bts dword ptr [x], 0; \
+ jb y
+#define RELEASE_SPINLOCK(x) mov byte ptr [x], 0
+#define SPIN_ON_LOCK(x, y) \
+1: \
+ test dword ptr [x], 1; \
+ jz y; \
+ pause; \
+ jmp 1b
+#else
+#define LOCK
+#define ACQUIRE_SPINLOCK(x, y)
+#define RELEASE_SPINLOCK(x)
+#endif
+
+/* FUNCTIONS
****************************************************************/
+
+/*
+ * NOTE: These functions must obey the following rules:
+ * - Acquire locks only on MP systems.
+ * - Be safe at HIGH_LEVEL (no paged access).
+ * - Preserve flags.
+ * - Disable interrups.
+ */
+
+/*VOID
+ *FASTCALL
+ *ExInterlockedAddLargeStatistic(IN PLARGE_INTEGER Addend,
+ * IN ULONG Increment)
+ */
+.global @ExInterlockedAddLargeStatistic@8
+@ExInterlockedAddLargeStatistic@8:
+
+#ifdef CONFIG_SMP
+ /* Do the addition */
+ lock add [ecx], edx
+
+ /* Check for carry bit and return */
+ jb 1f
+ ret
+
+1:
+ /* Add carry */
+ lock adc dword ptr [ecx+4], 0
+#else
+ /* Do the addition and add the carry */
+ add dword ptr [ecx], edx
+ adc dword ptr [ecx+4], 0
+#endif
+ /* Return */
+ ret
+
+/*ULONG
+ *FASTCALL
+ *ExfInterlockedAddUlong(IN PULONG Addend,
+ * IN ULONG Increment,
+ * IN PKSPIN_LOCK Lock)
+ */
+.global @ExfInterlockedAddUlong@12
+@ExfInterlockedAddUlong@12:
+
+ /* Save flags */
+ pushfd
+
+#ifdef CONFIG_SMP
+ /* Get lock address */
+ mov eax, [esp+8]
+.start1:
+#endif
+ /* Disable interrupts */
+ cli
+
+ /* Acquire lock */
+ ACQUIRE_SPINLOCK(eax, .spin1)
+
+ /* Do the add */
+ mov eax, [ecx]
+ add [ecx], edx
+
+#ifdef CONFIG_SMP
+ /* Get spinlock address and release it */
+ mov edx, [esp+8]
+ RELEASE_SPINLOCK(edx)
+#endif
+
+ /* Restore flags and return */
+ popfd
+ ret 4
+
+#ifdef CONFIG_SMP
+.spin1:
+ /* Restore flags and spin */
+ popfd
+ pushfd
+ SPIN_ON_LOCK(eax, .start1)
+#endif
+
+/*PLIST_ENTRY
+ *FASTCALL
+ *ExfInterlockedInsertHeadList(IN PLIST_ENTRY ListHead,
+ * IN PLIST_ENTRY ListEntry,
+ * IN PKSPIN_LOCK Lock)
+ */
+.global @ExfInterlockedInsertHeadList@12
+@ExfInterlockedInsertHeadList@12:
+
+#ifdef CONFIG_SMP
+ /* Save lock address */
+ push esi
+ mov esi, [esp+8]
+#endif
+
+ /* Save flags and disable interrupts */
+ pushfd
+.start2:
+ cli
+
+ /* Acquire lock */
+ ACQUIRE_SPINLOCK(esi, .spin2)
+
+ /* Get list pointer */
+ mov eax, [ecx]
+
+ /* Do the insert */
+ mov [edx], eax
+ mov [edx+4], ecx
+ mov [ecx], edx
+ mov [eax+4], edx
+
+ /* Release lock and restore flags */
+ RELEASE_SPINLOCK(esi)
+ popfd
+
+#ifdef CONFIG_SMP
+ pop esi
+#endif
+
+ /* Check if list was empty */
+ xor eax, ecx
+ jz 2f
+
+ /* Return list pointer */
+ xor eax, ecx
+2:
+ ret 4
+
+#ifdef CONFIG_SMP
+.spin2:
+ popfd
+ pushfd
+ SPIN_ON_LOCK(esi, .start2)
+#endif
+
+/*PLIST_ENTRY
+ *NTAPI
+ *ExfInterlockedInsertTailList(IN PLIST_ENTRY ListHead,
+ * IN PLIST_ENTRY ListEntry,
+ * IN PKSPIN_LOCK Lock)
+ */
+.global @ExfInterlockedInsertTailList@12
+@ExfInterlockedInsertTailList@12:
+
+#ifdef CONFIG_SMP
+ /* Save lock address */
+ push esi
+ mov esi, [esp+8]
+#endif
+
+ /* Save flags and disable interrupts */
+ pushfd
+.start3:
+ cli
+
+ /* Acquire lock */
+ ACQUIRE_SPINLOCK(esi, .spin3)
+
+ /* Get list pointer */
+ mov eax, [ecx+4]
+
+ /* Do the insert */
+ mov [edx], ecx
+ mov [edx+4], eax
+ mov [ecx+4], edx
+ mov [eax], edx
+
+ /* Release lock and restore flags */
+ RELEASE_SPINLOCK(esi)
+ popfd
+
+#ifdef CONFIG_SMP
+ pop esi
+#endif
+
+ /* Check if list was empty */
+ xor eax, ecx
+ jz 2f
+
+ /* Return list pointer */
+ xor eax, ecx
+2:
+ ret 4
+
+#ifdef CONFIG_SMP
+.spin3:
+ popfd
+ pushfd
+ SPIN_ON_LOCK(esi, .start3)
+#endif
+
+/*PLIST_ENTRY
+ *FASTCALL
+ *ExfInterlockedRemoveHeadList(IN PLIST_ENTRY ListHead,
+ * IN PKSPIN_LOCK Lock)
+ */
+.global @ExfInterlockedRemoveHeadList@8
+@ExfInterlockedRemoveHeadList@8:
+
+ /* Save flags and disable interrupts */
+.start4:
+ pushfd
+ cli
+ ACQUIRE_SPINLOCK(edx, .spin4)
+
+ /* Get list pointer */
+ mov eax, [ecx]
+
+ /* Check if it's empty */
+ cmp eax, ecx
+ je 2f
+
+ /* Get the next entry and do the deletion*/
+#ifdef CONFIG_SMP
+ push ebx
+ mov ebx, [eax]
+ mov [ecx], ebx
+ mov [ebx+4], ecx
+#else
+ mov edx, [eax]
+ mov [ecx], edx
+ mov [edx+4], ecx
+#endif
+
+ /* Release lock */
+#ifdef CONFIG_SMP
+ RELEASE_SPINLOCK(edx)
+ pop ebx
+#endif
+
+ /* Restore flags */
+ popfd
+
+ /* Return */
+ ret
+
+2:
+ /* Release lock */
+ RELEASE_SPINLOCK(edx)
+
+ /* Restore flags */
+ popfd
+
+ /* Return empty list */
+ xor eax, eax
+ ret
+
+#ifdef CONFIG_SMP
+.spin4:
+ popfd
+ SPIN_ON_LOCK(edx, .start4)
+#endif
+
+/*PSINGLE_LIST_ENTRY
+ *FASTCALL
+ *ExfInterlockedPopEntryList(IN PSINGLE_LIST_ENTRY ListHead,
+ * IN PKSPIN_LOCK Lock)
+ */
+.global @ExfInterlockedPopEntryList@8
+@ExfInterlockedPopEntryList@8:
+
+ /* Save flags and disable interrupts */
+.start5:
+ pushfd
+ cli
+ ACQUIRE_SPINLOCK(edx, .spin5)
+
+ /* Get list pointer */
+ mov eax, [ecx]
+
+ /* Check if it's empty */
+ or eax, eax
+ je 3f
+
+ /* Get next entry and do deletion */
+#ifdef CONFIG_SMP
+ push edx
+#endif
+ mov edx, [eax]
+ mov [ecx], edx
+#ifdef CONFIG_SMP
+ pop edx
+#endif
+
+2:
+ /* Release lock */
+ RELEASE_SPINLOCK(edx)
+
+ /* Restore flags */
+ popfd
+
+ /* Return */
+ ret
+
+3:
+ /* Return empty list */
+ xor eax, eax
+ jmp 2b
+
+#ifdef CONFIG_SMP
+.spin5:
+ popfd
+ SPIN_ON_LOCK(edx, .start5)
+#endif
+
+/*PSINGLE_LIST_ENTRY
+ *NTAPI
+ *ExfInterlockedPushEntryList(IN PSINGLE_LIST_ENTRY ListHead,
+ * IN PSINGLE_LIST_ENTRY ListEntry,
+ * IN PKSPIN_LOCK Lock)
+ */
+.global @ExfInterlockedPushEntryList@12
+@ExfInterlockedPushEntryList@12:
+
+ /* Save flags */
+ pushfd
+
+ /* Save lock pointer */
+#ifdef CONFIG_SMP
+ push edx
+ mov edx, [esp+12]
+#endif
+
+ /* Disable interrupts */
+.start6:
+ cli
+#ifdef CONFIG_SMP
+ ACQUIRE_SPINLOCK(edx, .spin6)
+ pop edx
+#endif
+
+ /* Get list pointer */
+ mov eax, [ecx]
+
+ /* Do push */
+ mov [edx], eax
+ mov [ecx], edx
+
+ /* Release lock */
+#ifdef CONFIG_SMP
+ mov edx, [esp+8]
+ RELEASE_SPINLOCK(edx)
+#endif
+
+ /* Restore flags */
+ popfd
+
+ /* Return */
+ ret 4
+
+#ifdef CONFIG_SMP
+.spin6:
+ pop edx
+ popfd
+ pushfd
+ push edx
+ mov edx, [esp+12]
+ SPIN_ON_LOCK(edx, .start6)
+#endif
+
+/*PSINGLE_LIST_ENTRY
+ *NTAPI
+ *ExInterlockedPopEntrySList(IN PSINGLE_LIST_ENTRY ListHead,
+ * IN PKSPIN_LOCK Lock)
+ */
+.global @ExInterlockedPopEntrySList@8
+.global @InterlockedPopEntrySList@4
+@ExInterlockedPopEntrySList@8:
+@InterlockedPopEntrySList@4:
+
+ /* Save registers */
+ push ebx
+ push ebp
+
+ /* Pointer to list */
+ mov ebp, ecx
+
+ /* Get sequence number and link pointer */
+ mov edx, [ebp+4]
+ mov eax, [ebp]
+
+1:
+ /* Check if the list is empty */
+ or eax, eax
+ jz 2f
+
+ /* Copy sequence number and adjust it */
+ lea ecx, [edx-1]
+
+ /* Get next pointer and do the exchange */
+ mov ebx, [eax]
+ LOCK cmpxchg8b qword ptr [ebp]
+ jnz 1b
+
+ /* Restore registers and return */
+2:
+ pop ebp
+ pop ebx
+ ret
+
+/*PSINGLE_LIST_ENTRY
+ *NTAPI
+ *ExInterlockedPushEntrySList(IN PSINGLE_LIST_ENTRY ListHead,
+ * IN PSINGLE_LIST_ENTRY ListEntry,
+ * IN PKSPIN_LOCK Lock)
+ */
+.global @ExInterlockedPushEntrySList@12
+@ExInterlockedPushEntrySList@12:
+
+ /* So we can fall through below */
+ pop [esp]
+
+.global @InterlockedPushEntrySList@8
+@InterlockedPushEntrySList@8:
+
+ /* Save registers */
+ push ebx
+ push ebp
+
+ /* Pointer to list */
+ mov ebp, ecx
+ mov ebx, edx
+
+ /* Get sequence number and link pointer */
+ mov edx, [ebp+4]
+ mov eax, [ebp]
+
+1:
+ /* Set link pointer */
+ mov [ebx], eax
+
+ /* Copy sequence number and adjust it */
+ lea ecx, [edx+0x10001]
+
+ /* Do the exchange */
+ LOCK cmpxchg8b qword ptr [ebp]
+ jnz 1b
+
+ /* Restore registers and return */
+2:
+ pop ebp
+ pop ebx
+ ret
+
+/*PSINGLE_LIST_ENTRY
+ *NTAPI
+ *ExInterlockedFlushSList(IN PSINGLE_LIST_ENTRY ListHead)
+ */
+.global @ExInterlockedFlushSList@4
+@ExInterlockedFlushSList@4:
+
+ /* Save registers */
+ push ebx
+ push ebp
+
+ /* Clear ebx */
+ xor ebx, ebx
+
+ /* Pointer to list */
+ mov ebp, ecx
+
+ /* Get sequence number and link pointer */
+ mov edx, [ebp+4]
+ mov eax, [ebp]
+
+1:
+ /* Check if the list is empty */
+ or eax, eax
+ jz 2f
+
+ /* Clear sequence and pointer */
+ mov ecx, edx
+ mov cx, bx
+
+ /* Do the exchange */
+ LOCK cmpxchg8b qword ptr [ebp]
+ jnz 1b
+
+ /* Restore registers and return */
+2:
+ pop ebp
+ pop ebx
+ ret
+
+/*INTERLOCKED_RESULT
+ *FASTCALL
+ *Exfi386InterlockedIncrementLong(IN PLONG Addend)
+ */
+.global @Exfi386InterlockedIncrementLong@4
+@Exfi386InterlockedIncrementLong@4:
+
+ /* Do the op */
+ LOCK add dword ptr [ecx], 1
+
+ /* Return */
+ lahf
+ and eax, EFLAG_SELECT
+ ret
+
+/*INTERLOCKED_RESULT
+ *FASTCALL
+ *Exfi386InterlockedDecrementLong(IN PLONG Addend)
+ */
+.global @Exfi386InterlockedDecrementLong@4
+@Exfi386InterlockedDecrementLong@4:
+
+ /* Do the op */
+ LOCK sub dword ptr [ecx], 1
+
+ /* Return */
+ lahf
+ and eax, EFLAG_SELECT
+ ret
+
+/*ULONG
+ *FASTCALL
+ *Exfi386InterlockedExchangeUlong(IN PULONG Taget,
+ * IN ULONG Value)
+ */
+.global @Exfi386InterlockedExchangeUlong@8
+.global @InterlockedExchange@8
+@InterlockedExchange@8:
+@Exfi386InterlockedExchangeUlong@8:
+
+#ifdef CONFIG_SMP
+ /* On MP, do the exchange */
+ xchg [ecx], edx
+ mov eax, edx
+#else
+ /* On UP, use cmpxchg */
+ mov eax, [ecx]
+1:
+ cmpxchg [ecx], edx
+ jnz 1b
+#endif
+
+ /* Return */
+ ret
+
+/*ULONG
+ *FASTCALL
+ *InterlockedIncrement(IN PLONG Addend)
+ */
+.global @InterlockedIncrement@4
+@InterlockedIncrement@4:
+
+ /* Do the op */
+ mov eax, 1
+ LOCK xadd dword ptr [ecx], eax
+
+ /* Return */
+ inc eax
+ ret
+
+/*ULONG
+ *FASTCALL
+ *InterlockedDecrement(IN PLONG Addend)
+ */
+.global @InterlockedDecrement@4
+@InterlockedDecrement@4:
+
+ /* Do the op */
+ mov eax, -1
+ LOCK xadd dword ptr [ecx], eax
+
+ /* Return */
+ dec eax
+ ret
+
+/*PVOID
+ *FASTCALL
+ *InterlockedCompareExchange(IN OUT PVOID *Destination,
+ * IN PVOID Exchange,
+ * IN PVOID Comperand)
+ */
+.global @InterlockedCompareExchange@12
+@InterlockedCompareExchange@12:
+
+ /* Get comperand */
+ mov eax, [esp+4]
+
+ /* Do the op */
+ LOCK cmpxchg dword ptr [ecx], edx
+
+ /* Return */
+ ret
+
+/*PVOID
+ *FASTCALL
+ *ExfInterlockedCompareExchange64(IN PLONGLONG Destination,
+ * IN PLONGLONG Exchange,
+ * IN PLONGLONG Comperand)
+ */
+.global @ExfInterlockedCompareExchange64@12
+@ExfInterlockedCompareExchange64@12:
+
+ /* Save registers */
+ push ebp
+ push ebp
+
+ /* Get desination pointer, exchange value and comperand
value/address */
+ mov ebp, ecx
+ mov ebx, [edx]
+ mov ecx, [edx+4]
+ mov edx, [esp+12]
+ mov eax, [edx]
+ mov edx, [edx+4]
+
+ /* Do the op */
+ LOCK cmpxchg8b qword ptr [ebp]
+
+ /* Restore volatiles */
+ pop ebp
+ pop ebx
+
+ /* Return */
+ ret 4
+
+/*PVOID
+ *FASTCALL
+ *ExfInterlockedCompareExchange64(IN PLONGLONG Destination,
+ * IN PLONGLONG Exchange,
+ * IN PLONGLONG Comperand,
+ * IN PKSPIN_LOCK Lock)
+ */
+.global @ExInterlockedCompareExchange64@16
+@ExInterlockedCompareExchange64@16:
+
+ /* Save registers */
+ push ebp
+ push ebp
+
+ /* Get desination pointer, exchange value and comperand
value/address */
+ mov ebp, ecx
+ mov ebx, [edx]
+ mov ecx, [edx+4]
+ mov edx, [esp+12]
+ mov eax, [edx]
+ mov edx, [edx+4]
+
+ /* Do the op */
+ LOCK cmpxchg8b qword ptr [ebp]
+
+ /* Restore volatiles */
+ pop ebp
+ pop ebx
+
+ /* Return */
+ ret 8
+
+/*PVOID
+ *FASTCALL
+ *InterlockedExchangeAdd(IN OUT PLONG Addend,
+ * IN LONG Increment)
+ */
+.global @InterlockedExchangeAdd@8
+@InterlockedExchangeAdd@8:
+
+ /* Do the op */
+ LOCK xadd dword ptr [ecx], edx
+
+ /* Return */
+ mov eax, edx
+ ret
+
+/* EOF */
_____
Modified: trunk/reactos/ntoskrnl/ex/i386/interlck_asm.S
--- trunk/reactos/ntoskrnl/ex/i386/interlck_asm.S 2005-12-30
23:50:05 UTC (rev 20473)
+++ trunk/reactos/ntoskrnl/ex/i386/interlck_asm.S 2005-12-30
23:53:13 UTC (rev 20474)
@@ -1,112 +1,583 @@
-/* $Id: interlck.c 15557 2005-05-28 07:26:38Z hbirr $
- *
+/*
* COPYRIGHT: See COPYING in the top level directory
* PROJECT: ReactOS kernel
- * FILE: ntoskrnl/ex/i386/interlck.c
- * PURPOSE: No purpose listed.
- *
- * PROGRAMMERS: No programmer listed.
+ * FILE: ntoskrnl/ex/i386/interlck_asm.S
+ * PURPOSE: STDCALL Interlocked Functions
+ * PROGRAMMERS: Alex Ionescu (alex(a)relsoft.net)
*/
+
+/* INCLUDES
******************************************************************/
+#include <ndk/asm.h>
+.intel_syntax noprefix
+//#define CONFIG_SMP
+
#ifdef CONFIG_SMP
#define LOCK lock
+#define ACQUIRE_SPINLOCK(x, y) \
+ lock bts dword ptr [x], 0; \
+ jb y
+#define RELEASE_SPINLOCK(x) mov byte ptr [x], 0
+#define SPIN_ON_LOCK(x, y) \
+1: \
+ test dword ptr [x], 1; \
+ jz y; \
+ pause; \
+ jmp 1b
#else
#define LOCK
+#define ACQUIRE_SPINLOCK(x, y)
+#define RELEASE_SPINLOCK(x)
#endif
-.global @Exfi386InterlockedIncrementLong@4
-@Exfi386InterlockedIncrementLong@4:
- LOCK
- addl $1,(%ecx)
- lahf
- andl $0xC000, %eax
- ret
+/* FUNCTIONS
****************************************************************/
-.global @Exfi386InterlockedDecrementLong@4
-@Exfi386InterlockedDecrementLong@4:
- LOCK
- subl $1,(%ecx)
- lahf
- andl $0xC000, %eax
- ret
+/*
+ * NOTE: These functions must obey the following rules:
+ * - Acquire locks only on MP systems.
+ * - Be safe at HIGH_LEVEL (no paged access).
+ * - Preserve flags.
+ * - Disable interrups.
+ */
-.global @Exfi386InterlockedExchangeUlong@8
-@Exfi386InterlockedExchangeUlong@8:
- LOCK
- xchgl %edx,(%ecx)
- movl %edx,%eax
- ret
+/*PLIST_ENTRY
+ *NTAPI
+ *ExInterlockedAddLargeInteger(IN PLIST_ENTRY ListHead,
+ * IN PLIST_ENTRY ListEntry,
+ * IN PKSPIN_LOCK Lock)
+ */
+.global _ExInterlockedAddLargeInteger@16
+_ExInterlockedAddLargeInteger@16:
+ /* Prepare stack frame */
+ push ebp
+ mov ebp, esp
+ sub esp, 8
+
+ /* Save lock pointer */
+#ifdef CONFIG_SMP
+ mov eax, [ebp+20]
+#endif
+
+ /* Save flags and disable interrupts */
+.start1:
+ pushfd
+ cli
+
+ /* Acquire lock */
+ ACQUIRE_SPINLOCK(eax, .spin1)
+
+ /* Do the calculation */
+ mov eax, [ebp+8]
+ mov ecx, [eax]
+ mov edx, [eax+4]
+
+ /* Save result */
+ mov [ebp-8], ecx
+ mov [ebp-4], edx
+
+ /* Add increment */
+ add ecx, [ebp+12]
+ adc edx, [ebp+16]
+
+ /* Save result */
+ mov [eax], ecx
+ mov [eax+4], edx
+
+ /* Release lock */
+#ifdef CONFIG_SMP
+ mov eax, [ebp+20]
+ RELEASE_SPINLOCK(eax)
+#endif
+
+ /* Restore flags */
+ popfd
+
+ /* Restore frame and return values */
+ mov eax, [ebp-8]
+ mov edx, [ebp-4]
+ mov esp, ebp
+ pop ebp
+ ret 16
+
+#ifdef CONFIG_SMP
+.spin1:
+ popfd
+ SPIN_ON_LOCK(eax, .start1)
+#endif
+
+/*PLIST_ENTRY
+ *NTAPI
+ *ExInterlockedInsertHeadList(IN PLIST_ENTRY ListHead,
+ * IN PLIST_ENTRY ListEntry,
+ * IN PKSPIN_LOCK Lock)
+ */
+.global _ExInterlockedAddUlong@12
+_ExInterlockedAddUlong@12:
+
+ /* Save flags and disable interrupts */
+ pushfd
+
+ /* Get lock address */
+#ifdef CONFIG_SMP
+ mov edx, [ebp+16]
+#endif
+.start2:
+ cli
+
+ /* Acquire lock */
+ ACQUIRE_SPINLOCK(edx, .spin2)
+
+ /* Do the calculation */
+ mov ecx, [esp+8]
+#ifdef CONFIG_SMP
+ mov eax, [ecx]
+ add eax, [esp+12]
+
+ /* Save result */
+ mov [ecx], eax
+#else
+
+ /* Do the calculation */
+ mov edx, [ecx]
+ mov eax, edx
+ add edx, [esp+12]
+
+ /* Save result */
+ mov [ecx], edx
+#endif
+
+ /* Release lock, restore flags and return */
+#ifdef CONFIG_SMP
+ sub eax, [esp+12]
+ RELEASE_SPINLOCK(edx)
+#endif
+ popfd
+ ret 12
+
+#ifdef CONFIG_SMP
+.spin2:
+ popfd
+ pushfd
+ SPIN_ON_LOCK(eax, .start2)
+#endif
+
+/*PLIST_ENTRY
+ *NTAPI
+ *ExInterlockedInsertHeadList(IN PLIST_ENTRY ListHead,
+ * IN PLIST_ENTRY ListEntry,
+ * IN PKSPIN_LOCK Lock)
+ */
+.global _ExInterlockedInsertHeadList@12
+_ExInterlockedInsertHeadList@12:
+
+ /* Save lock pointer */
+#ifdef CONFIG_SMP
+ mov edx, [esp+12]
+#endif
+
+ /* Save flags and disable interrupts */
+.start3:
+ pushfd
+ cli
+ ACQUIRE_SPINLOCK(edx, .spin3)
+
+ /* Get list pointers */
+ mov eax, [esp+8]
+ mov ecx, [esp+12]
+ mov edx, [eax]
+
+ /* Do the insert */
+ mov [ecx], edx
+ mov [ecx+4], eax
+ mov [eax], ecx
+ mov [edx+4], ecx
+
+ /* Release lock */
+#ifdef CONFIG_SMP
+ mov ecx, [esp+16]
+ RELEASE_SPINLOCK(ecx)
+#endif
+
+ /* Restore flags */
+ popfd
+
+ /* check if the list was empty and return NULL */
+ xor eax, edx
+ jz 2f
+
+ /* Return pointer */
+ mov eax, edx
+
+2:
+ ret 12
+
+#ifdef CONFIG_SMP
+.spin3:
+ popfd
+ SPIN_ON_LOCK(edx, .start3)
+#endif
+
+/*PLIST_ENTRY
+ *NTAPI
+ *ExInterlockedInsertTailList(IN PLIST_ENTRY ListHead,
+ * IN PLIST_ENTRY ListEntry,
+ * IN PKSPIN_LOCK Lock)
+ */
+.global _ExInterlockedInsertTailList@12
+_ExInterlockedInsertTailList@12:
+
+ /* Save lock pointer */
+#ifdef CONFIG_SMP
+ mov edx, [esp+12]
+#endif
+
+ /* Save flags and disable interrupts */
+.start4:
+ pushfd
+ cli
+ ACQUIRE_SPINLOCK(edx, .spin4)
+
+ /* Get list pointers */
+ mov eax, [esp+8]
+ mov ecx, [esp+12]
+ mov edx, [eax+4]
+
+ /* Do the insert */
+ mov [ecx], eax
+ mov [ecx+4], edx
+ mov [eax+4], ecx
[truncated at 1000 lines; 1285 more skipped]
