bit_spinlock: __always_inline (un)lock functions
The compiler might decide that it is a smart idea to not inline bit_spin_lock(), primarily when a couple of functions in the same file end up calling it. Especially when used in RMAP map/unmap code next, the compiler sometimes decides to not inline, which is then observable in some micro-benchmarks. Let's simply flag all lock/unlock functions as __always_inline; arch_test_and_set_bit_lock() and friends are already tagged like that (but not test_and_set_bit_lock() for some reason). If ever a problem, we could split it into a fast and a slow path, and only force the fast path to be inlined. But there is nothing particularly "big" here. Link: https://lkml.kernel.org/r/20250303163014.1128035-11-david@redhat.com Signed-off-by: David Hildenbrand <david@redhat.com> Cc: Andy Lutomirks^H^Hski <luto@kernel.org> Cc: Borislav Betkov <bp@alien8.de> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: Ingo Molnar <mingo@redhat.com> Cc: Jann Horn <jannh@google.com> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: Jonathan Corbet <corbet@lwn.net> Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Cc: Lance Yang <ioworker0@gmail.com> Cc: Liam Howlett <liam.howlett@oracle.com> Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com> Cc: Matthew Wilcow (Oracle) <willy@infradead.org> Cc: Michal Koutn <mkoutny@suse.com> Cc: Muchun Song <muchun.song@linux.dev> Cc: tejun heo <tj@kernel.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Vlastimil Babka <vbabka@suse.cz> Cc: Zefan Li <lizefan.x@bytedance.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
committed by
Andrew Morton
parent
932961c4b6
commit
b85aa9b11b
@@ -13,7 +13,7 @@
|
||||
* Don't use this unless you really need to: spin_lock() and spin_unlock()
|
||||
* are significantly faster.
|
||||
*/
|
||||
static inline void bit_spin_lock(int bitnum, unsigned long *addr)
|
||||
static __always_inline void bit_spin_lock(int bitnum, unsigned long *addr)
|
||||
{
|
||||
/*
|
||||
* Assuming the lock is uncontended, this never enters
|
||||
@@ -38,7 +38,7 @@ static inline void bit_spin_lock(int bitnum, unsigned long *addr)
|
||||
/*
|
||||
* Return true if it was acquired
|
||||
*/
|
||||
static inline int bit_spin_trylock(int bitnum, unsigned long *addr)
|
||||
static __always_inline int bit_spin_trylock(int bitnum, unsigned long *addr)
|
||||
{
|
||||
preempt_disable();
|
||||
#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
|
||||
@@ -54,7 +54,7 @@ static inline int bit_spin_trylock(int bitnum, unsigned long *addr)
|
||||
/*
|
||||
* bit-based spin_unlock()
|
||||
*/
|
||||
static inline void bit_spin_unlock(int bitnum, unsigned long *addr)
|
||||
static __always_inline void bit_spin_unlock(int bitnum, unsigned long *addr)
|
||||
{
|
||||
#ifdef CONFIG_DEBUG_SPINLOCK
|
||||
BUG_ON(!test_bit(bitnum, addr));
|
||||
@@ -71,7 +71,7 @@ static inline void bit_spin_unlock(int bitnum, unsigned long *addr)
|
||||
* non-atomic version, which can be used eg. if the bit lock itself is
|
||||
* protecting the rest of the flags in the word.
|
||||
*/
|
||||
static inline void __bit_spin_unlock(int bitnum, unsigned long *addr)
|
||||
static __always_inline void __bit_spin_unlock(int bitnum, unsigned long *addr)
|
||||
{
|
||||
#ifdef CONFIG_DEBUG_SPINLOCK
|
||||
BUG_ON(!test_bit(bitnum, addr));
|
||||
|
||||
Reference in New Issue
Block a user