Commit 3cd04622 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'smp-hotplug-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull CPU hotplug updates from Thomas Gleixner:
 "A small update for the SMP hotplug code code:

   - Track "booted once" CPUs in a cpumask so the x86 APIC code has an
     easy way to decide whether broadcast IPIs are safe to use or not.

   - Implement a cpumask_or_equal() helper for the IPI broadcast
     evaluation.

     The above two changes have been also pulled into the x86/apic
     branch for implementing the conditional IPI broadcast feature.

   - Cache the number of online CPUs instead of reevaluating it over and
     over. num_online_cpus() is an unreliable snapshot anyway except
     when it is used outside a cpu hotplug locked region. The cached
     access is not changing this, but it's definitely faster than
     calculating the bitmap wheight especially in hot paths"

* 'smp-hotplug-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  cpu/hotplug: Cache number of online CPUs
  cpumask: Implement cpumask_or_equal()
  smp/hotplug: Track booted once CPUs in a cpumask
parents 16208cd6 0c09ab96
...@@ -120,6 +120,10 @@ extern int __bitmap_empty(const unsigned long *bitmap, unsigned int nbits); ...@@ -120,6 +120,10 @@ extern int __bitmap_empty(const unsigned long *bitmap, unsigned int nbits);
extern int __bitmap_full(const unsigned long *bitmap, unsigned int nbits); extern int __bitmap_full(const unsigned long *bitmap, unsigned int nbits);
extern int __bitmap_equal(const unsigned long *bitmap1, extern int __bitmap_equal(const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits); const unsigned long *bitmap2, unsigned int nbits);
extern bool __pure __bitmap_or_equal(const unsigned long *src1,
const unsigned long *src2,
const unsigned long *src3,
unsigned int nbits);
extern void __bitmap_complement(unsigned long *dst, const unsigned long *src, extern void __bitmap_complement(unsigned long *dst, const unsigned long *src,
unsigned int nbits); unsigned int nbits);
extern void __bitmap_shift_right(unsigned long *dst, const unsigned long *src, extern void __bitmap_shift_right(unsigned long *dst, const unsigned long *src,
...@@ -321,6 +325,25 @@ static inline int bitmap_equal(const unsigned long *src1, ...@@ -321,6 +325,25 @@ static inline int bitmap_equal(const unsigned long *src1,
return __bitmap_equal(src1, src2, nbits); return __bitmap_equal(src1, src2, nbits);
} }
/**
* bitmap_or_equal - Check whether the or of two bitnaps is equal to a third
* @src1: Pointer to bitmap 1
* @src2: Pointer to bitmap 2 will be or'ed with bitmap 1
* @src3: Pointer to bitmap 3. Compare to the result of *@src1 | *@src2
*
* Returns: True if (*@src1 | *@src2) == *@src3, false otherwise
*/
static inline bool bitmap_or_equal(const unsigned long *src1,
const unsigned long *src2,
const unsigned long *src3,
unsigned int nbits)
{
if (!small_const_nbits(nbits))
return __bitmap_or_equal(src1, src2, src3, nbits);
return !(((*src1 | *src2) ^ *src3) & BITMAP_LAST_WORD_MASK(nbits));
}
static inline int bitmap_intersects(const unsigned long *src1, static inline int bitmap_intersects(const unsigned long *src1,
const unsigned long *src2, unsigned int nbits) const unsigned long *src2, unsigned int nbits)
{ {
......
...@@ -10,6 +10,7 @@ ...@@ -10,6 +10,7 @@
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/threads.h> #include <linux/threads.h>
#include <linux/bitmap.h> #include <linux/bitmap.h>
#include <linux/atomic.h>
#include <linux/bug.h> #include <linux/bug.h>
/* Don't assign or return these: may not be this big! */ /* Don't assign or return these: may not be this big! */
...@@ -95,8 +96,21 @@ extern struct cpumask __cpu_active_mask; ...@@ -95,8 +96,21 @@ extern struct cpumask __cpu_active_mask;
#define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask) #define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask)
#define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask) #define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask)
extern atomic_t __num_online_cpus;
#if NR_CPUS > 1 #if NR_CPUS > 1
#define num_online_cpus() cpumask_weight(cpu_online_mask) /**
* num_online_cpus() - Read the number of online CPUs
*
* Despite the fact that __num_online_cpus is of type atomic_t, this
* interface gives only a momentary snapshot and is not protected against
* concurrent CPU hotplug operations unless invoked from a cpuhp_lock held
* region.
*/
static inline unsigned int num_online_cpus(void)
{
return atomic_read(&__num_online_cpus);
}
#define num_possible_cpus() cpumask_weight(cpu_possible_mask) #define num_possible_cpus() cpumask_weight(cpu_possible_mask)
#define num_present_cpus() cpumask_weight(cpu_present_mask) #define num_present_cpus() cpumask_weight(cpu_present_mask)
#define num_active_cpus() cpumask_weight(cpu_active_mask) #define num_active_cpus() cpumask_weight(cpu_active_mask)
...@@ -115,6 +129,8 @@ extern struct cpumask __cpu_active_mask; ...@@ -115,6 +129,8 @@ extern struct cpumask __cpu_active_mask;
#define cpu_active(cpu) ((cpu) == 0) #define cpu_active(cpu) ((cpu) == 0)
#endif #endif
extern cpumask_t cpus_booted_once_mask;
static inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits) static inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
{ {
#ifdef CONFIG_DEBUG_PER_CPU_MAPS #ifdef CONFIG_DEBUG_PER_CPU_MAPS
...@@ -473,6 +489,20 @@ static inline bool cpumask_equal(const struct cpumask *src1p, ...@@ -473,6 +489,20 @@ static inline bool cpumask_equal(const struct cpumask *src1p,
nr_cpumask_bits); nr_cpumask_bits);
} }
/**
* cpumask_or_equal - *src1p | *src2p == *src3p
* @src1p: the first input
* @src2p: the second input
* @src3p: the third input
*/
static inline bool cpumask_or_equal(const struct cpumask *src1p,
const struct cpumask *src2p,
const struct cpumask *src3p)
{
return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p),
cpumask_bits(src3p), nr_cpumask_bits);
}
/** /**
* cpumask_intersects - (*src1p & *src2p) != 0 * cpumask_intersects - (*src1p & *src2p) != 0
* @src1p: the first input * @src1p: the first input
...@@ -805,14 +835,7 @@ set_cpu_present(unsigned int cpu, bool present) ...@@ -805,14 +835,7 @@ set_cpu_present(unsigned int cpu, bool present)
cpumask_clear_cpu(cpu, &__cpu_present_mask); cpumask_clear_cpu(cpu, &__cpu_present_mask);
} }
static inline void void set_cpu_online(unsigned int cpu, bool online);
set_cpu_online(unsigned int cpu, bool online)
{
if (online)
cpumask_set_cpu(cpu, &__cpu_online_mask);
else
cpumask_clear_cpu(cpu, &__cpu_online_mask);
}
static inline void static inline void
set_cpu_active(unsigned int cpu, bool active) set_cpu_active(unsigned int cpu, bool active)
......
...@@ -62,7 +62,6 @@ struct cpuhp_cpu_state { ...@@ -62,7 +62,6 @@ struct cpuhp_cpu_state {
bool rollback; bool rollback;
bool single; bool single;
bool bringup; bool bringup;
bool booted_once;
struct hlist_node *node; struct hlist_node *node;
struct hlist_node *last; struct hlist_node *last;
enum cpuhp_state cb_state; enum cpuhp_state cb_state;
...@@ -76,6 +75,10 @@ static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = { ...@@ -76,6 +75,10 @@ static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
.fail = CPUHP_INVALID, .fail = CPUHP_INVALID,
}; };
#ifdef CONFIG_SMP
cpumask_t cpus_booted_once_mask;
#endif
#if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP) #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
static struct lockdep_map cpuhp_state_up_map = static struct lockdep_map cpuhp_state_up_map =
STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map); STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
...@@ -433,7 +436,7 @@ static inline bool cpu_smt_allowed(unsigned int cpu) ...@@ -433,7 +436,7 @@ static inline bool cpu_smt_allowed(unsigned int cpu)
* CPU. Otherwise, a broadacasted MCE observing CR4.MCE=0b on any * CPU. Otherwise, a broadacasted MCE observing CR4.MCE=0b on any
* core will shutdown the machine. * core will shutdown the machine.
*/ */
return !per_cpu(cpuhp_state, cpu).booted_once; return !cpumask_test_cpu(cpu, &cpus_booted_once_mask);
} }
#else #else
static inline bool cpu_smt_allowed(unsigned int cpu) { return true; } static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
...@@ -1066,7 +1069,7 @@ void notify_cpu_starting(unsigned int cpu) ...@@ -1066,7 +1069,7 @@ void notify_cpu_starting(unsigned int cpu)
int ret; int ret;
rcu_cpu_starting(cpu); /* Enables RCU usage on this CPU. */ rcu_cpu_starting(cpu); /* Enables RCU usage on this CPU. */
st->booted_once = true; cpumask_set_cpu(cpu, &cpus_booted_once_mask);
while (st->state < target) { while (st->state < target) {
st->state++; st->state++;
ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL); ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
...@@ -2295,6 +2298,9 @@ EXPORT_SYMBOL(__cpu_present_mask); ...@@ -2295,6 +2298,9 @@ EXPORT_SYMBOL(__cpu_present_mask);
struct cpumask __cpu_active_mask __read_mostly; struct cpumask __cpu_active_mask __read_mostly;
EXPORT_SYMBOL(__cpu_active_mask); EXPORT_SYMBOL(__cpu_active_mask);
atomic_t __num_online_cpus __read_mostly;
EXPORT_SYMBOL(__num_online_cpus);
void init_cpu_present(const struct cpumask *src) void init_cpu_present(const struct cpumask *src)
{ {
cpumask_copy(&__cpu_present_mask, src); cpumask_copy(&__cpu_present_mask, src);
...@@ -2310,6 +2316,27 @@ void init_cpu_online(const struct cpumask *src) ...@@ -2310,6 +2316,27 @@ void init_cpu_online(const struct cpumask *src)
cpumask_copy(&__cpu_online_mask, src); cpumask_copy(&__cpu_online_mask, src);
} }
void set_cpu_online(unsigned int cpu, bool online)
{
/*
* atomic_inc/dec() is required to handle the horrid abuse of this
* function by the reboot and kexec code which invoke it from
* IPI/NMI broadcasts when shutting down CPUs. Invocation from
* regular CPU hotplug is properly serialized.
*
* Note, that the fact that __num_online_cpus is of type atomic_t
* does not protect readers which are not serialized against
* concurrent hotplug operations.
*/
if (online) {
if (!cpumask_test_and_set_cpu(cpu, &__cpu_online_mask))
atomic_inc(&__num_online_cpus);
} else {
if (cpumask_test_and_clear_cpu(cpu, &__cpu_online_mask))
atomic_dec(&__num_online_cpus);
}
}
/* /*
* Activate the first processor. * Activate the first processor.
*/ */
...@@ -2334,7 +2361,7 @@ void __init boot_cpu_init(void) ...@@ -2334,7 +2361,7 @@ void __init boot_cpu_init(void)
void __init boot_cpu_hotplug_init(void) void __init boot_cpu_hotplug_init(void)
{ {
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
this_cpu_write(cpuhp_state.booted_once, true); cpumask_set_cpu(smp_processor_id(), &cpus_booted_once_mask);
#endif #endif
this_cpu_write(cpuhp_state.state, CPUHP_ONLINE); this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
} }
......
...@@ -59,6 +59,26 @@ int __bitmap_equal(const unsigned long *bitmap1, ...@@ -59,6 +59,26 @@ int __bitmap_equal(const unsigned long *bitmap1,
} }
EXPORT_SYMBOL(__bitmap_equal); EXPORT_SYMBOL(__bitmap_equal);
bool __bitmap_or_equal(const unsigned long *bitmap1,
const unsigned long *bitmap2,
const unsigned long *bitmap3,
unsigned int bits)
{
unsigned int k, lim = bits / BITS_PER_LONG;
unsigned long tmp;
for (k = 0; k < lim; ++k) {
if ((bitmap1[k] | bitmap2[k]) != bitmap3[k])
return false;
}
if (!(bits % BITS_PER_LONG))
return true;
tmp = (bitmap1[k] | bitmap2[k]) ^ bitmap3[k];
return (tmp & BITMAP_LAST_WORD_MASK(bits)) == 0;
}
void __bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int bits) void __bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int bits)
{ {
unsigned int k, lim = BITS_TO_LONGS(bits); unsigned int k, lim = BITS_TO_LONGS(bits);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment