From d5d9a3b12adf339b851c8e595d2ca71c3d211363 Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Wed, 28 Sep 2011 17:45:15 -0700 Subject: [PATCH 1/9] jump_label: use proper atomic_t initializer ATOMIC_INIT() is the proper thing to use. Signed-off-by: Jeremy Fitzhardinge Acked-by: Jason Baron Acked-by: Peter Zijlstra --- include/linux/jump_label.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 66f23dc5e76a..1213e9d63f79 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -28,9 +28,9 @@ struct module; #ifdef HAVE_JUMP_LABEL #ifdef CONFIG_MODULES -#define JUMP_LABEL_INIT {{ 0 }, NULL, NULL} +#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL, NULL} #else -#define JUMP_LABEL_INIT {{ 0 }, NULL} +#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL} #endif static __always_inline bool static_branch(struct jump_label_key *key) From 189c3fd68c7016e37c1ffd7a00009e2c944a9d06 Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Fri, 30 Sep 2011 09:10:54 -0700 Subject: [PATCH 2/9] stop_machine: make stop_machine safe and efficient to call early Make stop_machine() safe to call early in boot, before stop_machine() has been set up, by simply calling the callback function directly if there's only one CPU online. [ Fixes from AKPM: - add comment - local_irq_flags, not save_flags - also call hard_irq_disable() for systems which need it Tejun suggested using an explicit flag rather than just looking at the online cpu count. ] Signed-off-by: Jeremy Fitzhardinge Acked-by: Tejun Heo Acked-by: Peter Zijlstra Cc: Rusty Russell Cc: Peter Zijlstra Cc: Andrew Morton Cc: H. Peter Anvin Cc: Ingo Molnar Cc: Steven Rostedt --- kernel/stop_machine.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c index ba5070ce5765..d3f960a6ca56 100644 --- a/kernel/stop_machine.c +++ b/kernel/stop_machine.c @@ -41,6 +41,7 @@ struct cpu_stopper { }; static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper); +static bool stop_machine_initialized = false; static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo) { @@ -386,6 +387,8 @@ static int __init cpu_stop_init(void) cpu_stop_cpu_callback(&cpu_stop_cpu_notifier, CPU_ONLINE, bcpu); register_cpu_notifier(&cpu_stop_cpu_notifier); + stop_machine_initialized = true; + return 0; } early_initcall(cpu_stop_init); @@ -485,6 +488,25 @@ int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus) .num_threads = num_online_cpus(), .active_cpus = cpus }; + if (!stop_machine_initialized) { + /* + * Handle the case where stop_machine() is called + * early in boot before stop_machine() has been + * initialized. + */ + unsigned long flags; + int ret; + + WARN_ON_ONCE(smdata.num_threads != 1); + + local_irq_save(flags); + hard_irq_disable(); + ret = (*fn)(data); + local_irq_restore(flags); + + return ret; + } + /* Set the initial state and stop all online cpus. */ set_state(&smdata, STOPMACHINE_PREPARE); return stop_cpus(cpu_online_mask, stop_machine_cpu_stop, &smdata); From 37348804e0289087d21ae8bff4c0732030a3c6ac Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Thu, 29 Sep 2011 11:10:05 -0700 Subject: [PATCH 3/9] jump_label: if a key has already been initialized, don't nop it out If a key has been enabled before jump_label_init() is called, don't nop it out. This removes arch_jump_label_text_poke_early() (which can only nop out a site) and uses arch_jump_label_transform() instead. Signed-off-by: Jeremy Fitzhardinge Acked-by: Jason Baron Acked-by: Peter Zijlstra --- include/linux/jump_label.h | 3 +-- kernel/jump_label.c | 20 ++++++++------------ 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 1213e9d63f79..12e804ea32ab 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -44,8 +44,7 @@ extern struct jump_entry __stop___jump_table[]; extern void jump_label_lock(void); extern void jump_label_unlock(void); extern void arch_jump_label_transform(struct jump_entry *entry, - enum jump_label_type type); -extern void arch_jump_label_text_poke_early(jump_label_t addr); + enum jump_label_type type); extern int jump_label_text_reserved(void *start, void *end); extern void jump_label_inc(struct jump_label_key *key); extern void jump_label_dec(struct jump_label_key *key); diff --git a/kernel/jump_label.c b/kernel/jump_label.c index a8ce45097f3d..059202d5b77a 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -121,13 +121,6 @@ static void __jump_label_update(struct jump_label_key *key, } } -/* - * Not all archs need this. - */ -void __weak arch_jump_label_text_poke_early(jump_label_t addr) -{ -} - static __init int jump_label_init(void) { struct jump_entry *iter_start = __start___jump_table; @@ -139,12 +132,15 @@ static __init int jump_label_init(void) jump_label_sort_entries(iter_start, iter_stop); for (iter = iter_start; iter < iter_stop; iter++) { - arch_jump_label_text_poke_early(iter->code); - if (iter->key == (jump_label_t)(unsigned long)key) + struct jump_label_key *iterk; + + iterk = (struct jump_label_key *)(unsigned long)iter->key; + arch_jump_label_transform(iter, jump_label_enabled(iterk) ? + JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE); + if (iterk == key) continue; - key = (struct jump_label_key *)(unsigned long)iter->key; - atomic_set(&key->enabled, 0); + key = iterk; key->entries = iter; #ifdef CONFIG_MODULES key->next = NULL; @@ -212,7 +208,7 @@ void jump_label_apply_nops(struct module *mod) return; for (iter = iter_start; iter < iter_stop; iter++) - arch_jump_label_text_poke_early(iter->code); + arch_jump_label_transform(iter, JUMP_LABEL_DISABLE); } static int jump_label_add_module(struct module *mod) From b7e3155818830a358be2b736e1087217c43db9eb Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Thu, 29 Sep 2011 11:11:09 -0700 Subject: [PATCH 4/9] x86/jump_label: drop arch_jump_label_text_poke_early() It is no longer used. Signed-off-by: Jeremy Fitzhardinge Acked-by: Jason Baron Acked-by: Peter Zijlstra --- arch/x86/kernel/jump_label.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c index 3fee346ef545..2ad029876c8c 100644 --- a/arch/x86/kernel/jump_label.c +++ b/arch/x86/kernel/jump_label.c @@ -42,10 +42,4 @@ void arch_jump_label_transform(struct jump_entry *entry, put_online_cpus(); } -void arch_jump_label_text_poke_early(jump_label_t addr) -{ - text_poke_early((void *)addr, ideal_nops[NOP_ATOMIC5], - JUMP_LABEL_NOP_SIZE); -} - #endif From 92e02396d7214fdc7c25239eb647eb48e6fe6ba9 Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Thu, 29 Sep 2011 11:11:09 -0700 Subject: [PATCH 5/9] sparc/jump_label: drop arch_jump_label_text_poke_early() It is no longer used. Signed-off-by: Jeremy Fitzhardinge Acked-by: Jason Baron Acked-by: Peter Zijlstra Cc: David Miller --- arch/sparc/kernel/jump_label.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c index ea2dafc93d78..971fd435a281 100644 --- a/arch/sparc/kernel/jump_label.c +++ b/arch/sparc/kernel/jump_label.c @@ -36,12 +36,4 @@ void arch_jump_label_transform(struct jump_entry *entry, put_online_cpus(); } -void arch_jump_label_text_poke_early(jump_label_t addr) -{ - u32 *insn_p = (u32 *) (unsigned long) addr; - - *insn_p = 0x01000000; - flushi(insn_p); -} - #endif From 20284aa77c0f6227da4783a920b72dc61d4bcc09 Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Mon, 3 Oct 2011 11:01:46 -0700 Subject: [PATCH 6/9] jump_label: add arch_jump_label_transform_static() to optimise non-live code updates When updating a newly loaded module, the code is definitely not yet executing on any processor, so it can be updated with no need for any heavyweight synchronization. This patch adds arch_jump_label_static() which is implemented as arch_jump_label_transform() by default, but architectures can override it if it avoids, say, a call to stop_machine(). Signed-off-by: Jeremy Fitzhardinge Acked-by: Jason Baron Acked-by: Peter Zijlstra --- include/linux/jump_label.h | 2 ++ kernel/jump_label.c | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 12e804ea32ab..56594e45b011 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -45,6 +45,8 @@ extern void jump_label_lock(void); extern void jump_label_unlock(void); extern void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type); +extern void arch_jump_label_transform_static(struct jump_entry *entry, + enum jump_label_type type); extern int jump_label_text_reserved(void *start, void *end); extern void jump_label_inc(struct jump_label_key *key); extern void jump_label_dec(struct jump_label_key *key); diff --git a/kernel/jump_label.c b/kernel/jump_label.c index 059202d5b77a..ff2028f35aa8 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -104,6 +104,18 @@ static int __jump_label_text_reserved(struct jump_entry *iter_start, return 0; } +/* + * Update code which is definitely not currently executing. + * Architectures which need heavyweight synchronization to modify + * running code can override this to make the non-live update case + * cheaper. + */ +void __weak arch_jump_label_transform_static(struct jump_entry *entry, + enum jump_label_type type) +{ + arch_jump_label_transform(entry, type); +} + static void __jump_label_update(struct jump_label_key *key, struct jump_entry *entry, struct jump_entry *stop, int enable) @@ -135,8 +147,8 @@ static __init int jump_label_init(void) struct jump_label_key *iterk; iterk = (struct jump_label_key *)(unsigned long)iter->key; - arch_jump_label_transform(iter, jump_label_enabled(iterk) ? - JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE); + arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ? + JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE); if (iterk == key) continue; @@ -208,7 +220,7 @@ void jump_label_apply_nops(struct module *mod) return; for (iter = iter_start; iter < iter_stop; iter++) - arch_jump_label_transform(iter, JUMP_LABEL_DISABLE); + arch_jump_label_transform_static(iter, JUMP_LABEL_DISABLE); } static int jump_label_add_module(struct module *mod) From 61f42183fd15d5e666236f474d73b3555c7b83d1 Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Thu, 29 Sep 2011 10:58:46 -0700 Subject: [PATCH 7/9] s390/jump-label: add arch_jump_label_transform_static() This allows jump-label entries to be cheaply updated on code which is not yet live. Signed-off-by: Jeremy Fitzhardinge Acked-by: Jason Baron Acked-by: Peter Zijlstra Cc: Jan Glauber --- arch/s390/kernel/jump_label.c | 51 +++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/arch/s390/kernel/jump_label.c b/arch/s390/kernel/jump_label.c index 44cc06bedf77..b987ab2c1541 100644 --- a/arch/s390/kernel/jump_label.c +++ b/arch/s390/kernel/jump_label.c @@ -18,26 +18,15 @@ struct insn { } __packed; struct insn_args { - unsigned long *target; - struct insn *insn; - ssize_t size; + struct jump_entry *entry; + enum jump_label_type type; }; -static int __arch_jump_label_transform(void *data) +static void __jump_label_transform(struct jump_entry *entry, + enum jump_label_type type) { - struct insn_args *args = data; - int rc; - - rc = probe_kernel_write(args->target, args->insn, args->size); - WARN_ON_ONCE(rc < 0); - return 0; -} - -void arch_jump_label_transform(struct jump_entry *entry, - enum jump_label_type type) -{ - struct insn_args args; struct insn insn; + int rc; if (type == JUMP_LABEL_ENABLE) { /* brcl 15,offset */ @@ -49,11 +38,33 @@ void arch_jump_label_transform(struct jump_entry *entry, insn.offset = 0; } - args.target = (void *) entry->code; - args.insn = &insn; - args.size = JUMP_LABEL_NOP_SIZE; + rc = probe_kernel_write((void *)entry->code, &insn, JUMP_LABEL_NOP_SIZE); + WARN_ON_ONCE(rc < 0); +} - stop_machine(__arch_jump_label_transform, &args, NULL); +static int __sm_arch_jump_label_transform(void *data) +{ + struct insn_args *args = data; + + __jump_label_transform(args->entry, args->type); + return 0; +} + +void arch_jump_label_transform(struct jump_entry *entry, + enum jump_label_type type) +{ + struct insn_args args; + + args.entry = entry; + args.type = type; + + stop_machine(__sm_arch_jump_label_transform, &args, NULL); +} + +void arch_jump_label_transform_static(struct jump_entry *entry, + enum jump_label_type type) +{ + __jump_label_transform(entry, type); } #endif From e71a5be15e47a73a2964712967fe93ee8ccf551b Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Thu, 29 Sep 2011 11:11:09 -0700 Subject: [PATCH 8/9] x86/jump_label: add arch_jump_label_transform_static() This allows jump-label entries to be cheaply updated on code which is not yet live. Signed-off-by: Jeremy Fitzhardinge Acked-by: Jason Baron Acked-by: Peter Zijlstra --- arch/x86/kernel/jump_label.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c index 2ad029876c8c..ea9d5f2f13ef 100644 --- a/arch/x86/kernel/jump_label.c +++ b/arch/x86/kernel/jump_label.c @@ -24,8 +24,9 @@ union jump_code_union { } __attribute__((packed)); }; -void arch_jump_label_transform(struct jump_entry *entry, - enum jump_label_type type) +static void __jump_label_transform(struct jump_entry *entry, + enum jump_label_type type, + void *(*poker)(void *, const void *, size_t)) { union jump_code_union code; @@ -35,11 +36,24 @@ void arch_jump_label_transform(struct jump_entry *entry, (entry->code + JUMP_LABEL_NOP_SIZE); } else memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE); + + (*poker)((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE); +} + +void arch_jump_label_transform(struct jump_entry *entry, + enum jump_label_type type) +{ get_online_cpus(); mutex_lock(&text_mutex); - text_poke_smp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE); + __jump_label_transform(entry, type, text_poke_smp); mutex_unlock(&text_mutex); put_online_cpus(); } +void arch_jump_label_transform_static(struct jump_entry *entry, + enum jump_label_type type) +{ + __jump_label_transform(entry, type, text_poke_early); +} + #endif From 97ce2c88f9ad42e3c60a9beb9fca87abf3639faa Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Wed, 12 Oct 2011 16:17:54 -0700 Subject: [PATCH 9/9] jump-label: initialize jump-label subsystem much earlier Initialize jump_labels much, much earlier, so they're available for use during system setup. Signed-off-by: Jeremy Fitzhardinge Acked-by: Peter Zijlstra --- include/linux/jump_label.h | 14 +++++++++----- init/main.c | 3 +++ kernel/jump_label.c | 5 +---- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 56594e45b011..388b0d425b50 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -16,7 +16,7 @@ struct jump_label_key { # include # define HAVE_JUMP_LABEL -#endif +#endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */ enum jump_label_type { JUMP_LABEL_DISABLE = 0, @@ -41,6 +41,7 @@ static __always_inline bool static_branch(struct jump_label_key *key) extern struct jump_entry __start___jump_table[]; extern struct jump_entry __stop___jump_table[]; +extern void jump_label_init(void); extern void jump_label_lock(void); extern void jump_label_unlock(void); extern void arch_jump_label_transform(struct jump_entry *entry, @@ -53,7 +54,7 @@ extern void jump_label_dec(struct jump_label_key *key); extern bool jump_label_enabled(struct jump_label_key *key); extern void jump_label_apply_nops(struct module *mod); -#else +#else /* !HAVE_JUMP_LABEL */ #include @@ -63,6 +64,10 @@ struct jump_label_key { atomic_t enabled; }; +static __always_inline void jump_label_init(void) +{ +} + static __always_inline bool static_branch(struct jump_label_key *key) { if (unlikely(atomic_read(&key->enabled))) @@ -97,7 +102,6 @@ static inline int jump_label_apply_nops(struct module *mod) { return 0; } +#endif /* HAVE_JUMP_LABEL */ -#endif - -#endif +#endif /* _LINUX_JUMP_LABEL_H */ diff --git a/init/main.c b/init/main.c index 2a9b88aa5e76..29d8d847de94 100644 --- a/init/main.c +++ b/init/main.c @@ -515,6 +515,9 @@ asmlinkage void __init start_kernel(void) parse_args("Booting kernel", static_command_line, __start___param, __stop___param - __start___param, &unknown_bootoption); + + jump_label_init(); + /* * These use large bootmem allocations and must precede * kmem_cache_init() diff --git a/kernel/jump_label.c b/kernel/jump_label.c index ff2028f35aa8..bbdfe2a462a0 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -133,7 +133,7 @@ static void __jump_label_update(struct jump_label_key *key, } } -static __init int jump_label_init(void) +void __init jump_label_init(void) { struct jump_entry *iter_start = __start___jump_table; struct jump_entry *iter_stop = __stop___jump_table; @@ -159,10 +159,7 @@ static __init int jump_label_init(void) #endif } jump_label_unlock(); - - return 0; } -early_initcall(jump_label_init); #ifdef CONFIG_MODULES