From: Ashok Raj This patch speeds up checks in move_irq which is called in intr_path. Earlier checks for the same under CONFIG_IRQBALANCE would check if the cpumask is empty, which is a slower operation when NR_CPUS is large. This patch adds a check to speed up processing in the irq_desc structure. Also adds a new function to generalize setting the pending_irq array so it can be used in other places. Signed-off-by: Ashok Raj Cc: Andi Kleen Cc: Zwane Mwaikambo Signed-off-by: Andrew Morton --- arch/i386/kernel/io_apic.c | 16 ++++------------ include/linux/irq.h | 27 ++++++++++++++++++++++----- kernel/irq/proc.c | 7 +------ 3 files changed, 27 insertions(+), 23 deletions(-) diff -puN arch/i386/kernel/io_apic.c~x86-x86_64-deferred-handling-of-writes-to-proc-irq-xx-smp_affinitypatch-added-to-mm-tree-fix-5 arch/i386/kernel/io_apic.c --- 25/arch/i386/kernel/io_apic.c~x86-x86_64-deferred-handling-of-writes-to-proc-irq-xx-smp_affinitypatch-added-to-mm-tree-fix-5 2005-05-03 19:13:04.000000000 -0700 +++ 25-akpm/arch/i386/kernel/io_apic.c 2005-05-03 19:13:04.000000000 -0700 @@ -334,12 +334,7 @@ static inline void balance_irq(int cpu, cpus_and(allowed_mask, cpu_online_map, irq_affinity[irq]); new_cpu = move(cpu, allowed_mask, now, 1); if (cpu != new_cpu) { - irq_desc_t *desc = irq_desc + irq; - unsigned long flags; - - spin_lock_irqsave(&desc->lock, flags); - pending_irq_cpumask[irq] = cpumask_of_cpu(new_cpu); - spin_unlock_irqrestore(&desc->lock, flags); + set_pending_irq(irq, cpumask_of_cpu(new_cpu)); } } @@ -534,16 +529,12 @@ tryanotherirq: cpus_and(tmp, target_cpu_mask, allowed_mask); if (!cpus_empty(tmp)) { - irq_desc_t *desc = irq_desc + selected_irq; - unsigned long flags; Dprintk("irq = %d moved to cpu = %d\n", selected_irq, min_loaded); /* mark for change destination */ - spin_lock_irqsave(&desc->lock, flags); - pending_irq_cpumask[selected_irq] = - cpumask_of_cpu(min_loaded); - spin_unlock_irqrestore(&desc->lock, flags); + set_pending_irq(selected_irq, cpumask_of_cpu(min_loaded)); + /* Since we made a change, come back sooner to * check for more variation. */ @@ -575,6 +566,7 @@ static int balanced_irq(void *unused) /* push everything to CPU 0 to give us a starting point. */ for (i = 0 ; i < NR_IRQS ; i++) { pending_irq_cpumask[i] = cpumask_of_cpu(0); + set_pending_irq(i, cpumask_of_cpu(0)); } for ( ; ; ) { diff -puN include/linux/irq.h~x86-x86_64-deferred-handling-of-writes-to-proc-irq-xx-smp_affinitypatch-added-to-mm-tree-fix-5 include/linux/irq.h --- 25/include/linux/irq.h~x86-x86_64-deferred-handling-of-writes-to-proc-irq-xx-smp_affinitypatch-added-to-mm-tree-fix-5 2005-05-03 19:13:04.000000000 -0700 +++ 25-akpm/include/linux/irq.h 2005-05-03 19:13:04.000000000 -0700 @@ -67,6 +67,9 @@ typedef struct irq_desc { unsigned int irq_count; /* For detecting broken interrupts */ unsigned int irqs_unhandled; spinlock_t lock; +#if defined (CONFIG_GENERIC_PENDING_IRQ) || defined (CONFIG_IRQBALANCE) + unsigned int move_irq; /* Flag need to re-target intr dest*/ +#endif } ____cacheline_aligned irq_desc_t; extern irq_desc_t irq_desc [NR_IRQS]; @@ -78,7 +81,6 @@ irq_descp (int irq) return irq_desc + irq; } - #include /* the arch dependent stuff */ extern int setup_irq(unsigned int irq, struct irqaction * new); @@ -101,19 +103,33 @@ static inline void set_native_irq_info(i #if defined (CONFIG_GENERIC_PENDING_IRQ) || defined (CONFIG_IRQBALANCE) extern cpumask_t pending_irq_cpumask[NR_IRQS]; -#endif -#ifdef CONFIG_GENERIC_PENDING_IRQ +static inline void set_pending_irq(unsigned int irq, cpumask_t mask) +{ + irq_desc_t *desc = irq_desc + irq; + unsigned long flags; + + spin_lock_irqsave(&desc->lock, flags); + desc->move_irq = 1; + pending_irq_cpumask[irq] = mask; + spin_unlock_irqrestore(&desc->lock, flags); +} + static inline void move_native_irq(int irq) { cpumask_t tmp; irq_desc_t *desc = irq_descp(irq); + if (likely (!desc->move_irq)) + return; + + desc->move_irq = 0; + if (likely(cpus_empty(pending_irq_cpumask[irq]))) return; - if (unlikely(!desc->handler->set_affinity)) + if (!desc->handler->set_affinity) return; /* note - we hold the desc->lock */ @@ -165,10 +181,11 @@ static inline void set_irq_info(int irq, } #endif // CONFIG_PCI_MSI -#else // CONFIG_GENERIC_PENDING_IRQ +#else // CONFIG_GENERIC_PENDING_IRQ || CONFIG_IRQBALANCE #define move_irq(x) #define move_native_irq(x) +#define set_pending_irq(x,y) static inline void set_irq_info(int irq, cpumask_t mask) { set_native_irq_info(irq, mask); diff -puN kernel/irq/proc.c~x86-x86_64-deferred-handling-of-writes-to-proc-irq-xx-smp_affinitypatch-added-to-mm-tree-fix-5 kernel/irq/proc.c --- 25/kernel/irq/proc.c~x86-x86_64-deferred-handling-of-writes-to-proc-irq-xx-smp_affinitypatch-added-to-mm-tree-fix-5 2005-05-03 19:13:04.000000000 -0700 +++ 25-akpm/kernel/irq/proc.c 2005-05-03 19:13:04.000000000 -0700 @@ -22,16 +22,11 @@ static struct proc_dir_entry *smp_affini #ifdef CONFIG_GENERIC_PENDING_IRQ void proc_set_irq_affinity(unsigned int irq, cpumask_t mask_val) { - irq_desc_t *desc = irq_descp(irq); - unsigned long flags; - /* * Save these away for later use. Re-progam when the * interrupt is pending */ - spin_lock_irqsave(&desc->lock, flags); - pending_irq_cpumask[irq] = mask_val; - spin_unlock_irqrestore(&desc->lock, flags); + set_pending_irq(irq, mask_val); } #else void proc_set_irq_affinity(unsigned int irq, cpumask_t mask_val) _