From: Christoph Lameter Add a new section called ".data.mostly_readonly" for data items that are mostly readonly like cpumaps etc. This is done to avoid placement of data that is mostly read into cachelines with variables that are updated frequently. The cachelines in mostly_readonly will rarely acquired as exclusive cachelines. The ability to share these cachelines will allow cpus to keep shared copies of cachelines around. Signed-off-by: Alok N Kataria Signed-off-by: Shobhit Dayal Signed-off-by: Christoph Lameter Signed-off-by: Shai Fultheim Signed-off-by: Andrew Morton --- arch/i386/kernel/cpu/intel.c | 2 +- arch/i386/kernel/smpboot.c | 21 +++++++++++---------- arch/i386/kernel/vmlinux.lds.S | 4 ++++ fs/bio.c | 2 +- include/linux/cache.h | 8 ++++++++ kernel/profile.c | 2 +- lib/radix-tree.c | 2 +- 7 files changed, 27 insertions(+), 14 deletions(-) diff -puN arch/i386/kernel/cpu/intel.c~optimise-storage-of-read-mostly-variables arch/i386/kernel/cpu/intel.c --- 25/arch/i386/kernel/cpu/intel.c~optimise-storage-of-read-mostly-variables 2005-05-03 19:41:46.000000000 -0700 +++ 25-akpm/arch/i386/kernel/cpu/intel.c 2005-05-03 19:41:46.000000000 -0700 @@ -25,7 +25,7 @@ extern int trap_init_f00f_bug(void); /* * Alignment at which movsl is preferred for bulk memory copies. */ -struct movsl_mask movsl_mask; +struct movsl_mask movsl_mask __cacheline_aligned_mostly_readonly; #endif void __init early_intel_workaround(struct cpuinfo_x86 *c) diff -puN arch/i386/kernel/smpboot.c~optimise-storage-of-read-mostly-variables arch/i386/kernel/smpboot.c --- 25/arch/i386/kernel/smpboot.c~optimise-storage-of-read-mostly-variables 2005-05-03 19:41:46.000000000 -0700 +++ 25-akpm/arch/i386/kernel/smpboot.c 2005-05-03 19:41:46.000000000 -0700 @@ -60,13 +60,13 @@ static int __initdata smp_b_stepping; /* Number of siblings per CPU package */ int smp_num_siblings = 1; -int phys_proc_id[NR_CPUS]; /* Package ID of each logical CPU */ +int phys_proc_id[NR_CPUS] __cacheline_aligned_mostly_readonly; /* Package ID of each logical CPU */ EXPORT_SYMBOL(phys_proc_id); -int cpu_core_id[NR_CPUS]; /* Core ID of each logical CPU */ +int cpu_core_id[NR_CPUS] __cacheline_aligned_mostly_readonly; /* Core ID of each logical CPU */ EXPORT_SYMBOL(cpu_core_id); /* bitmap of online cpus */ -cpumask_t cpu_online_map; +cpumask_t cpu_online_map __cacheline_aligned_mostly_readonly; cpumask_t cpu_callin_map; cpumask_t cpu_callout_map; @@ -75,8 +75,8 @@ static cpumask_t smp_commenced_mask; /* Per CPU bogomips and other parameters */ struct cpuinfo_x86 cpu_data[NR_CPUS] __cacheline_aligned; -u8 x86_cpu_to_apicid[NR_CPUS] = - { [0 ... NR_CPUS-1] = 0xff }; +u8 x86_cpu_to_apicid[NR_CPUS] __cacheline_aligned_mostly_readonly = + { [0 ... NR_CPUS-1] = 0xff }; EXPORT_SYMBOL(x86_cpu_to_apicid); /* @@ -474,10 +474,10 @@ extern struct { #ifdef CONFIG_NUMA /* which logical CPUs are on which nodes */ -cpumask_t node_2_cpu_mask[MAX_NUMNODES] = +cpumask_t node_2_cpu_mask[MAX_NUMNODES] __cacheline_aligned_mostly_readonly = { [0 ... MAX_NUMNODES-1] = CPU_MASK_NONE }; /* which node each logical CPU is on */ -int cpu_2_node[NR_CPUS] = { [0 ... NR_CPUS-1] = 0 }; +int cpu_2_node[NR_CPUS] __cacheline_aligned_mostly_readonly = { [0 ... NR_CPUS-1] = 0 }; EXPORT_SYMBOL(cpu_2_node); /* set up a mapping between cpu and node. */ @@ -505,7 +505,8 @@ static inline void unmap_cpu_to_node(int #endif /* CONFIG_NUMA */ -u8 cpu_2_logical_apicid[NR_CPUS] = { [0 ... NR_CPUS-1] = BAD_APICID }; +u8 cpu_2_logical_apicid[NR_CPUS] __cacheline_aligned_mostly_readonly = + { [0 ... NR_CPUS-1] = BAD_APICID }; static void map_cpu_to_logical_apicid(void) { @@ -886,8 +887,8 @@ static int boot_cpu_logical_apicid; /* Where the IO area was mapped on multiquad, always 0 otherwise */ void *xquad_portio; -cpumask_t cpu_sibling_map[NR_CPUS] __cacheline_aligned; -cpumask_t cpu_core_map[NR_CPUS] __cacheline_aligned; +cpumask_t cpu_sibling_map[NR_CPUS] __cacheline_aligned_mostly_readonly; +cpumask_t cpu_core_map[NR_CPUS] __cacheline_aligned_mostly_readonly; static void __init smp_boot_cpus(unsigned int max_cpus) { diff -puN arch/i386/kernel/vmlinux.lds.S~optimise-storage-of-read-mostly-variables arch/i386/kernel/vmlinux.lds.S --- 25/arch/i386/kernel/vmlinux.lds.S~optimise-storage-of-read-mostly-variables 2005-05-03 19:41:46.000000000 -0700 +++ 25-akpm/arch/i386/kernel/vmlinux.lds.S 2005-05-03 19:41:46.000000000 -0700 @@ -50,6 +50,10 @@ SECTIONS . = ALIGN(32); .data.cacheline_aligned : { *(.data.cacheline_aligned) } + /* rarely changed data like cpu maps */ + . = ALIGN(4096); + .data.mostly_readonly : { *(.data.mostly_readonly) } + _edata = .; /* End of data section */ . = ALIGN(THREAD_SIZE); /* init_task */ diff -puN fs/bio.c~optimise-storage-of-read-mostly-variables fs/bio.c --- 25/fs/bio.c~optimise-storage-of-read-mostly-variables 2005-05-03 19:41:46.000000000 -0700 +++ 25-akpm/fs/bio.c 2005-05-03 19:41:46.000000000 -0700 @@ -52,7 +52,7 @@ struct biovec_slab { */ #define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) } -static struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] = { +static struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __cacheline_aligned_mostly_readonly = { BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES), }; #undef BV diff -puN include/linux/cache.h~optimise-storage-of-read-mostly-variables include/linux/cache.h --- 25/include/linux/cache.h~optimise-storage-of-read-mostly-variables 2005-05-03 19:41:46.000000000 -0700 +++ 25-akpm/include/linux/cache.h 2005-05-03 19:41:46.000000000 -0700 @@ -25,6 +25,14 @@ #endif /* CONFIG_SMP */ #endif +#ifdef CONFIG_X86 +#define __cacheline_aligned_mostly_readonly \ + __attribute__((__aligned__(SMP_CACHE_BYTES), \ + __section__(".data.mostly_readonly"))) +#else +#define __cacheline_aligned_mostly_readonly __cacheline_aligned +#endif + #ifndef __cacheline_aligned #define __cacheline_aligned \ __attribute__((__aligned__(SMP_CACHE_BYTES), \ diff -puN kernel/profile.c~optimise-storage-of-read-mostly-variables kernel/profile.c --- 25/kernel/profile.c~optimise-storage-of-read-mostly-variables 2005-05-03 19:41:46.000000000 -0700 +++ 25-akpm/kernel/profile.c 2005-05-03 19:41:46.000000000 -0700 @@ -39,7 +39,7 @@ int (*timer_hook)(struct pt_regs *); static atomic_t *prof_buffer; static unsigned long prof_len, prof_shift; -static int prof_on; +static int prof_on __cacheline_aligned_mostly_readonly; static cpumask_t prof_cpu_mask = CPU_MASK_ALL; #ifdef CONFIG_SMP static DEFINE_PER_CPU(struct profile_hit *[2], cpu_profile_hits); diff -puN lib/radix-tree.c~optimise-storage-of-read-mostly-variables lib/radix-tree.c --- 25/lib/radix-tree.c~optimise-storage-of-read-mostly-variables 2005-05-03 19:41:46.000000000 -0700 +++ 25-akpm/lib/radix-tree.c 2005-05-03 19:41:46.000000000 -0700 @@ -58,7 +58,7 @@ struct radix_tree_path { #define RADIX_TREE_INDEX_BITS (8 /* CHAR_BIT */ * sizeof(unsigned long)) #define RADIX_TREE_MAX_PATH (RADIX_TREE_INDEX_BITS/RADIX_TREE_MAP_SHIFT + 2) -static unsigned long height_to_maxindex[RADIX_TREE_MAX_PATH]; +static unsigned long height_to_maxindex[RADIX_TREE_MAX_PATH] __cacheline_aligned_mostly_readonly; /* * Radix tree node cache. _