d0dc12e86b
During memory hotplugging we traverse struct pages three times: 1. memset(0) in sparse_add_one_section() 2. loop in __add_section() to set do: set_page_node(page, nid); and SetPageReserved(page); 3. loop in memmap_init_zone() to call __init_single_pfn() This patch removes the first two loops, and leaves only loop 3. All struct pages are initialized in one place, the same as it is done during boot. The benefits: - We improve memory hotplug performance because we are not evicting the cache several times and also reduce loop branching overhead. - Remove condition from hotpath in __init_single_pfn(), that was added in order to fix the problem that was reported by Bharata in the above email thread, thus also improve performance during normal boot. - Make memory hotplug more similar to the boot memory initialization path because we zero and initialize struct pages only in one function. - Simplifies memory hotplug struct page initialization code, and thus enables future improvements, such as multi-threading the initialization of struct pages in order to improve hotplug performance even further on larger machines. [pasha.tatashin@oracle.com: v5] Link: http://lkml.kernel.org/r/20180228030308.1116-7-pasha.tatashin@oracle.com Link: http://lkml.kernel.org/r/20180215165920.8570-7-pasha.tatashin@oracle.com Signed-off-by: Pavel Tatashin <pasha.tatashin@oracle.com> Reviewed-by: Ingo Molnar <mingo@kernel.org> Cc: Michal Hocko <mhocko@suse.com> Cc: Baoquan He <bhe@redhat.com> Cc: Bharata B Rao <bharata@linux.vnet.ibm.com> Cc: Daniel Jordan <daniel.m.jordan@oracle.com> Cc: Dan Williams <dan.j.williams@intel.com> Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org> Cc: "H. Peter Anvin" <hpa@zytor.com> Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Cc: Mel Gorman <mgorman@techsingularity.net> Cc: Steven Sistare <steven.sistare@oracle.com> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Vlastimil Babka <vbabka@suse.cz> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
147 lines
4.6 KiB
C
147 lines
4.6 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/*
|
|
* include/linux/memory.h - generic memory definition
|
|
*
|
|
* This is mainly for topological representation. We define the
|
|
* basic "struct memory_block" here, which can be embedded in per-arch
|
|
* definitions or NUMA information.
|
|
*
|
|
* Basic handling of the devices is done in drivers/base/memory.c
|
|
* and system devices are handled in drivers/base/sys.c.
|
|
*
|
|
* Memory block are exported via sysfs in the class/memory/devices/
|
|
* directory.
|
|
*
|
|
*/
|
|
#ifndef _LINUX_MEMORY_H_
|
|
#define _LINUX_MEMORY_H_
|
|
|
|
#include <linux/node.h>
|
|
#include <linux/compiler.h>
|
|
#include <linux/mutex.h>
|
|
#include <linux/notifier.h>
|
|
|
|
#define MIN_MEMORY_BLOCK_SIZE (1UL << SECTION_SIZE_BITS)
|
|
|
|
struct memory_block {
|
|
unsigned long start_section_nr;
|
|
unsigned long end_section_nr;
|
|
unsigned long state; /* serialized by the dev->lock */
|
|
int section_count; /* serialized by mem_sysfs_mutex */
|
|
int online_type; /* for passing data to online routine */
|
|
int phys_device; /* to which fru does this belong? */
|
|
void *hw; /* optional pointer to fw/hw data */
|
|
int (*phys_callback)(struct memory_block *);
|
|
struct device dev;
|
|
int nid; /* NID for this memory block */
|
|
};
|
|
|
|
int arch_get_memory_phys_device(unsigned long start_pfn);
|
|
unsigned long memory_block_size_bytes(void);
|
|
|
|
/* These states are exposed to userspace as text strings in sysfs */
|
|
#define MEM_ONLINE (1<<0) /* exposed to userspace */
|
|
#define MEM_GOING_OFFLINE (1<<1) /* exposed to userspace */
|
|
#define MEM_OFFLINE (1<<2) /* exposed to userspace */
|
|
#define MEM_GOING_ONLINE (1<<3)
|
|
#define MEM_CANCEL_ONLINE (1<<4)
|
|
#define MEM_CANCEL_OFFLINE (1<<5)
|
|
|
|
struct memory_notify {
|
|
unsigned long start_pfn;
|
|
unsigned long nr_pages;
|
|
int status_change_nid_normal;
|
|
int status_change_nid_high;
|
|
int status_change_nid;
|
|
};
|
|
|
|
/*
|
|
* During pageblock isolation, count the number of pages within the
|
|
* range [start_pfn, start_pfn + nr_pages) which are owned by code
|
|
* in the notifier chain.
|
|
*/
|
|
#define MEM_ISOLATE_COUNT (1<<0)
|
|
|
|
struct memory_isolate_notify {
|
|
unsigned long start_pfn; /* Start of range to check */
|
|
unsigned int nr_pages; /* # pages in range to check */
|
|
unsigned int pages_found; /* # pages owned found by callbacks */
|
|
};
|
|
|
|
struct notifier_block;
|
|
struct mem_section;
|
|
|
|
/*
|
|
* Priorities for the hotplug memory callback routines (stored in decreasing
|
|
* order in the callback chain)
|
|
*/
|
|
#define SLAB_CALLBACK_PRI 1
|
|
#define IPC_CALLBACK_PRI 10
|
|
|
|
#ifndef CONFIG_MEMORY_HOTPLUG_SPARSE
|
|
static inline int memory_dev_init(void)
|
|
{
|
|
return 0;
|
|
}
|
|
static inline int register_memory_notifier(struct notifier_block *nb)
|
|
{
|
|
return 0;
|
|
}
|
|
static inline void unregister_memory_notifier(struct notifier_block *nb)
|
|
{
|
|
}
|
|
static inline int memory_notify(unsigned long val, void *v)
|
|
{
|
|
return 0;
|
|
}
|
|
static inline int register_memory_isolate_notifier(struct notifier_block *nb)
|
|
{
|
|
return 0;
|
|
}
|
|
static inline void unregister_memory_isolate_notifier(struct notifier_block *nb)
|
|
{
|
|
}
|
|
static inline int memory_isolate_notify(unsigned long val, void *v)
|
|
{
|
|
return 0;
|
|
}
|
|
#else
|
|
extern int register_memory_notifier(struct notifier_block *nb);
|
|
extern void unregister_memory_notifier(struct notifier_block *nb);
|
|
extern int register_memory_isolate_notifier(struct notifier_block *nb);
|
|
extern void unregister_memory_isolate_notifier(struct notifier_block *nb);
|
|
int hotplug_memory_register(int nid, struct mem_section *section);
|
|
#ifdef CONFIG_MEMORY_HOTREMOVE
|
|
extern int unregister_memory_section(struct mem_section *);
|
|
#endif
|
|
extern int memory_dev_init(void);
|
|
extern int memory_notify(unsigned long val, void *v);
|
|
extern int memory_isolate_notify(unsigned long val, void *v);
|
|
extern struct memory_block *find_memory_block_hinted(struct mem_section *,
|
|
struct memory_block *);
|
|
extern struct memory_block *find_memory_block(struct mem_section *);
|
|
#define CONFIG_MEM_BLOCK_SIZE (PAGES_PER_SECTION<<PAGE_SHIFT)
|
|
#endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */
|
|
|
|
#ifdef CONFIG_MEMORY_HOTPLUG
|
|
#define hotplug_memory_notifier(fn, pri) ({ \
|
|
static __meminitdata struct notifier_block fn##_mem_nb =\
|
|
{ .notifier_call = fn, .priority = pri };\
|
|
register_memory_notifier(&fn##_mem_nb); \
|
|
})
|
|
#define register_hotmemory_notifier(nb) register_memory_notifier(nb)
|
|
#define unregister_hotmemory_notifier(nb) unregister_memory_notifier(nb)
|
|
#else
|
|
#define hotplug_memory_notifier(fn, pri) ({ 0; })
|
|
/* These aren't inline functions due to a GCC bug. */
|
|
#define register_hotmemory_notifier(nb) ({ (void)(nb); 0; })
|
|
#define unregister_hotmemory_notifier(nb) ({ (void)(nb); })
|
|
#endif
|
|
|
|
/*
|
|
* Kernel text modification mutex, used for code patching. Users of this lock
|
|
* can sleep.
|
|
*/
|
|
extern struct mutex text_mutex;
|
|
|
|
#endif /* _LINUX_MEMORY_H_ */
|