blob: 0ed00f36350e77cf876bebe2de780393e932d213 [file] [log] [blame]
// SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
/*
* Simple memory allocator
*
* Copyright 2013-2018 IBM Corp.
*/
#include <inttypes.h>
#include <skiboot.h>
#include <mem-map.h>
#include <libfdt_env.h>
#include <lock.h>
#include <device.h>
#include <cpu.h>
#include <chip.h>
#include <affinity.h>
#include <types.h>
#include <mem_region.h>
#include <mem_region-malloc.h>
/* Memory poisoning on free (if POISON_MEM_REGION set to 1) */
#ifdef DEBUG
#define POISON_MEM_REGION 1
#else
#define POISON_MEM_REGION 0
#endif
#define POISON_MEM_REGION_WITH 0x99
#define POISON_MEM_REGION_LIMIT 1*1024*1024*1024
/* Locking: The mem_region_lock protects the regions list from concurrent
* updates. Additions to, or removals from, the region list must be done
* with this lock held. This is typically done when we're establishing
* the memory & reserved regions.
*
* Each region has a lock (region->free_list_lock) to protect the free list
* from concurrent modification. This lock is used when we're allocating
* memory out of a specific region.
*
* If both locks are needed (eg, __local_alloc, where we need to find a region,
* then allocate from it), the mem_region_lock must be acquired before (and
* released after) the per-region lock.
*/
struct lock mem_region_lock = LOCK_UNLOCKED;
static struct list_head regions = LIST_HEAD_INIT(regions);
static struct list_head early_reserves = LIST_HEAD_INIT(early_reserves);
static bool mem_region_init_done = false;
static bool mem_regions_finalised = false;
unsigned long top_of_ram = SKIBOOT_BASE + SKIBOOT_SIZE;
static struct mem_region skiboot_os_reserve = {
.name = "ibm,os-reserve",
.start = 0,
.len = SKIBOOT_BASE,
.type = REGION_OS,
};
struct mem_region skiboot_heap = {
.name = "ibm,firmware-heap",
.start = HEAP_BASE,
.len = HEAP_SIZE,
.type = REGION_SKIBOOT_HEAP,
};
static struct mem_region skiboot_code_and_text = {
.name = "ibm,firmware-code",
.start = SKIBOOT_BASE,
.len = HEAP_BASE - SKIBOOT_BASE,
.type = REGION_SKIBOOT_FIRMWARE,
};
static struct mem_region skiboot_after_heap = {
.name = "ibm,firmware-data",
.start = HEAP_BASE + HEAP_SIZE,
.len = SKIBOOT_BASE + SKIBOOT_SIZE - (HEAP_BASE + HEAP_SIZE),
.type = REGION_SKIBOOT_FIRMWARE,
};
static struct mem_region skiboot_cpu_stacks = {
.name = "ibm,firmware-stacks",
.start = CPU_STACKS_BASE,
.len = 0, /* TBA */
.type = REGION_SKIBOOT_FIRMWARE,
};
static struct mem_region skiboot_mambo_kernel = {
.name = "ibm,firmware-mambo-kernel",
.start = (unsigned long)KERNEL_LOAD_BASE,
.len = KERNEL_LOAD_SIZE,
.type = REGION_SKIBOOT_FIRMWARE,
};
static struct mem_region skiboot_mambo_initramfs = {
.name = "ibm,firmware-mambo-initramfs",
.start = (unsigned long)INITRAMFS_LOAD_BASE,
.len = INITRAMFS_LOAD_SIZE,
.type = REGION_SKIBOOT_FIRMWARE,
};
struct alloc_hdr {
bool free : 1;
bool prev_free : 1;
bool printed : 1;
unsigned long num_longs : BITS_PER_LONG-3; /* Including header. */
const char *location;
};
struct free_hdr {
struct alloc_hdr hdr;
struct list_node list;
/* ... unsigned long tailer; */
};
#define ALLOC_HDR_LONGS (sizeof(struct alloc_hdr) / sizeof(long))
#define ALLOC_MIN_LONGS (sizeof(struct free_hdr) / sizeof(long) + 1)
/* Avoid ugly casts. */
static void *region_start(const struct mem_region *region)
{
return (void *)(unsigned long)region->start;
}
/* Each free block has a tailer, so we can walk backwards. */
static unsigned long *tailer(struct free_hdr *f)
{
return (unsigned long *)f + f->hdr.num_longs - 1;
}
/* This walks forward to the next hdr (or NULL if at the end). */
static struct alloc_hdr *next_hdr(const struct mem_region *region,
const struct alloc_hdr *hdr)
{
void *next;
next = ((unsigned long *)hdr + hdr->num_longs);
if (next >= region_start(region) + region->len)
next = NULL;
return next;
}
#if POISON_MEM_REGION == 1
static void mem_poison(struct free_hdr *f)
{
size_t poison_size = (void*)tailer(f) - (void*)(f+1);
/* We only poison up to a limit, as otherwise boot is
* kinda slow */
if (poison_size > POISON_MEM_REGION_LIMIT)
poison_size = POISON_MEM_REGION_LIMIT;
memset(f+1, POISON_MEM_REGION_WITH, poison_size);
}
#endif
/* Creates free block covering entire region. */
static void init_allocatable_region(struct mem_region *region)
{
struct free_hdr *f = region_start(region);
assert(region->type == REGION_SKIBOOT_HEAP ||
region->type == REGION_MEMORY);
f->hdr.num_longs = region->len / sizeof(long);
f->hdr.free = true;
f->hdr.prev_free = false;
*tailer(f) = f->hdr.num_longs;
list_head_init(&region->free_list);
list_add(&region->free_list, &f->list);
#if POISON_MEM_REGION == 1
mem_poison(f);
#endif
}
static void make_free(struct mem_region *region, struct free_hdr *f,
const char *location, bool skip_poison)
{
struct alloc_hdr *next;
#if POISON_MEM_REGION == 1
if (!skip_poison)
mem_poison(f);
#else
(void)skip_poison;
#endif
if (f->hdr.prev_free) {
struct free_hdr *prev;
unsigned long *prev_tailer = (unsigned long *)f - 1;
assert(*prev_tailer);
prev = (void *)((unsigned long *)f - *prev_tailer);
assert(prev->hdr.free);
assert(!prev->hdr.prev_free);
/* Expand to cover the one we just freed. */
prev->hdr.num_longs += f->hdr.num_longs;
f = prev;
} else {
f->hdr.free = true;
f->hdr.location = location;
list_add(&region->free_list, &f->list);
}
/* Fix up tailer. */
*tailer(f) = f->hdr.num_longs;
/* If next is free, coalesce it */
next = next_hdr(region, &f->hdr);
if (next) {
next->prev_free = true;
if (next->free) {
struct free_hdr *next_free = (void *)next;
list_del_from(&region->free_list, &next_free->list);
/* Maximum of one level of recursion */
make_free(region, next_free, location, true);
}
}
}
/* Can we fit this many longs with this alignment in this free block? */
static bool fits(struct free_hdr *f, size_t longs, size_t align, size_t *offset)
{
*offset = 0;
while (f->hdr.num_longs >= *offset + longs) {
size_t addr;
addr = (unsigned long)f
+ (*offset + ALLOC_HDR_LONGS) * sizeof(long);
if ((addr & (align - 1)) == 0)
return true;
/* Don't make tiny chunks! */
if (*offset == 0)
*offset = ALLOC_MIN_LONGS;
else
(*offset)++;
}
return false;
}
static void discard_excess(struct mem_region *region,
struct alloc_hdr *hdr, size_t alloc_longs,
const char *location, bool skip_poison)
{
/* Do we have excess? */
if (hdr->num_longs > alloc_longs + ALLOC_MIN_LONGS) {
struct free_hdr *post;
/* Set up post block. */
post = (void *)hdr + alloc_longs * sizeof(long);
post->hdr.num_longs = hdr->num_longs - alloc_longs;
post->hdr.prev_free = false;
/* Trim our block. */
hdr->num_longs = alloc_longs;
/* This coalesces as required. */
make_free(region, post, location, skip_poison);
}
}
static const char *hdr_location(const struct alloc_hdr *hdr)
{
/* Corrupt: step carefully! */
if (is_rodata(hdr->location))
return hdr->location;
return "*CORRUPT*";
}
static void bad_header(const struct mem_region *region,
const struct alloc_hdr *hdr,
const char *during,
const char *location)
{
/* Corrupt: step carefully! */
if (is_rodata(hdr->location))
prerror("%p (in %s) %s at %s, previously %s\n",
hdr-1, region->name, during, location, hdr->location);
else
prerror("%p (in %s) %s at %s, previously %p\n",
hdr-1, region->name, during, location, hdr->location);
abort();
}
static bool region_is_reservable(struct mem_region *region)
{
return region->type != REGION_OS;
}
static bool region_is_reserved(struct mem_region *region)
{
return region->type != REGION_OS && region->type != REGION_MEMORY;
}
void mem_dump_allocs(void)
{
struct mem_region *region;
struct alloc_hdr *h, *i;
/* Second pass: populate property data */
prlog(PR_INFO, "Memory regions:\n");
list_for_each(&regions, region, list) {
if (!(region->type == REGION_SKIBOOT_HEAP ||
region->type == REGION_MEMORY))
continue;
prlog(PR_INFO, " 0x%012llx..%012llx : %s\n",
(long long)region->start,
(long long)(region->start + region->len - 1),
region->name);
if (region->free_list.n.next == NULL) {
prlog(PR_INFO, " no allocs\n");
continue;
}
/*
* XXX: When dumping the allocation list we coalase allocations
* with the same location and size into a single line. This is
* quadratic, but it makes the dump human-readable and the raw
* dump sometimes causes the log buffer to wrap.
*/
for (h = region_start(region); h; h = next_hdr(region, h))
h->printed = false;
for (h = region_start(region); h; h = next_hdr(region, h)) {
unsigned long bytes;
int count = 0;
if (h->free)
continue;
if (h->printed)
continue;
for (i = h; i; i = next_hdr(region, i)) {
if (i->free)
continue;
if (i->num_longs != h->num_longs)
continue;
if (strcmp(i->location, h->location))
continue;
i->printed = true;
count++;
}
bytes = h->num_longs * sizeof(long);
prlog(PR_NOTICE, " % 8d allocs of 0x%.8lx bytes at %s (total 0x%lx)\n",
count, bytes, hdr_location(h), bytes * count);
}
}
}
int64_t mem_dump_free(void)
{
struct mem_region *region;
struct alloc_hdr *hdr;
int64_t total_free;
int64_t region_free;
total_free = 0;
prlog(PR_INFO, "Free space in HEAP memory regions:\n");
list_for_each(&regions, region, list) {
if (!(region->type == REGION_SKIBOOT_HEAP ||
region->type == REGION_MEMORY))
continue;
region_free = 0;
if (region->free_list.n.next == NULL) {
continue;
}
for (hdr = region_start(region); hdr; hdr = next_hdr(region, hdr)) {
if (!hdr->free)
continue;
region_free+= hdr->num_longs * sizeof(long);
}
prlog(PR_INFO, "Region %s free: %"PRIx64"\n",
region->name, region_free);
total_free += region_free;
}
prlog(PR_INFO, "Total free: %"PRIu64"\n", total_free);
return total_free;
}
static void *__mem_alloc(struct mem_region *region, size_t size, size_t align,
const char *location)
{
size_t alloc_longs, offset;
struct free_hdr *f;
struct alloc_hdr *next;
/* Align must be power of 2. */
assert(!((align - 1) & align));
/* This should be a constant. */
assert(is_rodata(location));
/* Unallocatable region? */
if (!(region->type == REGION_SKIBOOT_HEAP ||
region->type == REGION_MEMORY))
return NULL;
/* First allocation? */
if (region->free_list.n.next == NULL)
init_allocatable_region(region);
/* Don't do screwy sizes. */
if (size > region->len)
return NULL;
/* Don't do tiny alignments, we deal in long increments. */
if (align < sizeof(long))
align = sizeof(long);
/* Convert size to number of longs, too. */
alloc_longs = (size + sizeof(long)-1) / sizeof(long) + ALLOC_HDR_LONGS;
/* Can't be too small for when we free it, either. */
if (alloc_longs < ALLOC_MIN_LONGS)
alloc_longs = ALLOC_MIN_LONGS;
/* Walk free list. */
list_for_each(&region->free_list, f, list) {
/* We may have to skip some to meet alignment. */
if (fits(f, alloc_longs, align, &offset))
goto found;
}
return NULL;
found:
assert(f->hdr.free);
assert(!f->hdr.prev_free);
/* This block is no longer free. */
list_del_from(&region->free_list, &f->list);
f->hdr.free = false;
f->hdr.location = location;
next = next_hdr(region, &f->hdr);
if (next) {
assert(next->prev_free);
next->prev_free = false;
}
if (offset != 0) {
struct free_hdr *pre = f;
f = (void *)f + offset * sizeof(long);
assert(f >= pre + 1);
/* Set up new header. */
f->hdr.num_longs = pre->hdr.num_longs - offset;
/* f->hdr.prev_free will be set by make_free below. */
f->hdr.free = false;
f->hdr.location = location;
/* Fix up old header. */
pre->hdr.num_longs = offset;
pre->hdr.prev_free = false;
/* This coalesces as required. */
make_free(region, pre, location, true);
}
/* We might be too long; put the rest back. */
discard_excess(region, &f->hdr, alloc_longs, location, true);
/* Clear tailer for debugging */
*tailer(f) = 0;
/* Their pointer is immediately after header. */
return &f->hdr + 1;
}
void *mem_alloc(struct mem_region *region, size_t size, size_t align,
const char *location)
{
static bool dumped = false;
void *r;
assert(lock_held_by_me(&region->free_list_lock));
r = __mem_alloc(region, size, align, location);
if (r)
return r;
prerror("mem_alloc(0x%lx, 0x%lx, \"%s\", %s) failed !\n",
size, align, location, region->name);
if (!dumped) {
mem_dump_allocs();
dumped = true;
}
return NULL;
}
void mem_free(struct mem_region *region, void *mem, const char *location)
{
struct alloc_hdr *hdr;
/* This should be a constant. */
assert(is_rodata(location));
assert(lock_held_by_me(&region->free_list_lock));
/* Freeing NULL is always a noop. */
if (!mem)
return;
/* Your memory is in the region, right? */
assert(mem >= region_start(region) + sizeof(*hdr));
assert(mem < region_start(region) + region->len);
/* Grab header. */
hdr = mem - sizeof(*hdr);
if (hdr->free)
bad_header(region, hdr, "re-freed", location);
make_free(region, (struct free_hdr *)hdr, location, false);
}
size_t mem_allocated_size(const void *ptr)
{
const struct alloc_hdr *hdr = ptr - sizeof(*hdr);
return hdr->num_longs * sizeof(long) - sizeof(struct alloc_hdr);
}
bool mem_resize(struct mem_region *region, void *mem, size_t len,
const char *location)
{
struct alloc_hdr *hdr, *next;
struct free_hdr *f;
/* This should be a constant. */
assert(is_rodata(location));
assert(lock_held_by_me(&region->free_list_lock));
/* Get header. */
hdr = mem - sizeof(*hdr);
if (hdr->free)
bad_header(region, hdr, "resize", location);
/* Round up size to multiple of longs. */
len = (sizeof(*hdr) + len + sizeof(long) - 1) / sizeof(long);
/* Can't be too small for when we free it, either. */
if (len < ALLOC_MIN_LONGS)
len = ALLOC_MIN_LONGS;
/* Shrinking is simple. */
if (len <= hdr->num_longs) {
hdr->location = location;
discard_excess(region, hdr, len, location, false);
return true;
}
/* Check if we can expand. */
next = next_hdr(region, hdr);
if (!next || !next->free || hdr->num_longs + next->num_longs < len)
return false;
/* OK, it's free and big enough, absorb it. */
f = (struct free_hdr *)next;
list_del_from(&region->free_list, &f->list);
hdr->num_longs += next->num_longs;
hdr->location = location;
/* Update next prev_free */
next = next_hdr(region, &f->hdr);
if (next) {
assert(next->prev_free);
next->prev_free = false;
}
/* Clear tailer for debugging */
*tailer(f) = 0;
/* Now we might have *too* much. */
discard_excess(region, hdr, len, location, true);
return true;
}
bool mem_check(const struct mem_region *region)
{
size_t frees = 0;
struct alloc_hdr *hdr, *prev_free = NULL;
struct free_hdr *f;
/* Check it's sanely aligned. */
if (region->start % sizeof(long)) {
prerror("Region '%s' not sanely aligned (%llx)\n",
region->name, (unsigned long long)region->start);
return false;
}
if ((long)region->len % sizeof(long)) {
prerror("Region '%s' not sane length (%llu)\n",
region->name, (unsigned long long)region->len);
return false;
}
/* Not ours to play with, or empty? Don't do anything. */
if (!(region->type == REGION_MEMORY ||
region->type == REGION_SKIBOOT_HEAP) ||
region->free_list.n.next == NULL)
return true;
/* Walk linearly. */
for (hdr = region_start(region); hdr; hdr = next_hdr(region, hdr)) {
if (hdr->num_longs < ALLOC_MIN_LONGS) {
prerror("Region '%s' %s %p (%s) size %zu\n",
region->name, hdr->free ? "free" : "alloc",
hdr, hdr_location(hdr),
hdr->num_longs * sizeof(long));
return false;
}
if ((unsigned long)hdr + hdr->num_longs * sizeof(long) >
region->start + region->len) {
prerror("Region '%s' %s %p (%s) oversize %zu\n",
region->name, hdr->free ? "free" : "alloc",
hdr, hdr_location(hdr),
hdr->num_longs * sizeof(long));
return false;
}
if (hdr->free) {
if (hdr->prev_free || prev_free) {
prerror("Region '%s' free %p (%s) has prev_free"
" %p (%s) %sset?\n",
region->name, hdr, hdr_location(hdr),
prev_free,
prev_free ? hdr_location(prev_free)
: "NULL",
hdr->prev_free ? "" : "un");
return false;
}
prev_free = hdr;
frees ^= (unsigned long)hdr - region->start;
} else {
if (hdr->prev_free != (bool)prev_free) {
prerror("Region '%s' alloc %p (%s) has"
" prev_free %p %sset?\n",
region->name, hdr, hdr_location(hdr),
prev_free, hdr->prev_free ? "" : "un");
return false;
}
prev_free = NULL;
}
}
/* Now walk free list. */
list_for_each(&region->free_list, f, list)
frees ^= (unsigned long)f - region->start;
if (frees) {
prerror("Region '%s' free list and walk do not match!\n",
region->name);
return false;
}
return true;
}
bool mem_check_all(void)
{
struct mem_region *r;
list_for_each(&regions, r, list) {
if (!mem_check(r))
return false;
}
return true;
}
static struct mem_region *new_region(const char *name,
uint64_t start, uint64_t len,
struct dt_node *node,
enum mem_region_type type)
{
struct mem_region *region;
region = malloc(sizeof(*region));
if (!region)
return NULL;
region->name = name;
region->start = start;
region->len = len;
region->node = node;
region->type = type;
region->free_list.n.next = NULL;
init_lock(&region->free_list_lock);
return region;
}
/* We always split regions, so we only have to replace one. */
static struct mem_region *split_region(struct mem_region *head,
uint64_t split_at,
enum mem_region_type type)
{
struct mem_region *tail;
uint64_t end = head->start + head->len;
tail = new_region(head->name, split_at, end - split_at,
head->node, type);
/* Original region becomes head. */
if (tail)
head->len -= tail->len;
return tail;
}
static bool intersects(const struct mem_region *region, uint64_t addr)
{
return addr > region->start &&
addr < region->start + region->len;
}
static bool maybe_split(struct mem_region *r, uint64_t split_at)
{
struct mem_region *tail;
if (!intersects(r, split_at))
return true;
tail = split_region(r, split_at, r->type);
if (!tail)
return false;
/* Tail add is important: we may need to split again! */
list_add_after(&regions, &r->list, &tail->list);
return true;
}
static bool overlaps(const struct mem_region *r1, const struct mem_region *r2)
{
return (r1->start + r1->len > r2->start
&& r1->start < r2->start + r2->len);
}
static bool contains(const struct mem_region *r1, const struct mem_region *r2)
{
u64 r1_end = r1->start + r1->len;
u64 r2_end = r2->start + r2->len;
return (r1->start <= r2->start && r2_end <= r1_end);
}
static struct mem_region *get_overlap(const struct mem_region *region)
{
struct mem_region *i;
list_for_each(&regions, i, list) {
if (overlaps(region, i))
return i;
}
return NULL;
}
static void add_region_to_regions(struct mem_region *region)
{
struct mem_region *r;
list_for_each(&regions, r, list) {
if (r->start < region->start)
continue;
list_add_before(&regions, &r->list, &region->list);
return;
}
list_add_tail(&regions, &region->list);
}
static bool add_region(struct mem_region *region)
{
struct mem_region *r;
if (mem_regions_finalised) {
prerror("MEM: add_region(%s@0x%"PRIx64") called after finalise!\n",
region->name, region->start);
return false;
}
/* First split any regions which intersect. */
list_for_each(&regions, r, list) {
/*
* The new region should be fully contained by an existing one.
* If it's not then we have a problem where reservations
* partially overlap which is probably broken.
*
* NB: There *might* be situations where this is legitimate,
* but the region handling does not currently support this.
*/
if (overlaps(r, region) && !contains(r, region)) {
prerror("MEM: Partial overlap detected between regions:\n");
prerror("MEM: %s [0x%"PRIx64"-0x%"PRIx64"] (new)\n",
region->name, region->start,
region->start + region->len);
prerror("MEM: %s [0x%"PRIx64"-0x%"PRIx64"]\n",
r->name, r->start, r->start + r->len);
return false;
}
if (!maybe_split(r, region->start) ||
!maybe_split(r, region->start + region->len))
return false;
}
/* Now we have only whole overlaps, if any. */
while ((r = get_overlap(region)) != NULL) {
assert(r->start == region->start);
assert(r->len == region->len);
list_del_from(&regions, &r->list);
free(r);
}
/* Finally, add in our own region. */
add_region_to_regions(region);
return true;
}
static void mem_reserve(enum mem_region_type type, const char *name,
uint64_t start, uint64_t len)
{
struct mem_region *region;
bool added = true;
lock(&mem_region_lock);
region = new_region(name, start, len, NULL, type);
assert(region);
if (!mem_region_init_done)
list_add(&early_reserves, &region->list);
else
added = add_region(region);
assert(added);
unlock(&mem_region_lock);
}
void mem_reserve_fw(const char *name, uint64_t start, uint64_t len)
{
mem_reserve(REGION_FW_RESERVED, name, start, len);
}
void mem_reserve_hwbuf(const char *name, uint64_t start, uint64_t len)
{
mem_reserve(REGION_RESERVED, name, start, len);
}
static bool matches_chip_id(const __be32 ids[], size_t num, u32 chip_id)
{
size_t i;
for (i = 0; i < num; i++)
if (be32_to_cpu(ids[i]) == chip_id)
return true;
return false;
}
void *__local_alloc(unsigned int chip_id, size_t size, size_t align,
const char *location)
{
struct mem_region *region;
void *p = NULL;
bool use_local = true;
lock(&mem_region_lock);
restart:
list_for_each(&regions, region, list) {
const struct dt_property *prop;
const __be32 *ids;
if (!(region->type == REGION_SKIBOOT_HEAP ||
region->type == REGION_MEMORY))
continue;
/* Don't allocate from normal heap. */
if (region == &skiboot_heap)
continue;
/* First pass, only match node local regions */
if (use_local) {
if (!region->node)
continue;
prop = dt_find_property(region->node, "ibm,chip-id");
ids = (const __be32 *)prop->prop;
if (!matches_chip_id(ids, prop->len/sizeof(u32),
chip_id))
continue;
}
/* Second pass, match anything */
lock(&region->free_list_lock);
p = mem_alloc(region, size, align, location);
unlock(&region->free_list_lock);
if (p)
break;
}
/*
* If we can't allocate the memory block from the expected
* node, we bail to any one that can accommodate our request.
*/
if (!p && use_local) {
use_local = false;
goto restart;
}
unlock(&mem_region_lock);
return p;
}
static struct mem_region *mem_to_region(void *mem)
{
struct mem_region *region;
list_for_each(&regions, region, list) {
if (mem < region_start(region))
continue;
if (mem >= region_start(region) + region->len)
continue;
return region;
}
return NULL;
}
void __local_free(void *mem, const char *location)
{
struct mem_region *region;
lock(&mem_region_lock);
region = mem_to_region(mem);
if (!region) {
prerror("MEM: local_free mem=%p no matching region.\n", mem);
unlock(&mem_region_lock);
return;
}
lock(&region->free_list_lock);
mem_free(region, mem, location);
unlock(&region->free_list_lock);
unlock(&mem_region_lock);
}
struct mem_region *find_mem_region(const char *name)
{
struct mem_region *region;
list_for_each(&regions, region, list) {
if (streq(region->name, name))
return region;
}
return NULL;
}
bool mem_range_is_reserved(uint64_t start, uint64_t size)
{
uint64_t end = start + size;
struct mem_region *region;
struct list_head *search;
/* We may have the range covered by a number of regions, which could
* appear in any order. So, we look for a region that covers the
* start address, and bump start up to the end of that region.
*
* We repeat until we've either bumped past the end of the range,
* or we didn't find a matching region.
*
* This has a worst-case of O(n^2), but n is well bounded by the
* small number of reservations.
*/
if (!mem_region_init_done)
search = &early_reserves;
else
search = &regions;
for (;;) {
bool found = false;
list_for_each(search, region, list) {
if (!region_is_reserved(region))
continue;
/* does this region overlap the start address, and
* have a non-zero size? */
if (region->start <= start &&
region->start + region->len > start &&
region->len) {
start = region->start + region->len;
found = true;
}
}
/* 'end' is the first byte outside of the range */
if (start >= end)
return true;
if (!found)
break;
}
return false;
}
static void mem_region_parse_reserved_properties(void)
{
const struct dt_property *names, *ranges;
struct mem_region *region;
prlog(PR_DEBUG, "MEM: parsing reserved memory from "
"reserved-names/-ranges properties\n");
names = dt_find_property(dt_root, "reserved-names");
ranges = dt_find_property(dt_root, "reserved-ranges");
if (names && ranges) {
const uint64_t *range;
int n, len;
range = (const void *)ranges->prop;
for (n = 0; n < names->len; n += len, range += 2) {
char *name;
len = strlen(names->prop + n) + 1;
name = strdup(names->prop + n);
region = new_region(name,
dt_get_number(range, 2),
dt_get_number(range + 1, 2),
NULL, REGION_FW_RESERVED);
if (!add_region(region)) {
prerror("Couldn't add mem_region %s\n", name);
abort();
}
}
} else if (names || ranges) {
prerror("Invalid properties: reserved-names=%p "
"with reserved-ranges=%p\n",
names, ranges);
abort();
} else {
return;
}
}
static bool mem_region_parse_reserved_nodes(const char *path)
{
struct dt_node *parent, *node;
parent = dt_find_by_path(dt_root, path);
if (!parent)
return false;
prlog(PR_INFO, "MEM: parsing reserved memory from node %s\n", path);
dt_for_each_child(parent, node) {
const struct dt_property *reg;
struct mem_region *region;
int type;
reg = dt_find_property(node, "reg");
if (!reg) {
char *nodepath = dt_get_path(node);
prerror("node %s has no reg property, ignoring\n",
nodepath);
free(nodepath);
continue;
}
if (dt_has_node_property(node, "no-map", NULL))
type = REGION_RESERVED;
else
type = REGION_FW_RESERVED;
region = new_region(strdup(node->name),
dt_get_number(reg->prop, 2),
dt_get_number(reg->prop + sizeof(u64), 2),
node, type);
if (!add_region(region)) {
char *nodepath = dt_get_path(node);
prerror("node %s failed to add_region()\n", nodepath);
free(nodepath);
}
}
return true;
}
/* Trawl through device tree, create memory regions from nodes. */
void mem_region_init(void)
{
struct mem_region *region, *next;
struct dt_node *i;
bool rc;
/*
* Add associativity properties outside of the lock
* to avoid recursive locking caused by allocations
* done by add_chip_dev_associativity()
*/
dt_for_each_node(dt_root, i) {
if (!dt_has_node_property(i, "device_type", "memory") &&
!dt_has_node_property(i, "compatible", "pmem-region"))
continue;
/* Add associativity properties */
add_chip_dev_associativity(i);
}
/* Add each memory node. */
dt_for_each_node(dt_root, i) {
uint64_t start, len;
char *rname;
#define NODE_REGION_PREFIX "ibm,firmware-allocs-"
if (!dt_has_node_property(i, "device_type", "memory"))
continue;
rname = zalloc(strlen(i->name) + strlen(NODE_REGION_PREFIX) + 1);
assert(rname);
strcat(rname, NODE_REGION_PREFIX);
strcat(rname, i->name);
start = dt_get_address(i, 0, &len);
lock(&mem_region_lock);
region = new_region(rname, start, len, i, REGION_MEMORY);
if (!region) {
prerror("MEM: Could not add mem region %s!\n", i->name);
abort();
}
add_region_to_regions(region);
if ((start + len) > top_of_ram)
top_of_ram = start + len;
unlock(&mem_region_lock);
}
/*
* This is called after we know the maximum PIR of all CPUs,
* so we can dynamically set the stack length.
*/
skiboot_cpu_stacks.len = (cpu_max_pir + 1) * STACK_SIZE;
lock(&mem_region_lock);
/* Now carve out our own reserved areas. */
if (!add_region(&skiboot_os_reserve) ||
!add_region(&skiboot_code_and_text) ||
!add_region(&skiboot_heap) ||
!add_region(&skiboot_after_heap) ||
!add_region(&skiboot_cpu_stacks)) {
prerror("Out of memory adding skiboot reserved areas\n");
abort();
}
if (chip_quirk(QUIRK_MAMBO_CALLOUTS)) {
if (!add_region(&skiboot_mambo_kernel) ||
!add_region(&skiboot_mambo_initramfs)) {
prerror("Out of memory adding mambo payload\n");
abort();
}
}
/* Add reserved reanges from HDAT */
list_for_each_safe(&early_reserves, region, next, list) {
bool added;
list_del(&region->list);
added = add_region(region);
assert(added);
}
/* Add reserved ranges from the DT */
rc = mem_region_parse_reserved_nodes("/reserved-memory");
if (!rc)
rc = mem_region_parse_reserved_nodes(
"/ibm,hostboot/reserved-memory");
if (!rc)
mem_region_parse_reserved_properties();
mem_region_init_done = true;
unlock(&mem_region_lock);
}
static uint64_t allocated_length(const struct mem_region *r)
{
struct free_hdr *f, *last = NULL;
/* No allocations at all? */
if (r->free_list.n.next == NULL)
return 0;
/* Find last free block. */
list_for_each(&r->free_list, f, list)
if (f > last)
last = f;
/* No free blocks? */
if (!last)
return r->len;
/* Last free block isn't at end? */
if (next_hdr(r, &last->hdr))
return r->len;
return (unsigned long)last - r->start;
}
/* Separate out allocated sections into their own region. */
void mem_region_release_unused(void)
{
struct mem_region *r;
lock(&mem_region_lock);
assert(!mem_regions_finalised);
prlog(PR_INFO, "Releasing unused memory:\n");
list_for_each(&regions, r, list) {
uint64_t used_len;
/* If it's not allocatable, ignore it. */
if (!(r->type == REGION_SKIBOOT_HEAP ||
r->type == REGION_MEMORY))
continue;
used_len = allocated_length(r);
prlog(PR_INFO, " %s: %llu/%llu used\n",
r->name, (long long)used_len, (long long)r->len);
/* We keep the skiboot heap. */
if (r == &skiboot_heap)
continue;
/* Nothing used? Whole thing is for Linux. */
if (used_len == 0)
r->type = REGION_OS;
/* Partially used? Split region. */
else if (used_len != r->len) {
struct mem_region *for_linux;
struct free_hdr *last = region_start(r) + used_len;
/* Remove the final free block. */
list_del_from(&r->free_list, &last->list);
for_linux = split_region(r, r->start + used_len,
REGION_OS);
if (!for_linux) {
prerror("OOM splitting mem node %s for linux\n",
r->name);
abort();
}
list_add(&regions, &for_linux->list);
}
}
unlock(&mem_region_lock);
}
static void mem_clear_range(uint64_t s, uint64_t e)
{
uint64_t res_start, res_end;
/* Skip exception vectors */
if (s < EXCEPTION_VECTORS_END)
s = EXCEPTION_VECTORS_END;
/* Skip kernel preload area */
res_start = (uint64_t)KERNEL_LOAD_BASE;
res_end = res_start + KERNEL_LOAD_SIZE;
if (s >= res_start && s < res_end)
s = res_end;
if (e > res_start && e <= res_end)
e = res_start;
if (e <= s)
return;
if (s < res_start && e > res_end) {
mem_clear_range(s, res_start);
mem_clear_range(res_end, e);
return;
}
/* Skip initramfs preload area */
res_start = (uint64_t)INITRAMFS_LOAD_BASE;
res_end = res_start + INITRAMFS_LOAD_SIZE;
if (s >= res_start && s < res_end)
s = res_end;
if (e > res_start && e <= res_end)
e = res_start;
if (e <= s)
return;
if (s < res_start && e > res_end) {
mem_clear_range(s, res_start);
mem_clear_range(res_end, e);
return;
}
prlog(PR_DEBUG, "Clearing region %llx-%llx\n",
(long long)s, (long long)e);
memset((void *)s, 0, e - s);
}
struct mem_region_clear_job_args {
char *job_name;
uint64_t s,e;
};
static void mem_region_clear_job(void *data)
{
struct mem_region_clear_job_args *arg = (struct mem_region_clear_job_args*)data;
mem_clear_range(arg->s, arg->e);
}
#define MEM_REGION_CLEAR_JOB_SIZE (16ULL*(1<<30))
static struct cpu_job **mem_clear_jobs;
static struct mem_region_clear_job_args *mem_clear_job_args;
static int mem_clear_njobs = 0;
void start_mem_region_clear_unused(void)
{
struct mem_region *r;
uint64_t s,l;
uint64_t total = 0;
uint32_t chip_id;
char *path;
int i;
struct cpu_job **jobs;
struct mem_region_clear_job_args *job_args;
lock(&mem_region_lock);
assert(mem_regions_finalised);
mem_clear_njobs = 0;
list_for_each(&regions, r, list) {
if (!(r->type == REGION_OS))
continue;
mem_clear_njobs++;
/* One job per 16GB */
mem_clear_njobs += r->len / MEM_REGION_CLEAR_JOB_SIZE;
}
jobs = malloc(mem_clear_njobs * sizeof(struct cpu_job*));
job_args = malloc(mem_clear_njobs * sizeof(struct mem_region_clear_job_args));
mem_clear_jobs = jobs;
mem_clear_job_args = job_args;
prlog(PR_NOTICE, "Clearing unused memory:\n");
i = 0;
list_for_each(&regions, r, list) {
/* If it's not unused, ignore it. */
if (!(r->type == REGION_OS))
continue;
assert(r != &skiboot_heap);
s = r->start;
l = r->len;
while(l > MEM_REGION_CLEAR_JOB_SIZE) {
job_args[i].s = s+l - MEM_REGION_CLEAR_JOB_SIZE;
job_args[i].e = s+l;
l-=MEM_REGION_CLEAR_JOB_SIZE;
job_args[i].job_name = malloc(sizeof(char)*100);
total+=MEM_REGION_CLEAR_JOB_SIZE;
chip_id = __dt_get_chip_id(r->node);
if (chip_id == -1)
chip_id = 0;
path = dt_get_path(r->node);
snprintf(job_args[i].job_name, 100,
"clear %s, %s 0x%"PRIx64" len: %"PRIx64" on %d",
r->name, path,
job_args[i].s,
(job_args[i].e - job_args[i].s),
chip_id);
free(path);
jobs[i] = cpu_queue_job_on_node(chip_id,
job_args[i].job_name,
mem_region_clear_job,
&job_args[i]);
if (!jobs[i])
jobs[i] = cpu_queue_job(NULL,
job_args[i].job_name,
mem_region_clear_job,
&job_args[i]);
assert(jobs[i]);
i++;
}
job_args[i].s = s;
job_args[i].e = s+l;
job_args[i].job_name = malloc(sizeof(char)*100);
total+=l;
chip_id = __dt_get_chip_id(r->node);
if (chip_id == -1)
chip_id = 0;
path = dt_get_path(r->node);
snprintf(job_args[i].job_name,100,
"clear %s, %s 0x%"PRIx64" len: 0x%"PRIx64" on %d",
r->name, path,
job_args[i].s,
(job_args[i].e - job_args[i].s),
chip_id);
free(path);
jobs[i] = cpu_queue_job_on_node(chip_id,
job_args[i].job_name,
mem_region_clear_job,
&job_args[i]);
if (!jobs[i])
jobs[i] = cpu_queue_job(NULL,
job_args[i].job_name,
mem_region_clear_job,
&job_args[i]);
assert(jobs[i]);
i++;
}
unlock(&mem_region_lock);
cpu_process_local_jobs();
}
void wait_mem_region_clear_unused(void)
{
uint64_t l;
uint64_t total = 0;
int i;
for(i=0; i < mem_clear_njobs; i++) {
total += (mem_clear_job_args[i].e - mem_clear_job_args[i].s);
}
l = 0;
for(i=0; i < mem_clear_njobs; i++) {
cpu_wait_job(mem_clear_jobs[i], true);
l += (mem_clear_job_args[i].e - mem_clear_job_args[i].s);
printf("Clearing memory... %"PRIu64"/%"PRIu64"GB done\n",
l>>30, total>>30);
free(mem_clear_job_args[i].job_name);
}
free(mem_clear_jobs);
free(mem_clear_job_args);
}
static void mem_region_add_dt_reserved_node(struct dt_node *parent,
struct mem_region *region)
{
char *name, *p;
/* If a reserved region was established before skiboot, it may be
* referenced by a device-tree node with extra data. In that case,
* copy the node to /reserved-memory/, unless it's already there.
*
* We update region->node to the new copy here, as the prd code may
* update regions' device-tree nodes, and we want those updates to
* apply to the nodes in /reserved-memory/.
*/
if (region->type == REGION_FW_RESERVED && region->node) {
if (region->node->parent != parent)
region->node = dt_copy(region->node, parent);
return;
}
name = strdup(region->name);
assert(name);
/* remove any cell addresses in the region name; we have our own cell
* addresses here */
p = strchr(name, '@');
if (p)
*p = '\0';
region->node = dt_new_addr(parent, name, region->start);
assert(region->node);
dt_add_property_u64s(region->node, "reg", region->start, region->len);
/*
* This memory is used by hardware and may need special handling. Ask
* the host kernel not to map it by default.
*/
if (region->type == REGION_RESERVED)
dt_add_property(region->node, "no-map", NULL, 0);
free(name);
}
void mem_region_add_dt_reserved(void)
{
int names_len, ranges_len, len;
const struct dt_property *prop;
struct mem_region *region;
void *names, *ranges;
struct dt_node *node;
fdt64_t *range;
char *name;
names_len = 0;
ranges_len = 0;
/* Finalise the region list, so we know that the regions list won't be
* altered after this point. The regions' free lists may change after
* we drop the lock, but we don't access those. */
lock(&mem_region_lock);
mem_regions_finalised = true;
/* establish top-level reservation node */
node = dt_find_by_path(dt_root, "reserved-memory");
if (!node) {
node = dt_new(dt_root, "reserved-memory");
dt_add_property_cells(node, "#address-cells", 2);
dt_add_property_cells(node, "#size-cells", 2);
dt_add_property(node, "ranges", NULL, 0);
}
prlog(PR_INFO, "Reserved regions:\n");
/* First pass, create /reserved-memory/ nodes for each reservation,
* and calculate the length for the /reserved-names and
* /reserved-ranges properties */
list_for_each(&regions, region, list) {
if (!region_is_reservable(region))
continue;
prlog(PR_INFO, " 0x%012llx..%012llx : %s\n",
(long long)region->start,
(long long)(region->start + region->len - 1),
region->name);
mem_region_add_dt_reserved_node(node, region);
/* calculate the size of the properties populated later */
names_len += strlen(region->node->name) + 1;
ranges_len += 2 * sizeof(uint64_t);
}
name = names = malloc(names_len);
range = ranges = malloc(ranges_len);
/* Second pass: populate the old-style reserved-names and
* reserved-regions arrays based on the node data */
list_for_each(&regions, region, list) {
if (!region_is_reservable(region))
continue;
len = strlen(region->node->name) + 1;
memcpy(name, region->node->name, len);
name += len;
range[0] = cpu_to_fdt64(region->start);
range[1] = cpu_to_fdt64(region->len);
range += 2;
}
unlock(&mem_region_lock);
prop = dt_find_property(dt_root, "reserved-names");
if (prop)
dt_del_property(dt_root, (struct dt_property *)prop);
prop = dt_find_property(dt_root, "reserved-ranges");
if (prop)
dt_del_property(dt_root, (struct dt_property *)prop);
dt_add_property(dt_root, "reserved-names", names, names_len);
dt_add_property(dt_root, "reserved-ranges", ranges, ranges_len);
free(names);
free(ranges);
}
struct mem_region *mem_region_next(struct mem_region *region)
{
struct list_node *node;
assert(lock_held_by_me(&mem_region_lock));
node = region ? &region->list : &regions.n;
if (node->next == &regions.n)
return NULL;
return list_entry(node->next, struct mem_region, list);
}