justin/jsix
1
1
Fork 0
mirror of https://github.com/justinian/jsix.git synced 2025-12-10 00:14:32 -08:00
Code Issues Packages Projects Releases Wiki Activity

Move page table allocation to top 256GiB.

Browse Source 
I forgot to account for tracking page table physical addresses, so
this is a bit of an overhaul. Major changes:
- Refactor bootstrap code into more functions and:
  - Only allocate 32 pages of scratch space
  - Remap remaining space into top 256GiB, the "page table space"
- Use the page table space to directly offset-map page table pages
  from their physical addresses, to avoid tracking overhead.
- Refactor page_block list functions into static functions to better
  handle null/empty lists
This commit is contained in:
Justin C. Miller
2018-04-22 21:52:59 -07:00
parent 571cc5a1da
commit 1de73de2e3
4 changed files with 364 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

316
src/kernel/memory_bootstrap.cpp
View File

@@ -4,6 +4,8 @@
#include "memory.h"
#include "memory_pages.h"
const unsigned efi_page_size = 0x1000;
enum class efi_memory_type : uint32_t
{
reserved,
@@ -137,9 +139,75 @@ count_table_pages_needed(page_block *used)
}
page_block *
remove_block_for(page_block **list, uint64_t phys_start, uint64_t pages, page_block **cache)
{
// This is basically just the removal portion of page_manager::unmap_pages,
// but with physical addresses, and only ever removing a single block.
page_block *prev = nullptr;
page_block *cur = *list;
while (cur && !cur->contains_physical(phys_start)) {
prev = cur;
cur = cur->next;
}
kassert(cur, "Couldn't find block to remove");
uint64_t size = page_manager::page_size * pages;
uint64_t end = phys_start + size;
uint64_t leading = phys_start - cur->physical_address;
uint64_t trailing = cur->physical_end() - end;
if (leading) {
uint64_t pages = leading / page_manager::page_size;
page_block *lead_block = *cache;
*cache = (*cache)->next;
lead_block->copy(cur);
lead_block->next = cur;
lead_block->count = pages;
cur->count -= pages;
cur->physical_address += leading;
if (cur->virtual_address)
cur->virtual_address += leading;
if (prev) {
prev->next = lead_block;
} else {
prev = lead_block;
*list = prev;
}
}
if (trailing) {
uint64_t pages = trailing / page_manager::page_size;
page_block *trail_block = *cache;
*cache = (*cache)->next;
trail_block->copy(cur);
trail_block->next = cur->next;
trail_block->count = pages;
trail_block->physical_address += size;
if (cur->virtual_address)
trail_block->virtual_address += size;
cur->count -= pages;
cur->next = trail_block;
}
prev->next = cur->next;
cur->next = nullptr;
return cur;
}
uint64_t
gather_block_lists(
uint64_t scratch_phys,
uint64_t scratch_virt,
const void *memory_map,
size_t map_length,
@@ -148,8 +216,8 @@ gather_block_lists(
page_block **used_head)
{
int i = 0;
page_block **free = free_head;
page_block **used = used_head;
page_block *free = nullptr;
page_block *used = nullptr;
page_block *block_list = reinterpret_cast<page_block *>(scratch_virt);
efi_memory_descriptor const *desc = reinterpret_cast<efi_memory_descriptor const *>(memory_map);
@@ -171,26 +239,7 @@ gather_block_lists(
case efi_memory_type::boot_services_code:
case efi_memory_type::boot_services_data:
case efi_memory_type::available:
if (scratch_phys >= block->physical_address && scratch_phys < block->physical_end()) {
// This is the scratch memory block, split off what we're not using
block->virtual_address = block->physical_address + page_manager::high_offset;
block->flags = page_block_flags::used;
if (block->count > 1024) {
page_block *rest = &block_list[i++];
rest->physical_address = desc->physical_start + (1024*page_manager::page_size);
rest->virtual_address = 0;
rest->flags = page_block_flags::free;
rest->count = desc->pages - 1024;
rest->next = nullptr;
*free = rest;
free = &rest->next;
block->count = 1024;
}
} else {
block->flags = page_block_flags::free;
}
block->flags = page_block_flags::free;
break;
case efi_memory_type::acpi_reclaim:
@@ -207,101 +256,133 @@ gather_block_lists(
}
if (block->has_flag(page_block_flags::used)) {
if (block->virtual_address != 0)
if (block->virtual_address == block->physical_address)
block->flags |= page_block_flags::mapped;
*used = block;
used = &block->next;
used = page_block::insert_virtual(used, block);
} else {
*free = block;
free = &block->next;
free = page_block::insert_physical(free, block);
}
desc = desc_incr(desc, desc_length);
}
*free_head = free;
*used_head = used;
return reinterpret_cast<uint64_t>(&block_list[i]);
}
page_block *
fill_page_with_blocks(uint64_t start) {
uint64_t end = page_align(start);
page_block *blocks = reinterpret_cast<page_block *>(start);
page_block *endp = reinterpret_cast<page_block *>(end - sizeof(page_block));
if (blocks >= endp)
return nullptr;
uint64_t count = (end - start) / sizeof(page_block);
if (count == 0) return nullptr;
page_block *cur = blocks;
while (cur < endp) {
cur->zero(cur + 1);
cur += 1;
}
cur->next = 0;
page_block *blocks = reinterpret_cast<page_block *>(start);
for (unsigned i = 0; i < count; ++i)
blocks[i].zero(&blocks[i+1]);
blocks[count - 1].next = nullptr;
return blocks;
}
void
copy_new_table(page_table *base, unsigned index, page_table *new_table)
{
uint64_t entry = base->entries[index];
// If this is a large page and not a a table, bail out.
if(entry & 0x80) return;
if (entry & 0x1) {
page_table *old_next = base->next(index);
for (int i = 0; i < 512; ++i) new_table->entries[i] = old_next->entries[i];
} else {
for (int i = 0; i < 512; ++i) new_table->entries[i] = 0;
}
base->entries[index] = reinterpret_cast<uint64_t>(new_table) | 0xb;
}
static uint64_t
find_efi_free_aligned_pages(const void *memory_map, size_t map_length, size_t desc_length, unsigned pages)
{
efi_memory_descriptor const *desc =
reinterpret_cast<efi_memory_descriptor const *>(memory_map);
efi_memory_descriptor const *end = desc_incr(desc, map_length);
const unsigned want_space = pages * page_manager::page_size;
uint64_t start_phys = 0;
for (; desc < end; desc = desc_incr(desc, desc_length)) {
if (desc->type != efi_memory_type::available)
continue;
// See if the first wanted pages fit in one page table. If we
// find free memory at zero, skip ahead because we're not ready
// to deal with 0 being a valid pointer yet.
start_phys = desc->physical_start;
if (start_phys == 0)
start_phys += efi_page_size;
const uint64_t desc_end =
desc->physical_start + desc->pages * efi_page_size;
uint64_t end = start_phys + want_space;
if (end < desc_end) {
page_table_indices start_idx{start_phys};
page_table_indices end_idx{end};
if (start_idx[0] == end_idx[0] &&
start_idx[1] == end_idx[1] &&
start_idx[2] == end_idx[2])
break;
// Try seeing if the page-table-aligned version fits
start_phys = page_table_align(start_phys);
end = start_phys + want_space;
if (end < desc_end)
break;
}
}
kassert(desc < end, "Couldn't find wanted pages of aligned scratch space.");
return start_phys;
}
void
memory_initialize_managers(const void *memory_map, size_t map_length, size_t desc_length)
{
console *cons = console::get();
// The bootloader reserved 4 pages for page tables, which we'll use to bootstrap.
// The bootloader reserved 16 pages for page tables, which we'll use to bootstrap.
// The first one is the already-installed PML4, so grab it from CR3.
page_table *tables = nullptr;
__asm__ __volatile__ ( "mov %%cr3, %0" : "=r" (tables) );
page_table *tables = page_manager::get_pml4();
// Now go through EFi's memory map and find a 4MiB region of free space to
// use as a scratch space. We'll use the 2MiB that fits naturally aligned
// into a single page table.
efi_memory_descriptor const *desc =
reinterpret_cast<efi_memory_descriptor const *>(memory_map);
efi_memory_descriptor const *end = desc_incr(desc, map_length);
while (desc < end) {
if (desc->type == efi_memory_type::available && desc->pages >= 1024)
break;
desc = desc_incr(desc, desc_length);
}
kassert(desc < end, "Couldn't find 4MiB of contiguous scratch space.");
// Now go through EFi's memory map and find a region of scratch space.
const unsigned want_pages = 32;
uint64_t free_region_start_phys =
find_efi_free_aligned_pages(memory_map, map_length, desc_length, want_pages);
// Offset-map this region into the higher half.
uint64_t free_start_phys = desc->physical_start;
uint64_t free_start = free_start_phys + page_manager::high_offset;
uint64_t free_aligned_phys = page_table_align(free_start_phys);
uint64_t free_next = free_aligned_phys + page_manager::high_offset;
uint64_t free_region_start_virt =
free_region_start_phys + page_manager::high_offset;
uint64_t free_next = free_region_start_virt;
// We'll need to copy any existing tables (except the PML4 which the
// bootloader gave us) into our 4 reserved pages so we can edit them.
page_table_indices fr_idx{free_aligned_phys};
fr_idx[0] += 256; // Flip the highest bit of the address
page_table_indices fr_idx{free_region_start_virt};
if (tables[0].entries[fr_idx[0]] & 0x1) {
page_table *old_pdpt = tables[0].next(fr_idx[0]);
for (int i = 0; i < 512; ++i) tables[1].entries[i] = old_pdpt->entries[i];
} else {
for (int i = 0; i < 512; ++i) tables[1].entries[i] = 0;
}
tables[0].entries[fr_idx[0]] = reinterpret_cast<uint64_t>(&tables[1]) | 0xb;
copy_new_table(&tables[0], fr_idx[0], &tables[1]);
copy_new_table(&tables[1], fr_idx[1], &tables[2]);
copy_new_table(&tables[2], fr_idx[2], &tables[3]);
page_in(&tables[0], free_region_start_phys, free_region_start_virt, want_pages, nullptr);
if (tables[1].entries[fr_idx[1]] & 0x1) {
page_table *old_pdt = tables[1].next(fr_idx[1]);
for (int i = 0; i < 512; ++i) tables[2].entries[i] = old_pdt->entries[i];
} else {
for (int i = 0; i < 512; ++i) tables[2].entries[i] = 0;
}
tables[1].entries[fr_idx[1]] = reinterpret_cast<uint64_t>(&tables[2]) | 0xb;
// No need to copy the last-level page table, we're overwriting the whole thing
tables[2].entries[fr_idx[2]] = reinterpret_cast<uint64_t>(&tables[3]) | 0xb;
page_in(&tables[0], free_aligned_phys, free_next, 512, nullptr);
// We now have 2MiB starting at "free_aligned_phys" to bootstrap ourselves. Start by
// We now have pages starting at "free_next" to bootstrap ourselves. Start by
// taking inventory of free pages.
page_block *free_head = nullptr;
page_block *used_head = nullptr;
free_next = gather_block_lists(
free_aligned_phys, free_next,
memory_map, map_length, desc_length,
free_next, memory_map, map_length, desc_length,
&free_head, &used_head);
// Unused page_block structs go here - finish out the current page with them
@@ -309,16 +390,68 @@ memory_initialize_managers(const void *memory_map, size_t map_length, size_t des
free_next = page_align(free_next);
// Now go back through these lists and consolidate
page_block *freed = free_head->list_consolidate();
cache_head->list_append(freed);
page_block *freed = page_block::consolidate(free_head);
cache_head = page_block::append(cache_head, freed);
freed = used_head->list_consolidate();
cache_head->list_append(freed);
freed = page_block::consolidate(used_head);
cache_head = page_block::append(cache_head, freed);
// Pull out the block that represents the bootstrap pages we've used
uint64_t used = free_next - free_region_start_virt;
uint64_t used_pages = used / page_manager::page_size;
uint64_t remaining_pages = want_pages - used_pages;
page_block *removed = remove_block_for(
&free_head,
free_region_start_phys,
used_pages,
&cache_head);
kassert(removed, "remove_block_for didn't find the bootstrap region.");
kassert(removed->physical_address == free_region_start_phys,
"remove_block_for found the wrong region.");
// Add it to the used list
removed->virtual_address = free_region_start_virt;
removed->flags = page_block_flags::used;
used_head = page_block::insert_virtual(used_head, removed);
// Pull out the block that represents the rest
uint64_t free_next_phys = free_region_start_phys + used;
removed = remove_block_for(
&free_head,
free_next_phys,
remaining_pages,
&cache_head);
kassert(removed, "remove_block_for didn't find the page table region.");
kassert(removed->physical_address == free_next_phys,
"remove_block_for found the wrong region.");
uint64_t pt_start_phys = removed->physical_address;
uint64_t pt_start_virt = removed->physical_address + page_manager::page_offset;
// Record that we're about to remap it into the page table address space
removed->virtual_address = pt_start_virt;
removed->flags = page_block_flags::used;
used_head = page_block::insert_virtual(used_head, removed);
// Actually remap them into page table space
page_out(&tables[0], free_next, remaining_pages);
page_table_indices pg_idx{pt_start_virt};
copy_new_table(&tables[0], pg_idx[0], &tables[4]);
copy_new_table(&tables[4], pg_idx[1], &tables[5]);
copy_new_table(&tables[5], pg_idx[2], &tables[6]);
page_in(&tables[0], pt_start_phys, pt_start_virt, remaining_pages, tables + 4);
// Ok, now build an acutal set of kernel page tables that just contains
// what the kernel actually has mapped.
page_table *pages = reinterpret_cast<page_table *>(free_next);
unsigned consumed_pages = 1; // We're about to make a PML4, start with 1:w
page_table *pages = reinterpret_cast<page_table *>(pt_start_virt);
unsigned consumed_pages = 1; // We're about to make a PML4, start with 1
// Finally, remap the existing mappings, but making everything writable
// (especially the page tables themselves)
@@ -333,11 +466,12 @@ memory_initialize_managers(const void *memory_map, size_t map_length, size_t des
free_next += (consumed_pages * page_manager::page_size);
// Put our new PML4 into CR3 to start using it
__asm__ __volatile__ ( "mov %%cr3, %0" : "=r" (pml4) );
// page_manager::set_pml4(pml4);
// We now have all used memory mapped ourselves. Let the page_manager take
// over from here.
g_page_manager.init(
free_head, used_head, cache_head,
free_start, 1024, free_next);
free_region_start_virt, used_pages,
free_next, remaining_pages - consumed_pages);
}