[bootproto] Create new bootproto lib

This is a rather large commit that is widely focused on cleaning things
out of the 'junk drawer' that is src/include. Most notably, several
things that were put in there because they needed somewhere where both
the kernel, boot, and init could read them have been moved to a new lib,
'bootproto'.

- Moved kernel_args.h and init_args.h to bootproto as kernel.h and
  init.h, respectively.

- Moved counted.h and pointer_manipulation.h into util, renaming the
  latter to util/pointers.h.

- Created a new src/include/arch for very arch-dependent definitions,
  and moved some kernel_memory.h constants like frame size, page table
  entry count, etc to arch/amd64/memory.h. Also created arch/memory.h
  which detects platform and includes the former.

- Got rid of kernel_memory.h entirely in favor of a new, cog-based
  approach. The new definitions/memory_layout.csv lists memory regions
  in descending order from the top of memory, their sizes, and whether
  they are shared outside the kernel (ie, boot needs to know them). The
  new header bootproto/memory.h exposes the addresses of the shared
  regions, while the kernel's memory.h gains the start and size of all
  the regions. Also renamed the badly-named page-offset area the linear
  area.

- The python build scripts got a few new features: the ability to parse
  the csv mentioned above in a new memory.py module; the ability to add
  dependencies to existing source files (The list of files that I had to
  pull out of the main list just to add them with the dependency on
  memory.h was getting too large. So I put them back into the sources
  list, and added the dependency post-hoc.); and the ability to
  reference 'source_root', 'build_root', and 'module_root' variables in
  .module files.

- Some utility functions that were in the kernel's memory.h got moved to
  util/pointers.h and util/misc.h, and misc.h's byteswap was renamed
  byteswap32 to be more specific.

src/libraries/bootproto/bootproto.module

@@ -0,0 +1,12 @@
+# vim: ft=python
+
+bp = module("bootproto",
+    kind = "lib",
+    includes = [ "include" ],
+    sources = [
+    ])
+
+from os.path import join
+
+layout = join(source_root, "definitions/memory_layout.csv")
+bp.add_input("include/bootproto/memory.h.cog", deps=[layout])

src/libraries/bootproto/include/bootproto/init.h

@@ -0,0 +1,68 @@
+#pragma once
+/// \file bootproto/init.h
+/// Data structures for initializing the init server
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <util/counted.h>
+
+namespace bootproto {
+
+enum class module_type : uint8_t {
+    none,
+    program,
+    framebuffer,
+};
+
+enum class module_flags : uint8_t {
+    none      = 0x00,
+
+    /// This module was already handled by the bootloader,
+    /// no action is needed. The module is included for
+    /// informational purposes only.
+    no_load   = 0x01,
+};
+
+struct module
+{
+    module_type mod_type;
+    module_flags mod_flags;
+    uint32_t mod_length;
+};
+
+struct module_program :
+    public module
+{
+    uintptr_t base_address;
+    size_t size;
+    char filename[];
+};
+
+enum class fb_layout : uint8_t { rgb8, bgr8, unknown = 0xff };
+enum class fb_type : uint8_t { uefi };
+
+struct video_mode
+{
+    uint32_t vertical;
+    uint32_t horizontal;
+    uint32_t scanline;
+    fb_layout layout;
+};
+
+struct module_framebuffer :
+    public module
+{
+    util::buffer framebuffer;
+    video_mode mode;
+    fb_type type;
+};
+
+struct modules_page
+{
+    uint8_t count;
+    module *modules;
+    uintptr_t next;
+};
+
+} // namespace bootproto

src/libraries/bootproto/include/bootproto/kernel.h

@@ -0,0 +1,160 @@
+#pragma once
+/// \file bootproto/kernel.h
+/// Data structures for initializing the kernel
+
+#include <stdalign.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include <util/counted.h>
+
+namespace bootproto {
+
+constexpr uint32_t args_magic = 'j6bp'; // "jsix boot protocol"
+constexpr uint16_t args_version = 1;
+
+constexpr uint64_t header_magic = 0x4c454e52454b366aull; // 'j6KERNEL'
+constexpr uint16_t header_version = 2;
+constexpr uint16_t min_header_version = 2;
+
+enum class section_flags : uint32_t {
+    none    = 0,
+    execute = 1,
+    write   = 2,
+    read    = 4,
+};
+
+struct program_section {
+    uintptr_t phys_addr;
+    uintptr_t virt_addr;
+    uint32_t size;
+    section_flags type;
+};
+
+struct program {
+    uintptr_t entrypoint;
+    uintptr_t phys_base;
+    util::counted<program_section> sections;
+};
+
+enum class mem_type : uint32_t {
+    free,
+    pending,
+    acpi,
+    uefi_runtime,
+    mmio,
+    persistent
+};
+
+/// Structure to hold an entry in the memory map.
+struct mem_entry
+{
+    uintptr_t start;
+    size_t pages;
+    mem_type type;
+    uint32_t attr;
+};
+
+enum class allocation_type : uint8_t {
+    none, page_table, mem_map, frame_map, file, program, init_args,
+};
+
+/// A single contiguous allocation of pages
+struct page_allocation
+{
+    uintptr_t address;
+    uint32_t count;
+    allocation_type type;
+};
+
+/// A page-sized register of page_allocation entries
+struct allocation_register
+{
+    allocation_register *next;
+    uint8_t count;
+
+    uint8_t reserved0;
+    uint16_t reserved1;
+    uint32_t reserved2;
+
+    page_allocation entries[255];
+};
+
+enum class frame_flags : uint32_t {
+    uncacheable         = 0x00000001,
+    write_combining     = 0x00000002,
+    write_through       = 0x00000004,
+    write_back          = 0x00000008,
+    uncache_exported    = 0x00000010,
+
+    write_protect       = 0x00001000,
+    read_protect        = 0x00002000,
+    exec_protect        = 0x00004000,
+    non_volatile        = 0x00008000,
+
+    read_only           = 0x00020000,
+    earmarked           = 0x00040000,
+    hw_crypto           = 0x00080000,
+};
+
+
+constexpr size_t frames_per_block = 64 * 64 * 64;
+
+struct frame_block
+{
+    uintptr_t base;
+    uint32_t count;
+    frame_flags flags;
+    uint64_t map1;
+    uint64_t map2[64];
+    uint64_t *bitmap;
+};
+
+enum class boot_flags : uint16_t {
+    none  = 0x0000,
+    debug = 0x0001
+};
+
+struct args
+{
+    uint32_t magic;
+    uint16_t version;
+    boot_flags flags;
+
+    void *pml4;
+    util::counted<void> page_tables;
+    util::counted<mem_entry> mem_map;
+    util::counted<frame_block> frame_blocks;
+
+    program *kernel;
+    program *init;
+    program *panic;
+    allocation_register *allocations;
+    uintptr_t modules;
+    uintptr_t symbol_table;
+
+    void *runtime_services;
+    void *acpi_table;
+}
+__attribute__((aligned(alignof(max_align_t))));
+
+struct header
+{
+    uint64_t magic;
+
+    uint16_t length;
+    uint16_t version;
+
+    uint16_t version_major;
+    uint16_t version_minor;
+    uint16_t version_patch;
+    uint16_t reserved;
+
+    uint32_t version_gitsha;
+
+    uint64_t flags;
+};
+
+using entrypoint = __attribute__((sysv_abi)) void (*)(args *);
+
+} // namespace bootproto

src/libraries/bootproto/include/bootproto/memory.h.cog

@@ -0,0 +1,24 @@
+#pragma once
+// vim: ft=cpp
+
+/// \file bootproto/memory.h
+/// Import memory layout constants necessary for boot
+
+namespace bootproto {
+namespace mem {
+
+/*[[[cog code generation
+from os.path import join
+from memory import Layout
+
+layout = Layout(join(definitions_path, "memory_layout.csv"))
+
+for region in layout.regions:
+    if region.shared:
+        cog.outl(f"constexpr uintptr_t {region.name}_offset = 0x{region.start:x};")
+
+]]]*/
+///[[[end]]]
+
+} // namespace mem
+} // namespace bootproto

src/libraries/elf/elf.module

@@ -3,6 +3,7 @@
 module("elf",
     kind = "lib",
     includes = [ "include" ],
+    deps = [ "util" ],
     sources = [
         "file.cpp",
     ])

src/libraries/elf/file.cpp

@@ -1,6 +1,5 @@
-#include "elf/file.h"
-#include "elf/headers.h"
-#include "pointer_manipulation.h"
+#include <elf/file.h>
+#include <elf/headers.h>
 
 static const uint32_t expected_magic = 0x464c457f;  // "\x7f" "ELF"
 
@@ -8,9 +7,14 @@ namespace elf {
 
 inline const file_header * fh(const void *data) { return reinterpret_cast<const file_header*>(data); }
 
+template <typename T>
+const T *convert(const void *data, size_t offset) {
+    return reinterpret_cast<const T*>(util::offset_pointer(data, offset));
+}
+
 file::file(const void *data, size_t size) :
-    m_programs(offset_ptr<program_header>(data, fh(data)->ph_offset), fh(data)->ph_entsize, fh(data)->ph_num),
-    m_sections(offset_ptr<section_header>(data, fh(data)->sh_offset), fh(data)->sh_entsize, fh(data)->sh_num),
+    m_programs(convert<program_header>(data, fh(data)->ph_offset), fh(data)->ph_entsize, fh(data)->ph_num),
+    m_sections(convert<section_header>(data, fh(data)->sh_offset), fh(data)->sh_entsize, fh(data)->sh_num),
     m_data(data),
     m_size(size)
 {

src/libraries/elf/include/elf/file.h

@@ -2,7 +2,7 @@
 #include <stddef.h>
 #include <stdint.h>
 
-#include "pointer_manipulation.h"
+#include <util/pointers.h>
 
 namespace elf {
 
@@ -14,7 +14,7 @@ template <typename T>
 class subheaders
 {
 public:
-    using iterator = const_offset_iterator<T>;
+    using iterator = util::const_offset_iterator<T>;
 
     subheaders(const T *start, size_t size, unsigned count) :
         m_start(start), m_size(size), m_count(count) {}
@@ -22,9 +22,9 @@ public:
     inline size_t size() const { return m_size; }
     inline unsigned count() const { return m_count; }
 
-    inline const T & operator [] (int i) const { return *offset_ptr<T>(m_start, m_size*i); }
+    inline const T & operator [] (int i) const { return *util::offset_pointer<T>(m_start, m_size*i); }
     inline const iterator begin() const { return iterator(m_start, m_size); }
-    inline const iterator end() const { return offset_ptr<T>(m_start, m_size*m_count); }
+    inline const iterator end() const { return util::offset_pointer(m_start, m_size*m_count); }
 
 private:
     const T *m_start;

src/libraries/util/include/util/counted.h

@@ -0,0 +1,50 @@
+#pragma once
+/// \file counted.h
+/// Definition of the `counted` template class
+
+#include <util/pointers.h>
+
+namespace util {
+
+/// A pointer and an associated count. Memory pointed to is not managed.
+/// Depending on the usage, the count may be size or number of elements.
+/// Helper methods provide the ability to treat the pointer like an array.
+template <typename T>
+struct counted
+{
+    T *pointer;
+    size_t count;
+
+    /// Index this object as an array of type T
+    inline T & operator [] (int i) { return pointer[i]; }
+
+    /// Index this object as a const array of type T
+    inline const T & operator [] (int i) const { return pointer[i]; }
+
+    using iterator = offset_iterator<T>;
+    using const_iterator = const_offset_iterator<T>;
+
+    /// Return an iterator to the beginning of the array
+    inline iterator begin() { return iterator(pointer, sizeof(T)); }
+
+    /// Return an iterator to the end of the array
+    inline iterator end() { return offset_pointer<T>(pointer, sizeof(T)*count); }
+
+    /// Return an iterator to the beginning of the array
+    inline const_iterator begin() const { return const_iterator(pointer, sizeof(T)); }
+
+    /// Return an iterator to the end of the array
+    inline const_iterator end() const { return offset_pointer<const T>(pointer, sizeof(T)*count); }
+};
+
+/// Specialize for `void` which cannot be indexed or iterated
+template <>
+struct counted<void>
+{
+    void *pointer;
+    size_t count;
+};
+
+using buffer = counted<void>;
+
+} // namespace util

src/libraries/util/include/util/misc.h

@@ -2,11 +2,23 @@
 
 namespace util {
 
+/// Reverse the order of bytes in a 32 bit integer
 constexpr uint32_t
-byteswap(uint32_t x)
-{
+byteswap32(uint32_t x) {
     return ((x >> 24) & 0x000000ff) | ((x >>  8) & 0x0000ff00)
         | ((x <<  8) & 0x00ff0000) | ((x << 24) & 0xff000000);
 }
 
+/// Do a simple byte-wise checksum of an area of memory. The area
+/// summed will be the bytes at indicies [off, len).
+/// \arg p    The start of the memory region
+/// \arg len  The number of bytes in the region
+/// \arg off  An optional offset into the region
+uint8_t checksum(const void *p, size_t len, size_t off = 0) {
+    uint8_t sum = 0;
+    const uint8_t *c = reinterpret_cast<const uint8_t *>(p);
+    for (size_t i = off; i < len; ++i) sum += c[i];
+    return sum;
+}
+
 }

src/libraries/util/include/util/pointers.h

@@ -0,0 +1,95 @@
+/// \file pointers.h
+/// Helper functions and types for doing type-safe byte-wise pointer math.
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+namespace util {
+
+/// Return a pointer offset from `input` by `offset` bytes.
+/// \arg input  The original pointer
+/// \arg offset Offset `input` by this many bytes
+template <typename T>
+inline T* offset_pointer(T* input, ptrdiff_t offset) {
+    return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(input) + offset);
+}
+
+/// Return a pointer with the given bits masked out
+/// \arg input The original pointer
+/// \arg mask  A bitmask of bits to clear from p
+/// \returns   The masked pointer
+template <typename T>
+inline T* mask_pointer(T *input, uintptr_t mask) {
+    return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(input) & ~mask);
+}
+
+/// Read a value of type T from a location in memory
+/// \arg p   The location in memory to read
+/// \returns The value at the given location cast to T
+template <typename T>
+inline T read_from(const void *p) {
+    return *reinterpret_cast<const T *>(p);
+}
+
+/// Iterator for an array of `const T` whose size is known at runtime
+/// \tparam T  Type of the objects in the array, whose size might not be
+///            what is returned by sizeof(T).
+template <typename T>
+class const_offset_iterator
+{
+public:
+    /// Constructor.
+    /// \arg t    Pointer to the first item in the array
+    /// \arg off  Offset applied to reach successive items. Default is 0,
+    ///           which creates an effectively constant iterator.
+    const_offset_iterator(T const *t, size_t off=0) : m_t(t), m_off(off) {}
+
+    const T * operator++() { m_t = offset_pointer(m_t, m_off); return m_t; }
+    const T * operator++(int) { T* tmp = m_t; operator++(); return tmp; }
+
+    bool operator==(T* p) const { return p == m_t; }
+    bool operator!=(T* p) const { return p != m_t; }
+    bool operator==(const_offset_iterator<T> &i) const { return i.m_t == m_t; }
+    bool operator!=(const_offset_iterator<T> &i) const { return i.m_t != m_t; }
+
+    const T& operator*() const { return *m_t; }
+    operator const T& () const { return *m_t; }
+    const T* operator->() const { return m_t; }
+
+private:
+    T const *m_t;
+    size_t m_off;
+};
+
+/// iterator for an array of `const T` whose size is known at runtime
+/// \tparam T  type of the objects in the array, whose size might not be
+///            what is returned by sizeof(T).
+template <typename T>
+class offset_iterator
+{
+public:
+    /// constructor.
+    /// \arg t    pointer to the first item in the array
+    /// \arg off  offset applied to reach successive items. default is 0,
+    ///           which creates an effectively constant iterator.
+    offset_iterator(T *t, size_t off=0) : m_t(t), m_off(off) {}
+
+    T * operator++() { m_t = offset_pointer(m_t, m_off); return m_t; }
+    T * operator++(int) { T* tmp = m_t; operator++(); return tmp; }
+
+    bool operator==(T *p) const { return p == m_t; }
+    bool operator!=(T *p) const { return p != m_t; }
+    bool operator==(offset_iterator<T> &i) const { return i.m_t == m_t; }
+    bool operator!=(offset_iterator<T> &i) const { return i.m_t != m_t; }
+
+    T & operator*() const { return *m_t; }
+    operator T & () const { return *m_t; }
+    T * operator->() const { return m_t; }
+
+private:
+    T *m_t;
+    size_t m_off;
+};
+
+} // namespace util