From 2b3c276f332be73840c8bcd83f69db99384770b3 Mon Sep 17 00:00:00 2001 From: "Justin C. Miller" Date: Tue, 4 Jul 2023 17:43:23 -0700 Subject: [PATCH] [util] Abstract out radix_tree class from page_tree Generalized the radix tree code from page_tree as util::radix_tree so that it can be used elsewhere. --- src/kernel/objects/vm_area.cpp | 3 +- src/kernel/objects/vm_area.h | 3 +- src/kernel/page_tree.cpp | 129 ++----------------- src/kernel/page_tree.h | 44 ++----- src/libraries/util/util.module | 1 + src/libraries/util/util/radix_tree.h | 181 +++++++++++++++++++++++++++ src/libraries/util/util/util.h | 2 +- 7 files changed, 200 insertions(+), 163 deletions(-) create mode 100644 src/libraries/util/util/radix_tree.h diff --git a/src/kernel/objects/vm_area.cpp b/src/kernel/objects/vm_area.cpp index 0bd01f9..c972c91 100644 --- a/src/kernel/objects/vm_area.cpp +++ b/src/kernel/objects/vm_area.cpp @@ -3,7 +3,6 @@ #include "frame_allocator.h" #include "memory.h" #include "objects/vm_area.h" -#include "page_tree.h" #include "vm_space.h" namespace obj { @@ -146,7 +145,7 @@ vm_area_open::get_page(uintptr_t offset, uintptr_t &phys, bool alloc) if (alloc) return page_tree::find_or_add(m_mapped, offset, phys); else - return page_tree::find(m_mapped, offset, phys); + return page_tree::find(m_mapped, offset, &phys); } void diff --git a/src/kernel/objects/vm_area.h b/src/kernel/objects/vm_area.h index 517315b..263bfb6 100644 --- a/src/kernel/objects/vm_area.h +++ b/src/kernel/objects/vm_area.h @@ -11,11 +11,10 @@ #include "block_allocator.h" #include "objects/kobject.h" +#include "page_tree.h" -class page_tree; class vm_space; - namespace obj { enum class vm_flags : uint32_t diff --git a/src/kernel/page_tree.cpp b/src/kernel/page_tree.cpp index 5251a97..b6019ef 100644 --- a/src/kernel/page_tree.cpp +++ b/src/kernel/page_tree.cpp @@ -22,129 +22,14 @@ // Level 6: 003f 0000 0000 0xxx 4T pages / 16 PiB -- 54-bit addressing // Level 7: 0fc0 0000 0000 0xxx 256T pages / 1 EiB -- 60-bit addressing -static_assert(sizeof(page_tree) == 66 * sizeof(uintptr_t)); - -static constexpr unsigned max_level = 5; -static constexpr unsigned bits_per_level = 6; - -inline int level_shift(uint8_t level) { return level * bits_per_level + arch::frame_bits; } -inline uint64_t level_mask(uint8_t level) { return ~0x3full << level_shift(level); } -inline int index_for(uint64_t off, uint8_t level) { return (off >> level_shift(level)) & 0x3full; } - -page_tree::page_tree(uint64_t base, uint8_t level) : - m_base {base & level_mask(level)}, - m_level {level} -{ - memset(m_entries, 0, sizeof(m_entries)); -} - -page_tree::~page_tree() -{ - if (m_level) { - for (auto &e : m_entries) - delete e.child; - } else { - auto &fa = frame_allocator::get(); - for (auto &e : m_entries) { - if (e.entry & 1) - fa.free(e.entry & ~0xfffull, 1); - } - } -} - -bool -page_tree::contains(uint64_t offset, uint8_t &index) const -{ - return (offset & level_mask(m_level)) == m_base; -} - -uintptr_t & -page_tree::get_entry(page_tree * &root, uint64_t offset) -{ - page_tree *level0 = nullptr; - - if (!root) { - // There's no root yet, just make a level0 and make it - // the root. - level0 = new page_tree(offset, 0); - root = level0; - } else { - // Find or insert an existing level0 - page_tree **parent = &root; - page_tree *node = root; - uint8_t parent_level = max_level + 1; - - while (node) { - uint8_t level = node->m_level; - uint8_t index = 0; - if (!node->contains(offset, index)) { - // We found a valid parent but the slot where this node should - // go contains another node. Insert an intermediate parent of - // this node and a new level0 into the parent. - uint64_t other = node->m_base; - uint8_t lcl = parent_level; - while (index_for(offset, lcl) == index_for(other, lcl)) --lcl; - - page_tree *inter = new page_tree(offset, lcl); - inter->m_entries[index_for(other, lcl)].child = node; - *parent = inter; - - level0 = new page_tree(offset, 0); - inter->m_entries[index_for(offset, lcl)].child = level0; - break; - } - - if (!level) { - level0 = node; - break; - } - - parent = &node->m_entries[index].child; - node = *parent; - } - - kassert( node || parent, "Both node and parent were null in page_tree::get_entry"); - - if (!node) { - // We found a parent with an empty spot where this node should - // be. Insert a new level0 there. - level0 = new page_tree(offset, 0); - *parent = level0; - } - } - - kassert(level0, "Got through page_tree::get_entry without a level0"); - - uint8_t index = index_for(offset, 0); - return level0->m_entries[index].entry; -} - -bool -page_tree::find(const page_tree *root, uint64_t offset, uintptr_t &page) -{ - page_tree const *node = root; - while (node) { - uint8_t level = node->m_level; - uint8_t index = 0; - if (!node->contains(offset, index)) - return false; - - if (!level) { - uintptr_t entry = node->m_entries[index].entry; - page = entry & ~0xfffull; - return (entry & 1); // bit 0 marks 'present' - } - - node = node->m_entries[index].child; - } - - return false; -} +static_assert(sizeof(page_tree) == 67 * sizeof(uintptr_t)); bool page_tree::find_or_add(page_tree * &root, uint64_t offset, uintptr_t &page) { - uint64_t &ent = get_entry(root, offset); + node_type * radix_root = root; + uint64_t &ent = radix_tree::find_or_add(radix_root, offset); + root = static_cast(radix_root); if (!(ent & 1)) { // No entry for this page exists, so make one @@ -160,8 +45,10 @@ page_tree::find_or_add(page_tree * &root, uint64_t offset, uintptr_t &page) void page_tree::add_existing(page_tree * &root, uint64_t offset, uintptr_t page) { - uint64_t &ent = get_entry(root, offset); - kassert(!(ent & 1), "Replacing existing mapping in page_tree::add_existing"); + node_type * radix_root = root; + uint64_t &ent = radix_tree::find_or_add(radix_root, offset); + root = static_cast(radix_root); + kassert(!(ent & 1), "Replacing existing mapping in page_tree::add_existing"); ent = page | 1; } diff --git a/src/kernel/page_tree.h b/src/kernel/page_tree.h index db9162c..68018b9 100644 --- a/src/kernel/page_tree.h +++ b/src/kernel/page_tree.h @@ -4,19 +4,17 @@ #include +#include +#include + /// A radix tree node that tracks mapped pages -class page_tree +class page_tree : + public util::radix_tree { public: - /// Get the physical address of the page at the given offset. - /// \arg root The root node of the tree - /// \arg offset Offset into the VMA, in bytes - /// \arg page [out] Receives the page physical address, if found - /// \returns True if a page was found - static bool find(const page_tree *root, uint64_t offset, uintptr_t &page); - /// Get the physical address of the page at the given offset. If one does - /// not exist yet, allocate a page, insert it, and return that. + /// not exist yet, allocate a page, insert it, and return that. Overrides + /// `util::radix_tree::find_or_add`. /// \arg root [inout] The root node of the tree. This pointer may be updated. /// \arg offset Offset into the VMA, in bytes /// \arg page [out] Receives the page physical address, if found @@ -28,32 +26,4 @@ public: /// \arg offset Offset into the VMA, in bytes /// \arg page The mapped page physical address static void add_existing(page_tree * &root, uint64_t offset, uintptr_t page); - - ~page_tree(); - -private: - page_tree(uint64_t base, uint8_t level); - - /// Check if this node should contain the given virtual address - /// \arg offset The offset into the VMA, in bytes - /// \arg index [out] If found, what entry index should contain addr - /// \returns True if the address is contained - bool contains(uintptr_t offset, uint8_t &index) const; - - /// Get a (writable) reference to a page in the tree - static uintptr_t & get_entry(page_tree * &root, uint64_t offset); - - /// Stores the page offset of the start of this node's pages virtual addresses - uint64_t m_base; - - /// Level of this node: 0 maps actual physical pages. Other levels N point to - /// nodes of level N-1. - uint8_t m_level; - - /// For a level 0 node, the entries area all physical page addresses. - /// Other nodes contain pointers to child tree nodes. - union { - uintptr_t entry; - page_tree *child; - } m_entries[64]; }; diff --git a/src/libraries/util/util.module b/src/libraries/util/util.module index d5f0181..72648ad 100644 --- a/src/libraries/util/util.module +++ b/src/libraries/util/util.module @@ -25,6 +25,7 @@ module("util", "util/no_construct.h", "util/node_map.h", "util/pointers.h", + "util/radix_tree.h", "util/spinlock.h", "util/util.h", "util/vector.h", diff --git a/src/libraries/util/util/radix_tree.h b/src/libraries/util/util/radix_tree.h new file mode 100644 index 0000000..62d76b0 --- /dev/null +++ b/src/libraries/util/util/radix_tree.h @@ -0,0 +1,181 @@ +#pragma once +/// \file radix_tree.h +/// Definition of a generic radix_tree structure + +#include +#include +#include + +namespace util { + +template +class radix_tree +{ +public: + using node_type = radix_tree; + + static_assert(sizeof(T) == sizeof(void*), + "Only pointer-sized types are valid radix tree values."); + + /// Get the entry with the given key. + /// \arg root The root node of the tree + /// \arg key Key of the object to find + /// \arg entry [out] points to the entry if found, or null + /// \returns True if an entry was found + static bool find(const node_type *root, uint64_t key, const T *entry) { + node_type const *node = root; + + while (node) { + uint8_t level = node->m_level; + size_t index = 0; + if (!node->contains(key, index)) + return false; + + if (!level) { + entry = &node->m_entries.entries[index]; + return true; + } + + node = node->m_entries.children[index]; + } + + return false; + } + + static bool find(node_type *root, uint64_t key, T *entry) { + node_type *node = root; + + while (node) { + uint8_t level = node->m_level; + size_t index = 0; + if (!node->contains(key, index)) + return false; + + if (!level) { + *entry = node->m_entries.entries[index]; + return true; + } + + node = node->m_entries.children[index]; + } + + return false; + } + + /// Get the entry with the given key. If one does not exist yet, + /// create a new one, insert it, and return that. + /// \arg root [inout] The root node of the tree. This pointer may be updated. + /// \arg key Key of the entry to find + /// \returns A reference to the entry that was found or created + static T& find_or_add(node_type * &root, uint64_t key) { + node_type *leaf = nullptr; + bool existing = false; + + if (!root) { + // There's no root yet, just make a leaf and make it + // the root. + leaf = new node_type(key, 0); + root = leaf; + } else { + // Find or insert an existing leaf + node_type **parent = &root; + node_type *node = root; + while (node) { + uint8_t level = node->m_level; + size_t index = 0; + if (!node->contains(key, index)) { + // We found a valid parent but the slot where this node should + // go contains another node. Insert an intermediate parent of + // this node and a new leaf into the parent. + uint64_t other = node->m_base; + uint8_t lcl = max_level + 1; + while (index_for(key, lcl) == index_for(other, lcl)) --lcl; + + node_type *inter = new node_type(key, lcl); + + size_t other_index = index_for(other, lcl); + inter->m_entries.children[other_index] = node; + *parent = inter; + + leaf = new node_type(key, 0); + size_t key_index = index_for(key, lcl); + inter->m_entries.children[key_index] = leaf; + break; + } + + if (!level) { + leaf = node; + break; + } + + parent = &node->m_entries.children[index]; + node = *parent; + } + + assert( (node || parent) && + "Both node and parent were null in find_or_add"); + + if (!node) { + // We found a parent with an empty spot where this node should + // be. Insert a new leaf there. + leaf = new node_type(key, 0); + *parent = leaf; + } + } + + assert(leaf && "Got through find_or_add without a leaf"); + + uint8_t index = index_for(key, 0); + return leaf->m_entries.entries[index]; + } + + virtual ~radix_tree() { + if (m_level) { + for (auto &c : m_entries.children) + delete c; + } + } + +public: + constexpr static size_t bits_per_level = util::const_log2(N); + +protected: + static inline size_t level_shift(uint8_t level) { return level * bits_per_level + extra_shift; } + static inline size_t level_mask(uint8_t level) { return -1ull << level_shift(level); } + + static inline size_t index_for(uint64_t key, uint8_t level) { + return (key >> level_shift(level)) & (N-1); + } + + radix_tree(uint64_t base, uint8_t level) : + m_base {base & level_mask(level+1)}, + m_level {level} { + memset(&m_entries, 0, sizeof(m_entries)); + } + + /// Check if this node should contain the given key + /// \arg key The key being searched for + /// \arg index [out] If found, what entry index should contain it + /// \returns True if the key is contained + bool contains(uintptr_t key, size_t &index) const { + if ((key & level_mask(m_level+1)) != m_base) + return false; + index = index_for(key, m_level); + return true; + } + + /// The base key of the range this node encompasses + uint64_t m_base; + + /// Level of this node: Level 0 nodes contain entries, other levels M point + /// to nodes of level M-1. + uint8_t m_level; + + union { + T entries[N]; + node_type *children[N]; + } m_entries; + +}; + +} // namespace util diff --git a/src/libraries/util/util/util.h b/src/libraries/util/util/util.h index 83407a5..c0c2f53 100644 --- a/src/libraries/util/util/util.h +++ b/src/libraries/util/util/util.h @@ -8,7 +8,7 @@ namespace util { constexpr size_t const_log2(size_t n) { - return n < 2 ? 1 : 1 + const_log2(n/2); + return n < 2 ? 0 : 1 + const_log2(n/2); } constexpr bool is_pow2(size_t n) {