[tools] Commit memory debug (et al) tooling

These are some changes I made to debug tooling while tracking down the
bugfix in the previous commit.

Each `scripts/debug_*_alloc.gdb` script has gdb output a `*_allocs.txt`
file, which in turn can be parsed by the `scripts/parse_*_allocs.py`
script to find errors.

.gitignore

@@ -13,3 +13,6 @@ sysroot
 __pycache__
 /venv
 compile_commands.json
+buddy_allocs.txt
+frame_allocs.txt
+heap_allocs.txt

.vscode/c_cpp_properties.json

@@ -3,12 +3,14 @@
         {
             "name": "Linux",
             "includePath": [
-                "${workspaceFolder}/**"
+                "${workspaceFolder}/src/libraries/**",
+                "${workspaceFolder}/build/**",
+                "${workspaceFolder}/sysroot/include"
             ],
             "defines": [],
             "compilerPath": "/usr/bin/clang",
             "cStandard": "c17",
-            "cppStandard": "c++14",
+            "cppStandard": "c++17",
             "intelliSenseMode": "linux-clang-x64",
             "compileCommands": "${workspaceFolder}/compile_commands.json"
         }

.vscode/launch.json

@@ -18,7 +18,9 @@
             "stopAtConnect": true,
             "stopAtEntry": false,
 
-            "setupCommands": [],
+            "setupCommands": [
+                {"text": "dashboard -enabled off", "ignoreFailures": true}
+            ],
 
             "MIMode": "gdb",
             "miDebuggerServerAddress": "localhost:1234",

assets/debugging/jsix.elf-gdb.py

@@ -4,6 +4,8 @@ import gdb.printing
 import sys
 sys.path.append('./scripts')
 
+import re
+
 from collections import namedtuple
 Capability = namedtuple("Capability", ["id", "parent", "refcount", "caps", "type", "koid"])
 LogEntry = namedtuple("LogHeader", ["id", "bytes", "severity", "area", "message"])
@@ -339,6 +341,23 @@ class DumpLogCommand(gdb.Command):
             print(f"{area:>7}:{level:7}  {entry.message}")
 
 
+class ShowCurrentProcessCommand(gdb.Command):
+    def __init__(self):
+        super().__init__("j6current", gdb.COMMAND_DATA)
+
+    def invoke(self, arg, from_tty):
+        def get_obj_and_id(name):
+            obj = int(gdb.parse_and_eval(f"((cpu_data*)$gs_base)->{name}"))
+            oid = -1
+            if obj != 0:
+                oid = int(gdb.parse_and_eval(f"((obj::kobject*){obj:#x})->m_obj_id"))
+            return obj, oid
+
+        process, pid = get_obj_and_id("process")
+        thread, tid = get_obj_and_id("thread")
+        print(f"{pid:02x}/{tid:02x}  [ {process:x} / {thread:x} ]")
+
+
 class CapTablePrinter:
     def __init__(self, val):
         node_map = val["m_caps"]
@@ -472,8 +491,10 @@ TableWalkCommand()
 GetThreadsCommand()
 PrintProfilesCommand()
 DumpLogCommand()
+ShowCurrentProcessCommand()
 
 gdb.execute("display/i $rip")
+gdb.execute("define hook-quit\nkill\nend")
 if not gdb.selected_inferior().was_attached:
     gdb.execute("add-symbol-file build/panic.serial.elf")
     gdb.execute("target remote :1234")

scripts/debug_buddy_alloc.gdb

@@ -0,0 +1,62 @@
+das -enabled off
+
+break heap_allocator.cpp:200
+commands
+silent
+printf "n %016lx %d\n", m_end, current
+continue
+end
+
+break heap_allocator.cpp:206
+commands
+silent
+printf "N %016lx %d\n", m_end, order
+continue
+end
+
+break heap_allocator::register_free_block
+commands
+silent
+printf "F %016lx %d\n", block, order
+continue
+end
+
+break heap_allocator.cpp:118
+commands
+silent
+printf "f %016lx %d\n", block, info->order
+continue
+end
+
+break heap_allocator.cpp:241
+commands
+silent
+printf "S %016lx %d\n", block, order
+continue
+end
+
+break heap_allocator.cpp:158
+commands
+silent
+printf "P %016lx %d\n", block, order
+continue
+end
+
+break heap_allocator.cpp:180
+commands
+silent
+printf "p %016lx %d\n", block, order
+continue
+end
+
+break heap_allocator.cpp:182
+commands
+silent
+printf "M %016lx %016lx %d\n", block, buddy, order
+continue
+end
+
+set logging file buddy_allocs.txt
+set logging overwrite on
+set logging enabled on
+continue

scripts/debug_frame_alloc.gdb

@@ -0,0 +1,18 @@
+das -enabled off
+break frame_allocator.cpp:62
+commands
+silent
+printf "+ %016lx %3d\n", *address, n
+continue
+end
+break frame_allocator.cpp:95
+commands
+silent
+printf "- %016lx %3d\n", address, count
+continue
+end
+
+set logging file frame_allocs.txt
+set logging overwrite on
+set logging enabled on
+continue

scripts/debug_heap_alloc.gdb

@@ -0,0 +1,29 @@
+das -enabled off
+break heap_allocator.cpp:81
+commands
+silent
+printf "+ %016lx %4d\n", block, length
+continue
+end
+break heap_allocator.cpp:86
+commands
+silent
+printf "+ %016lx %4d\n", block, length
+continue
+end
+break heap_allocator.cpp:120
+commands
+silent
+printf "- %016lx\n", p
+continue
+end
+break heap_allocator.cpp:140
+commands
+silent
+printf "> %016lx %4d %4d\n", p, old_length, new_length
+continue
+end
+set logging file heap_allocs.txt
+set logging overwrite on
+set logging enabled on
+continue

scripts/parse_buddy_allocs.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+
+class block:
+    def __init__(self, start, order, free=False):
+        self.start = start
+        self.order = order
+        self.free = free
+
+    @property
+    def end(self):
+        return self.start + (1<<self.order)
+
+    def overlaps(self, other):
+        return other.start < self.end and other.end > self.start
+
+    def __str__(self):
+        return f"[{self.start:016x} {self.order:2} {self.free and 'free' or 'used'}]"
+
+def get_block(blocks, addr, order, reason):
+    b = blocks.get(addr)
+    if b is None:
+        print(f"ERROR: {reason} unknown block [{addr:016x}, {order}]")
+    elif b.order != order:
+        print(f"ERROR: {reason} block {b} for order {order}")
+    else:
+        return b
+    return None
+ 
+def new_block(blocks, addr, order):
+    b = block(addr, order)
+    for existing in blocks.values():
+        if b.overlaps(existing):
+            print(f"ERROR: new block {b} overlaps existing {existing}")
+    blocks[addr] = b
+
+def free_block(blocks, addr, order, free):
+    s = free and "freeing" or "popping"
+    b = get_block(blocks, addr, order, s)
+    if b and b.free == free:
+        print(f"ERROR: {s} block {b}")
+    elif b:
+        b.free = free
+
+def split_block(blocks, addr, order):
+    b = get_block(blocks, addr, order+1, "splitting")
+    if b:
+        b.order = order
+        buddy = b.start ^ (1<<order)
+        blocks[buddy] = block(buddy, order)
+
+def merge_blocks(blocks, addr1, addr2, order):
+    b1 = get_block(blocks, addr1, order, "merging")
+    b2 = get_block(blocks, addr2, order, "merging")
+    if b1.start > b2.start:
+        b1, b2 = b2, b1
+    del blocks[b2.start]
+    b1.order = order + 1
+
+def parse_line(blocks, line):
+    args = line.strip().split()
+    match args:
+        case ['N', addr, order]:
+            new_block(blocks, int(addr, base=16), int(order))
+        case ['n', addr, order]:
+            new_block(blocks, int(addr, base=16), int(order))
+        case ['P', addr, order]:
+            free_block(blocks, int(addr, base=16), int(order), False)
+        case ['p', addr, order]:
+            free_block(blocks, int(addr, base=16), int(order), False)
+        case ['F', addr, order]:
+            free_block(blocks, int(addr, base=16), int(order), True)
+        case ['S', addr, order]:
+            split_block(blocks, int(addr, base=16), int(order))
+        case ['M', addr1, addr2, order]:
+            merge_blocks(blocks, int(addr1, base=16), int(addr2, base=16), int(order))
+        case _:
+            pass
+
+def parse_file(f):
+    blocks = {}
+    for line in f.readlines():
+        parse_line(blocks, line)
+
+    #for addr in sorted(blocks.keys()):
+    #    print(f"{addr:09x}: {blocks[addr]}")
+
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) > 1:
+        with open(sys.argv[1]) as f:
+            parse_file(f)
+    else:
+        parse_file(sys.stdin)

scripts/parse_frame_allocs.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+def add_maps(allocs, addr, count):
+    for i in range(count):
+        if addr+i in allocs:
+            print(f"ERROR: frame {addr+i:012x} map collision.")
+        else:
+            #print(f"       frame {addr+i:012x} mapped")
+            allocs.add(addr+i)
+
+def remove_maps(allocs, addr, count):
+    for i in range(count):
+        if addr+i not in allocs:
+            print(f" WARN: removing unmapped frame {addr+i:012x}")
+        else:
+            #print(f"       frame {addr+i:012x} unmapped")
+            allocs.remove(addr+i)
+
+def parse_line(allocs, line):
+    args = line.strip().split()
+    match args:
+        case ['+', addr, count]:
+            add_maps(allocs, int(addr, base=16), int(count))
+        case ['-', addr, count]:
+            remove_maps(allocs, int(addr, base=16), int(count))
+        case _:
+            pass
+
+def parse_file(f):
+    allocs = set()
+    for line in f.readlines():
+        parse_line(allocs, line)
+
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) > 1:
+        with open(sys.argv[1]) as f:
+            parse_file(f)
+    else:
+        parse_file(sys.stdin)

scripts/parse_heap_allocs.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+from bisect import bisect_left, insort
+from operator import attrgetter
+
+by_start = attrgetter('start')
+
+class alloc:
+    def __init__(self, start, size):
+        self.start = start
+        self.size = size
+
+    @property
+    def end(self):
+        return self.start + self.size
+
+    def overlaps(self, other):
+        return other.start < self.end and other.end > self.start
+
+    def __str__(self):
+        return f"[{self.start:012x} - {self.end:012x}]"
+
+    def __gt__(self, other):
+        return self.start > other.start
+
+def add_alloc(allocs, addr, length):
+    a = alloc(addr, length)
+    for existing in allocs:
+        if a.overlaps(existing):
+            print(f"ERROR: allocation {a} overlaps existing {existing}")
+    insort(allocs, a)
+
+def remove_alloc(allocs, addr):
+    a = alloc(addr, 0)
+    i = bisect_left(allocs, a)
+    if len(allocs) > i and allocs[i].start == addr:
+        del allocs[i]
+    else:
+        print(f"ERROR: freeing unallocated {a}")
+
+def parse_line(allocs, line):
+    args = line.strip().split()
+    match args:
+        case ['+', addr, length]:
+            add_alloc(allocs, int(addr, base=16), int(length))
+        case ['-', addr]:
+            remove_alloc(allocs, int(addr, base=16))
+        case _:
+            pass
+
+def parse_file(f):
+    allocs = []
+    for line in f.readlines():
+        parse_line(allocs, line)
+
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) > 1:
+        with open(sys.argv[1]) as f:
+            parse_file(f)
+    else:
+        parse_file(sys.stdin)

scripts/wsl_set_vnchost.fish

@@ -0,0 +1 @@
+set -gx VNCHOST (ip -j route list default | jq -r '.[0].gateway'):5500