WIP linux multiarch support

Changing bonnibel to respect the --arch flag to configure. This requires
some reworking of modules, mostly in the addition of the ModuleList
class instead of just a dict of modules.

.github/workflows/sphinx_deploy.yml

@@ -0,0 +1,56 @@
+name: Deploy docs site with Sphinx
+on:
+  push:
+    branches: ["main"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+# Default to bash
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Setup Pages
+        id: pages
+        uses: actions/configure-pages@v4
+      - name: Sphinx build
+        uses: jsix-os/sphinx-action@master
+        with:
+          docs-folder: "docs/"
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: ./docs/_build/html
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.gitignore

@@ -3,7 +3,7 @@
 /build*
 *.bak
 tags
-jsix.log
+*.log
 *.out
 *.o
 *.a
@@ -13,3 +13,7 @@ sysroot
 __pycache__
 /venv
 compile_commands.json
+buddy_allocs.txt
+frame_allocs.txt
+heap_allocs.txt
+/docs/_build

.vscode/c_cpp_properties.json

@@ -0,0 +1,19 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/src/libraries/**",
+                "${workspaceFolder}/build/**",
+                "${workspaceFolder}/sysroot/include"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/bin/clang",
+            "cStandard": "c17",
+            "cppStandard": "c++17",
+            "intelliSenseMode": "linux-clang-x64",
+            "compileCommands": "${workspaceFolder}/compile_commands.json"
+        }
+    ],
+    "version": 4
+}

.vscode/launch.json

@@ -0,0 +1,32 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "QEMU Debug Server",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "${workspaceFolder}/build/jsix.elf",
+            "args": [],
+            "cwd": "${workspaceFolder}",
+            "logging": {
+                //"engineLogging": true,
+                "programOutput": true
+            },  
+            "stopAtConnect": true,
+            "stopAtEntry": false,
+
+            "setupCommands": [
+                {"text": "dashboard -enabled off", "ignoreFailures": true}
+            ],
+
+            "MIMode": "gdb",
+            "miDebuggerServerAddress": "localhost:1234",
+            "debugServerPath": "${workspaceFolder}/qemu.sh",
+            "debugServerArgs": "--debug --no-build",
+            "serverLaunchTimeout": 5000,
+        }
+    ]
+}

.vscode/tasks.json

@@ -1,28 +1,41 @@
 {
     // See https://go.microsoft.com/fwlink/?LinkId=733558
     // for the documentation about the tasks.json format
-    "version": "0.1.0",
+    "version": "2.0.0",
     "tasks": [
         {
-            "taskName": "make",
-            "command": "make.bat",
-            "isBuildCommand": true
+            "label": "Ninja",
+            "type": "shell",
+            "command": "source ${workspaceFolder}/venv/bin/activate.fish; ninja",
+            "detail": "Build the project",
+            "options": {
+                "cwd": "${workspaceFolder}/build"
+            },
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            },
+            "problemMatcher": []
         },
         {
-            "taskName": "clean",
-            "command": "make.bat",
-            "args": [ "clean" ],
-            "isShellCommand": true
+            "label": "Run QEMU",
+            "command": "./qemu.sh",
+            "args": [ "--no-build", "--kvm" ],
+            "dependsOn": ["Ninja"],
+            "options": {
+                "cwd": "${workspaceFolder}"
+            },
         },
         {
-            "taskName": "qemu (windowed)",
-            "command": "qemu-win.bat",
-            "showOutput": "never",
-            "isTestCommand": true
+            "label": "clean",
+            "command": "${workspaceFolder}/venv/bin/ninja",
+            "options": {
+                "cwd": "${workspaceFolder}/build"
             },
-        {
-            "taskName": "qemu",
-            "command": "qemu.bat"
+            "args": [
+                "-t",
+                "clean"
+            ]
         }
     ]
 }

assets/build/amd64/config.debug.yaml

@@ -0,0 +1,9 @@
+---
+ccflags: [
+  "-g3",
+  "-ggdb",
+]
+
+ldflags: [
+  "-g",
+]

assets/build/amd64/config.release.yaml

@@ -0,0 +1,3 @@
+ccflags: [
+  "-O3",
+]

assets/build/amd64/global.yaml

@@ -0,0 +1,39 @@
+---
+cc: "${source_root}/sysroot/bin/clang"
+cxx: "${source_root}/sysroot/bin/clang++"
+ld: "${source_root}/sysroot/bin/ld.lld"
+ar: ar
+nasm: nasm
+objcopy: objcopy
+
+ccflags: [
+  "-I${source_root}/src/include",
+  "-fcolor-diagnostics",
+  "-U__STDCPP_THREADS__",
+  "-D_LIBCPP_HAS_NO_THREADS",
+  "-D__jsix_config=${build_config}",
+  "-D__jsix_config_${build_config}",
+  "-DVERSION_MAJOR=${version_major}",
+  "-DVERSION_MINOR=${version_minor}",
+  "-DVERSION_PATCH=${version_patch}",
+  "-DVERSION_GITSHA=0x${version_sha}",
+  '-DGIT_VERSION=\"${version_major}.${version_minor}.${version_patch}+${version_sha}\"',
+  '-DGIT_VERSION_WIDE=L\"${version_major}.${version_minor}.${version_patch}+${version_sha}\"',
+
+  "-Wformat=2", "-Winit-self", "-Winline", "-Wmissing-format-attribute",
+  "-Wmissing-include-dirs", "-Wswitch", "-Wundef", "-Wdisabled-optimization",
+  "-Wpointer-arith", "-Wno-attributes", "-Wno-sign-compare", "-Wno-multichar",
+  "-Wno-div-by-zero", "-Wno-endif-labels", "-Wno-pragmas", "-Wno-format-extra-args",
+  "-Wno-unused-result", "-Wno-deprecated-declarations", "-Wno-unused-function",
+  "-Wno-address-of-packed-member", "-Wno-invalid-offsetof", "-Wno-format-nonliteral",
+  "-Werror" ]
+
+asflags: [
+  "-DVERSION_MAJOR=${version_major}",
+  "-DVERSION_MINOR=${version_minor}",
+  "-DVERSION_PATCH=${version_patch}",
+  "-DVERSION_GITSHA=0x${version_sha}",
+  "-I${source_root}/src/include" ]
+
+cflags: [ "-std=c11" ]
+cxxflags: [ "-std=c++17" ]

assets/build/amd64/rules.ninja

@@ -1,8 +1,6 @@
-# This file is automatically generated by bonnibel
-
 rule compile.c
   command = $cc -MMD -MF $out.d $cflags $ccflags -o $out -c $in
-  description = Compiling $name
+  description = Compiling [$target]:$name
   depfile = $out.d
   deps = gcc
 
@@ -17,7 +15,7 @@ rule dump_c_run
 
 rule compile.cpp
   command = $cxx -MMD -MF $out.d $cxxflags $ccflags -o $out -c $in
-  description = Compiling $name
+  description = Compiling [$target]:$name
   depfile = $out.d
   deps = gcc
 
@@ -32,25 +30,33 @@ rule dump_cpp_run
 
 rule compile.s
   command = $nasm -o $out -felf64 -MD $out.d $asflags $in
-  description = Assembling $name
+  description = Assembling [$target]:$name
   depfile = $out.d
   deps = gcc
 
 rule parse.cog
   command = cog -o $out -d -D target=$target $cogflags $in
-  description = Parsing $name
+  description = Parsing [$target]:$name
 
 rule exe
   command = $ld $ldflags -o $out $in $libs
-  description = Linking $name
+  description = Linking exe [$target]:$name
+
+rule driver
+  command = $ld $ldflags -o $out $in $libs
+  description = Linking driver [$target]:$name
 
 rule lib
+  command = $ld -shared -soname $soname $ldflags -o $out $in $libs
+  description = Linking [$target]:$name
+
+rule lib_static
   command = $ar qcs $out $in
-  description = Archiving $name
+  description = Archiving [$target]:$name
 
 rule cp
   command = cp $in $out
-  description = Copying $name
+  description = Copying [$target]:$name
 
 rule dump
   command = objdump -DSC -M intel $in > $out

assets/build/amd64/target.boot.yaml

@@ -0,0 +1,26 @@
+---
+ld: clang++
+
+ccflags: [
+  "-nostdlib",
+  "-nodefaultlibs",
+  "-fno-builtin",
+
+  "-I${source_root}/external",
+  "--target=x86_64-unknown-windows",
+  "-ffreestanding",
+  "-mno-red-zone",
+  "-fshort-wchar",
+  "-fno-omit-frame-pointer",
+]
+
+cxxflags: [ "-fno-exceptions", "-fno-rtti" ]
+
+ldflags: [
+  "--target=x86_64-unknown-windows",
+  "-nostdlib",
+  "-Wl,-entry:efi_main",
+  "-Wl,-subsystem:efi_application",
+  "-fuse-ld=lld-link",
+]
+

assets/build/amd64/target.init.yaml

@@ -0,0 +1,40 @@
+---
+ccflags: [
+  "--target=x86_64-jsix-elf",
+  "-fno-omit-frame-pointer",
+  "-fno-stack-protector",
+
+  "-fvisibility=hidden",
+  "-fvisibility-inlines-hidden",
+
+  "-D__ELF__",
+  "-D__jsix__",
+  "-U__linux",
+  "-U__linux__",
+
+  "-DMSPACES",
+
+  "--sysroot='${source_root}/sysroot'"
+]
+
+
+cxxflags: [
+  "-fno-exceptions",
+  "-fno-rtti",
+  "-isystem", "${source_root}/sysroot/include/c++/v1",
+]
+
+ldflags: [
+  "-Bstatic",
+  "-m", "elf_x86_64",
+  "--sysroot='${source_root}/sysroot'",
+  "--no-eh-frame-hdr",
+  "-L", "${source_root}/sysroot/lib",
+  "-z", "separate-code",
+  "-lc++", "-lc++abi", "-lunwind",
+  "--no-dependent-libraries",
+]
+
+libs: [
+  "${target_dir}/crt0.o",
+]

assets/build/amd64/target.kernel.yaml

@@ -0,0 +1,52 @@
+---
+asflags: [ "-I${source_root}/src/kernel/" ]
+
+ccflags: [
+  "--target=x86_64-jsix-elf",
+  "-fno-stack-protector",
+
+  "-I${source_root}/external",
+
+  "-nostdinc",
+  "-nostdlib",
+  "-ffreestanding",
+  "-nodefaultlibs",
+  "-fno-builtin",
+  "-fno-plt",
+
+  "-mno-sse",
+  "-fno-omit-frame-pointer",
+  "-mno-red-zone",
+  "-mcmodel=kernel",
+  "-fvisibility=hidden",
+  "-fvisibility-inlines-hidden",
+
+  "-D__ELF__",
+  "-D__jsix__",
+  "-D__j6kernel",
+  "-U__linux",
+  "-U__linux__",
+  "-DPRINTF_ALIAS_STANDARD_FUNCTION_NAMES=1",
+  "-DPRINTF_INCLUDE_CONFIG_H=1",
+
+  "--sysroot='${source_root}/sysroot'"
+]
+
+
+cflags: [ '-nostdinc' ]
+
+cxxflags: [
+  "-fno-exceptions",
+  "-fno-rtti",
+  "-nostdinc",
+]
+
+ldflags: [
+  "-m", "elf_x86_64",
+  "-nostdlib",
+  "-Bstatic",
+  "--no-eh-frame-hdr",
+  "-z", "norelro",
+  "-z", "separate-code"
+]
+

assets/build/amd64/target.user.exe.yaml

@@ -0,0 +1,15 @@
+---
+
+ccflags: [
+  "-fpie"
+]
+
+ldflags: [
+  "-pie",
+  "--dynamic-linker", "/jsix/lib/ld.so",
+  "--push-state", "--as-needed", "-Bstatic", "-lc++", "-lc++abi", "-lunwind", "--pop-state",
+]
+
+libs: [
+  "${target_dir}/crt0.o",
+]

assets/build/amd64/target.user.shared.yaml

@@ -0,0 +1,7 @@
+---
+ccflags: [
+]
+
+ldflags: [
+  "-shared",
+]

assets/build/amd64/target.user.yaml

@@ -0,0 +1,34 @@
+---
+asflags: []
+
+ccflags: [
+  "--target=x86_64-jsix-elf",
+  "-fno-omit-frame-pointer",
+  "-fno-stack-protector",
+
+  "-fvisibility=hidden",
+  "-fvisibility-inlines-hidden",
+
+  "-D__ELF__",
+  "-D__jsix__",
+  "-U__linux",
+  "-U__linux__",
+
+  "--sysroot='${source_root}/sysroot'",
+  "-fpic",
+]
+
+cxxflags: [
+  "-fno-exceptions",
+  "-fno-rtti",
+  "-isystem", "${source_root}/sysroot/include/c++/v1",
+  ]
+
+ldflags: [
+  "-m", "elf_x86_64",
+  "--sysroot='${source_root}/sysroot'",
+  "--no-eh-frame-hdr",
+  "-L", "${source_root}/sysroot/lib",
+  "-z", "separate-code",
+  "--no-dependent-libraries",
+]

assets/build/base.yaml

@@ -1,39 +0,0 @@
----
-variables:
-  cc: "${source_root}/sysroot/bin/clang"
-  cxx: "${source_root}/sysroot/bin/clang++"
-  ld: "${source_root}/sysroot/bin/ld.lld"
-  ar: ar
-  nasm: nasm
-  objcopy: objcopy
-
-  ccflags: [
-    "-I${source_root}/src/include",
-    "-I${source_root}/src/include/x86_64",
-    "-fcolor-diagnostics",
-    "-U__STDCPP_THREADS__",
-    "-D_LIBCPP_HAS_NO_THREADS",
-    "-DVERSION_MAJOR=${version_major}",
-    "-DVERSION_MINOR=${version_minor}",
-    "-DVERSION_PATCH=${version_patch}",
-    "-DVERSION_GITSHA=0x${version_sha}",
-    '-DGIT_VERSION=\"${version_major}.${version_minor}.${version_patch}+${version_sha}\"',
-    '-DGIT_VERSION_WIDE=L\"${version_major}.${version_minor}.${version_patch}+${version_sha}\"',
-
-    "-Wformat=2", "-Winit-self", "-Wfloat-equal", "-Winline", "-Wmissing-format-attribute",
-    "-Wmissing-include-dirs", "-Wswitch", "-Wundef", "-Wdisabled-optimization",
-    "-Wpointer-arith", "-Wno-attributes", "-Wno-sign-compare", "-Wno-multichar",
-    "-Wno-div-by-zero", "-Wno-endif-labels", "-Wno-pragmas", "-Wno-format-extra-args",
-    "-Wno-unused-result", "-Wno-deprecated-declarations", "-Wno-unused-function",
-    "-Wno-address-of-packed-member", "-Wno-invalid-offsetof", "-Wno-format-nonliteral",
-    "-Werror" ]
-
-  asflags: [
-    "-DVERSION_MAJOR=${version_major}",
-    "-DVERSION_MINOR=${version_minor}",
-    "-DVERSION_PATCH=${version_patch}",
-    "-DVERSION_GITSHA=0x${version_sha}",
-    "-I${source_root}/src/include" ]
-
-  cflags: [ "-std=c11" ]
-  cxxflags: [ "-std=c++17" ]

assets/build/boot-debug.yaml

@@ -1,30 +0,0 @@
----
-extends: base
-
-variables:
-  ld: clang++
-
-  ccflags: [
-    "-nostdlib",
-    "-nodefaultlibs",
-    "-fno-builtin",
-
-    "-I${source_root}/external",
-    "--target=x86_64-unknown-windows",
-    "-ffreestanding",
-    "-mno-red-zone",
-    "-fshort-wchar",
-    "-fno-omit-frame-pointer",
-    "-ggdb",
-    "-g3" ]
-
-  cxxflags: [ "-fno-exceptions", "-fno-rtti" ]
-
-  ldflags: [
-    "--target=x86_64-unknown-windows",
-    "-nostdlib",
-    "-Wl,-entry:efi_main",
-    "-Wl,-subsystem:efi_application",
-    "-fuse-ld=lld-link",
-    "-g" ]
-

assets/build/kernel-debug.yaml

@@ -1,59 +0,0 @@
----
-extends: base
-
-variables:
-  asflags: [ "-I${source_root}/src/kernel/" ]
-
-  ccflags: [
-    "--target=x86_64-jsix-elf",
-    "-fno-stack-protector",
-
-    "-I${source_root}/external",
-
-    "-nostdinc",
-    "-nostdlib",
-    "-ffreestanding",
-    "-nodefaultlibs",
-    "-fno-builtin",
-    "-fno-plt",
-
-    "-mno-sse",
-    "-fno-omit-frame-pointer",
-    "-mno-red-zone",
-    "-mcmodel=kernel",
-    "-fvisibility=hidden",
-    "-fvisibility-inlines-hidden",
-
-    "-g3",
-    "-ggdb",
-
-    "-D__ELF__",
-    "-D__jsix__",
-    "-D__j6kernel",
-    "-U__linux",
-    "-U__linux__",
-    "-DPRINTF_ALIAS_STANDARD_FUNCTION_NAMES=1",
-    "-DPRINTF_INCLUDE_CONFIG_H=1",
-
-    "-isystem${build_root}/include/libc",
-    "-isystem${source_root}/sysroot/include",
-    "--sysroot='${source_root}/sysroot'" ]
-
-
-  cflags: [ '-nostdinc' ]
-
-  cxxflags: [
-    "-fno-exceptions",
-    "-fno-rtti",
-    "-nostdinc",
-    "-isystem${source_root}/sysroot/include/c++/v1" ]
-
-  ldflags: [
-    "-g",
-    "-m", "elf_x86_64",
-    "-nostdlib",
-    "-Bstatic",
-    "--no-eh-frame-hdr",
-    "-z", "norelro",
-    "-z", "separate-code" ]
-

assets/build/linux/config.debug.yaml

@@ -0,0 +1,9 @@
+---
+ccflags: [
+  "-g3",
+  "-ggdb",
+]
+
+ldflags: [
+  "-g",
+]

assets/build/linux/config.release.yaml

@@ -0,0 +1,3 @@
+ccflags: [
+  "-O3",
+]

assets/build/linux/global.yaml

@@ -0,0 +1,39 @@
+---
+cc: "clang-16"
+cxx: "clang++-16"
+ld: "ld.lld-16"
+ar: ar
+nasm: nasm
+objcopy: objcopy
+
+ccflags: [
+  "-I${source_root}/src/include",
+  "-I${source_root}/sysroot/include/c++/v1",
+  "-fcolor-diagnostics",
+  "-U__STDCPP_THREADS__",
+  "-D__jsix_config=${build_config}",
+  "-D__jsix_config_${build_config}",
+  "-DVERSION_MAJOR=${version_major}",
+  "-DVERSION_MINOR=${version_minor}",
+  "-DVERSION_PATCH=${version_patch}",
+  "-DVERSION_GITSHA=0x${version_sha}",
+  '-DGIT_VERSION=\"${version_major}.${version_minor}.${version_patch}+${version_sha}\"',
+  '-DGIT_VERSION_WIDE=L\"${version_major}.${version_minor}.${version_patch}+${version_sha}\"',
+
+  "-Wformat=2", "-Winit-self", "-Winline", "-Wmissing-format-attribute",
+  "-Wmissing-include-dirs", "-Wswitch", "-Wundef", "-Wdisabled-optimization",
+  "-Wpointer-arith", "-Wno-attributes", "-Wno-sign-compare", "-Wno-multichar",
+  "-Wno-div-by-zero", "-Wno-endif-labels", "-Wno-pragmas", "-Wno-format-extra-args",
+  "-Wno-unused-result", "-Wno-deprecated-declarations", "-Wno-unused-function",
+  "-Wno-address-of-packed-member", "-Wno-invalid-offsetof", "-Wno-format-nonliteral",
+  "-Werror" ]
+
+asflags: [
+  "-DVERSION_MAJOR=${version_major}",
+  "-DVERSION_MINOR=${version_minor}",
+  "-DVERSION_PATCH=${version_patch}",
+  "-DVERSION_GITSHA=0x${version_sha}",
+  "-I${source_root}/src/include" ]
+
+cflags: [ "-std=c11" ]
+cxxflags: [ "-std=c++17" ]

assets/build/linux/rules.ninja

@@ -0,0 +1,91 @@
+rule compile.c
+  command = $cc -MMD -MF $out.d $cflags $ccflags -o $out -c $in
+  description = Compiling [$target]:$name
+  depfile = $out.d
+  deps = gcc
+
+rule dump_c_defs
+  command = echo | $cc $ccflags $cflags -dM -E - > $out
+  description = Dumping C defines for $target
+
+rule dump_c_run
+  command = echo '#!/bin/bash' > $out; echo '$cc $ccflags $cflags $$*' >> $
+      $out; chmod a+x $out
+  description = Dumping C arguments for $target
+
+rule compile.cpp
+  command = $cxx -MMD -MF $out.d $cxxflags $ccflags -o $out -c $in
+  description = Compiling [$target]:$name
+  depfile = $out.d
+  deps = gcc
+
+rule dump_cpp_defs
+  command = echo | $cxx -x c++ $ccflags $cxxflags -dM -E - > $out
+  description = Dumping C++ defines for $target
+
+rule dump_cpp_run
+  command = echo '#!/bin/bash' > $out; echo '$cxx $ccflags $cxxflags $$*' $
+      >> $out; chmod a+x $out
+  description = Dumping C++ arguments for $target
+
+rule compile.s
+  command = $nasm -o $out -felf64 -MD $out.d $asflags $in
+  description = Assembling [$target]:$name
+  depfile = $out.d
+  deps = gcc
+
+rule parse.cog
+  command = cog -o $out -d -D target=$target $cogflags $in
+  description = Parsing [$target]:$name
+
+rule exe
+  command = $ld $ldflags -o $out $in $libs
+  description = Linking exe [$target]:$name
+
+rule driver
+  command = $ld $ldflags -o $out $in $libs
+  description = Linking driver [$target]:$name
+
+rule lib
+  command = $ld -shared -soname $soname $ldflags -o $out $in $libs
+  description = Linking [$target]:$name
+
+rule lib_static
+  command = $ar qcs $out $in
+  description = Archiving [$target]:$name
+
+rule cp
+  command = cp $in $out
+  description = Copying [$target]:$name
+
+rule dump
+  command = objdump -DSC -M intel $in > $out
+  description = Dumping decompiled $name
+
+rule makest
+    description = Making symbol table
+    command = nm -n -S --demangle $in | ${source_root}/scripts/build_symbol_table.py $out
+
+rule makeinitrd
+    description = Creating $name
+    command = ${source_root}/scripts/mkj6romfs.py -c $format $in $out
+
+rule makefat
+    description = Creating $name
+    command = $
+        cp $in $out; $
+        mcopy -s -D o -i $out@@1M ${build_root}/fatroot/* ::/
+
+rule strip
+    description = Stripping $name
+    command = $
+        cp $in $out; $
+        objcopy --only-keep-debug $out $debug; $
+        strip --discard-all -g $out; $
+        objcopy --add-gnu-debuglink=$debug $out
+
+rule touch
+    command = touch $out
+
+rule compdb
+    command = ninja -t compdb > $out

assets/build/linux/target.init.yaml

@@ -0,0 +1,39 @@
+---
+ccflags: [
+  "--target=x86_64-jsix-elf",
+  "-fno-omit-frame-pointer",
+  "-fno-stack-protector",
+
+  "-fvisibility=hidden",
+  "-fvisibility-inlines-hidden",
+
+  "-D__ELF__",
+  "-D__jsix__",
+  "-U__linux",
+  "-U__linux__",
+
+  "-DMSPACES",
+
+  "--sysroot='${source_root}/sysroot'"
+]
+
+
+cxxflags: [
+  "-fno-exceptions",
+  "-fno-rtti",
+]
+
+ldflags: [
+  "-Bstatic",
+  "-m", "elf_x86_64",
+  "--sysroot='${source_root}/sysroot'",
+  "--no-eh-frame-hdr",
+  "-L", "${source_root}/sysroot/lib",
+  "-z", "separate-code",
+  "-lc++", "-lc++abi", "-lunwind",
+  "--no-dependent-libraries",
+]
+
+libs: [
+  "${target_dir}/crt0.o",
+]

assets/build/linux/target.user.exe.yaml

@@ -0,0 +1,16 @@
+---
+
+ccflags: [
+  "-fpie"
+]
+
+ldflags: [
+  "-pie",
+  "-rpath", "${target_dir}",
+  "--dynamic-linker", "/lib64/ld-linux-x86-64.so.2",
+  "--push-state", "--as-needed", "-Bstatic", "-lc++", "-lc++abi", "-lunwind", "--pop-state",
+]
+
+libs: [
+  "${target_dir}/crt0.o",
+]

assets/build/linux/target.user.shared.yaml

@@ -0,0 +1,7 @@
+---
+ccflags: [
+]
+
+ldflags: [
+  "-shared",
+]

assets/build/linux/target.user.yaml

@@ -0,0 +1,33 @@
+---
+asflags: []
+
+ccflags: [
+  "--target=x86_64-jsix-elf",
+  "-fno-omit-frame-pointer",
+  "-fno-stack-protector",
+
+  "-fvisibility=hidden",
+  "-fvisibility-inlines-hidden",
+
+  "-D__ELF__",
+  "-D__jsix__",
+  "-U__linux",
+  "-U__linux__",
+
+  "--sysroot='${source_root}/sysroot'",
+  "-fpic",
+]
+
+cxxflags: [
+  "-fno-exceptions",
+  "-fno-rtti",
+  ]
+
+ldflags: [
+  "-m", "elf_x86_64",
+  "--sysroot='${source_root}/sysroot'",
+  "--no-eh-frame-hdr",
+  "-L", "${source_root}/sysroot/lib",
+  "-z", "separate-code",
+  "--no-dependent-libraries",
+]

assets/build/user-debug.yaml

@@ -1,38 +0,0 @@
----
-extends: base
-
-variables:
-  asflags: [ "-I${source_root}/src/kernel/" ]
-
-  ccflags: [
-    "--target=x86_64-jsix-elf",
-    "-fno-omit-frame-pointer",
-    "-fno-stack-protector",
-
-    "-g",
-
-    "-D__ELF__",
-    "-D__jsix__",
-    "-U__linux",
-    "-U__linux__",
-
-    "-isystem${source_root}/sysroot/include",
-    "-isystem${build_root}/include/libc",
-    "--sysroot='${source_root}/sysroot'" ]
-
-
-  cxxflags: [
-    "-fno-exceptions",
-    "-fno-rtti",
-    "-isystem${source_root}/sysroot/include/c++/v1" ]
-
-  ldflags: [
-    "-g",
-    "-Bstatic",
-    "--sysroot='${source_root}/sysroot'",
-    "-L", "${source_root}/sysroot/lib",
-    "-z", "separate-code",
-    "-lc++", "-lc++abi", "-lunwind",
-    "--no-dependent-libraries",
-  ]
-

assets/debugging/jsix.elf-gdb.py

@@ -1,8 +1,14 @@
 import gdb
 import gdb.printing
 
+import sys
+sys.path.append('./scripts')
+
+import re
+
 from collections import namedtuple
 Capability = namedtuple("Capability", ["id", "parent", "refcount", "caps", "type", "koid"])
+LogEntry = namedtuple("LogHeader", ["id", "bytes", "severity", "area", "message"])
 
 class PrintStackCommand(gdb.Command):
     def __init__(self):
@@ -168,12 +174,12 @@ class GetThreadsCommand(gdb.Command):
         rsp = int(gdb.parse_and_eval(f"{tcb}->rsp"))
         pri = int(gdb.parse_and_eval(f"{tcb}->priority"))
         flags = int(gdb.parse_and_eval(f"{thread}->m_state"))
-        koid = int(gdb.parse_and_eval(f"{thread}->m_koid"))
-        proc = int(gdb.parse_and_eval(f"{thread}->m_parent.m_koid"))
+        koid = int(gdb.parse_and_eval(f"{thread}->m_obj_id"))
+        proc = int(gdb.parse_and_eval(f"{thread}->m_parent.m_obj_id"))
 
         creator = int(gdb.parse_and_eval(f"{thread}->m_creator"))
         if creator != 0:
-            creator_koid = int(gdb.parse_and_eval(f"{thread}->m_creator->m_koid"))
+            creator_koid = int(gdb.parse_and_eval(f"{thread}->m_creator->m_obj_id"))
             creator = f"{creator_koid:x}"
         else:
             creator = "<no thread>"
@@ -233,11 +239,7 @@ class GetThreadsCommand(gdb.Command):
             self.print_cpudata(cpu)
 
             previous = int(gdb.parse_and_eval(f"{runlist}.prev"))
-            if previous != 0:
-                tcb = f"((TCB*){previous:#x})"
-                thread = f"({tcb}->thread)"
-                koid = int(gdb.parse_and_eval(f"{thread}->m_koid"))
-                print(f"            prev: {koid:x}")
+            print(f"            prev: {previous:x}")
             print()
 
             for pri in range(8):
@@ -283,6 +285,79 @@ class PrintProfilesCommand(gdb.Command):
             print(f"{name:>{max_len}}: {avg:15.3f}")
 
 
+class DumpLogCommand(gdb.Command):
+    level_names = ["", "fatal", "error", "warn", "info", "verbose", "spam"]
+
+    def __init__(self):
+        super().__init__("j6log", gdb.COMMAND_DATA)
+
+        from memory import Layout
+        layout = Layout("definitions/memory_layout.yaml")
+        for region in layout.regions:
+            if region.name == "logs":
+                self.base_addr = region.start
+                break
+
+        self.areas = []
+        area_re = re.compile(r"LOG\(\s*(\w+).*")
+        with open("src/libraries/j6/include/j6/tables/log_areas.inc", 'r') as areas_inc:
+            for line in areas_inc:
+                m = area_re.match(line)
+                if m:
+                    self.areas.append(m.group(1))
+
+    def get_entry(self, addr):
+        addr = int(addr)
+        size = int(gdb.parse_and_eval(f"((j6_log_entry*){addr:#x})->bytes"))
+        mlen = size - 8
+
+        return LogEntry(
+                int(gdb.parse_and_eval(f"((j6_log_entry*){addr:#x})->id")),
+                size,
+                int(gdb.parse_and_eval(f"((j6_log_entry*){addr:#x})->severity")),
+                int(gdb.parse_and_eval(f"((j6_log_entry*){addr:#x})->area")),
+                gdb.parse_and_eval(f"((j6_log_entry*){addr:#x})->message").string(length=mlen))
+
+    def invoke(self, arg, from_tty):
+        start = gdb.parse_and_eval("g_logger.m_start & (g_logger.m_buffer.count - 1)")
+        end = gdb.parse_and_eval("g_logger.m_end & (g_logger.m_buffer.count - 1)")
+        if end < start:
+            end += gdb.parse_and_eval("g_logger.m_buffer.count")
+
+        print(f"Logs are {start} -> {end}")
+
+        addr = self.base_addr + start
+        end += self.base_addr
+        while addr < end:
+            entry = self.get_entry(addr)
+            if entry.bytes < 8:
+                print(f"Bad log header size: {entry.bytes}")
+                break
+            addr += entry.bytes
+            area = "??"
+            if entry.area < len(self.areas):
+                area = self.areas[entry.area]
+            level = self.level_names[entry.severity]
+            print(f"{area:>7}:{level:7}  {entry.message}")
+
+
+class ShowCurrentProcessCommand(gdb.Command):
+    def __init__(self):
+        super().__init__("j6current", gdb.COMMAND_DATA)
+
+    def invoke(self, arg, from_tty):
+        def get_obj_and_id(name):
+            obj = int(gdb.parse_and_eval(f"((cpu_data*)$gs_base)->{name}"))
+            oid = -1
+            if obj != 0:
+                oid = int(gdb.parse_and_eval(f"((obj::kobject*){obj:#x})->m_obj_id"))
+            return obj, oid
+
+        process, pid = get_obj_and_id("process")
+        thread, tid = get_obj_and_id("thread")
+        print(f"{pid:02x}/{tid:02x}  [ {process:x} / {thread:x} ]")
+
+
 class CapTablePrinter:
     def __init__(self, val):
         node_map = val["m_caps"]
@@ -307,7 +382,7 @@ class CapTablePrinter:
                     refcount = int(node["holders"]),
                     caps = int(node["caps"]),
                     type = str(node["type"])[14:],
-                    koid = node['object']['m_koid']))
+                    koid = node['object']['m_obj_id']))
 
             self.nodes.sort(key=lambda n: n.id, reverse=True)
 
@@ -399,11 +474,51 @@ class HandleSetPrinter:
         return self._iterator(self.node_map['m_nodes'], self.capacity)
 
 
+class LinkedListPrinter:
+    def __init__(self, llist):
+        self.name = llist.type.tag
+        self.head = llist['m_head']
+        self.tail = llist['m_tail']
+
+        self.items = []
+        current = self.head
+        while current:
+            item = current.dereference()
+            self.items.append((str(len(self.items)), item))
+            current = item['m_next']
+
+    class _iterator:
+        def __iter__(self):
+            return self
+
+        def __next__(self):
+            raise StopIteration
+
+    def to_string(self):
+        return f"{self.name}[{len(self.items)}]"
+
+    def children(self):
+        return self.items
+
+
+class IsRunning(gdb.Function):
+    def __init__(self):
+        super(IsRunning, self).__init__("is_running")
+
+    def invoke(self):
+        inferior = gdb.selected_inferior()
+        return \
+            inferior is not None and \
+            inferior.is_valid() and \
+            len(inferior.threads()) > 0
+
+
 def build_pretty_printers():
     pp = gdb.printing.RegexpCollectionPrettyPrinter("jsix")
     pp.add_printer("cap table", '^cap_table$', CapTablePrinter)
     pp.add_printer("handle set", '^util::node_set<unsigned long, 0, heap_allocated>$', HandleSetPrinter)
     pp.add_printer("vector", '^util::vector<.*>$', VectorPrinter)
+    pp.add_printer("linked list", '^util::linked_list<.*>$', LinkedListPrinter)
     return pp
 
 gdb.printing.register_pretty_printer(
@@ -415,8 +530,12 @@ PrintBacktraceCommand()
 TableWalkCommand()
 GetThreadsCommand()
 PrintProfilesCommand()
+DumpLogCommand()
+ShowCurrentProcessCommand()
+IsRunning()
 
 gdb.execute("display/i $rip")
+gdb.execute("define hook-quit\nif $is_running()\n  kill\nend\nend")
 if not gdb.selected_inferior().was_attached:
     gdb.execute("add-symbol-file build/panic.serial.elf")
     gdb.execute("target remote :1234")

assets/grammars/definitions.g

@@ -35,7 +35,7 @@ name: IDENTIFIER
 options: "[" ( OPTION | IDENTIFIER )+ "]"
 description: COMMENT+
 
-PRIMITIVE: INT_TYPE | "size" | "string" | "buffer" | "address"
+PRIMITIVE: INT_TYPE "*"? | "size" | "string" | "buffer" | "address"
 INT_TYPE: /u?int(8|16|32|64)?/
 NUMBER: /(0x)?[0-9a-fA-F]+/
 UID: /[0-9a-fA-F]{16}/

assets/manifests/default.yaml

@@ -8,5 +8,9 @@ panic:
   - panic.serial
 services:
   - srv.logger
+  - testapp
 drivers:
   - drv.uart
+  - drv.uefi_fb
+libs:
+  - ld.so

assets/manifests/minimal.yaml

@@ -0,0 +1,14 @@
+---
+location: jsix
+init: srv.init
+initrd:
+  name: initrd.dat
+  format: zstd
+panic:
+  - panic.serial
+services:
+  - srv.logger
+drivers:
+  - drv.uart
+libs:
+  - ld.so

assets/manifests/shell.yaml

@@ -0,0 +1,14 @@
+---
+location: jsix
+init: srv.init
+initrd:
+  name: initrd.dat
+  format: zstd
+panic:
+  - panic.serial
+services:
+  - 6s
+drivers:
+  - drv.uart
+libs:
+  - ld.so

assets/manifests/test.yaml

@@ -9,5 +9,3 @@ panic:
   - panic.serial
 services:
   - test_runner
-drivers:
-  - drv.uart

configure

@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
 
-def generate(output, config, manifest):
+def generate(output, config, arch, manifest):
     from os import makedirs
     from glob import iglob
     from pathlib import Path
-    from bonnibel.module import Module
+    from bonnibel.module import Module, ModuleList
     from bonnibel.project import Project
 
     root = Path(__file__).parent.resolve()
@@ -18,16 +18,17 @@ def generate(output, config, manifest):
         str(root / "external/*.module"),
     ]
 
-    modules = {}
+    modules = ModuleList(arch)
     for source in sources:
         for modfile in iglob(source, recursive=True):
-            path = Path(modfile).parent
+            modfile = Path(modfile)
+            path = modfile.parent
 
             def module_init(name, **kwargs):
                 if not "root" in kwargs:
                     kwargs["root"] = path
                 m = Module(name, modfile, **kwargs)
-                modules[m.name] = m
+                modules.add(m)
                 return m
 
             glo = {
@@ -35,18 +36,17 @@ def generate(output, config, manifest):
                 "source_root": root,
                 "build_root": output,
                 "module_root": path,
+                "config": config,
+                "arch": arch,
             }
             code = compile(open(modfile, 'r').read(), modfile, "exec")
 
             loc = {}
             exec(code, glo, loc)
 
-    Module.update(modules)
-
     makedirs(output.resolve(), exist_ok=True)
-    project.generate(root, output, modules, config, manifest)
-    for mod in modules.values():
-        mod.generate(output)
+    project.generate(root, output, modules, config, arch, manifest)
+    modules.generate(output)
 
 if __name__ == "__main__":
     import sys
@@ -56,18 +56,25 @@ if __name__ == "__main__":
     from argparse import ArgumentParser
     from bonnibel import BonnibelError
 
+    default_arch = "amd64"
+
     p = ArgumentParser(description="Generate jsix build files")
     p.add_argument("--manifest", "-m", metavar="FILE", default="assets/manifests/default.yaml",
             help="File to use as the system manifest")
-    p.add_argument("--config", "-c", metavar="NAME", default="debug",
+    p.add_argument("--conf", "-c", metavar="NAME", default="debug",
             help="Configuration to build (eg, 'debug' or 'release')")
-    p.add_argument("output", metavar="DIR", default="build", nargs='?',
+    p.add_argument("--arch", "-a", metavar="NAME", default=default_arch,
+            help="Architecture to build (eg, 'amd64' or 'linux')")
+    p.add_argument("--verbose", "-v", action='count', default=0,
+            help="More verbose log output")
+    p.add_argument("output", metavar="DIR", default=None, nargs='?',
             help="Where to create the build root")
 
     args = p.parse_args()
 
+    output = args.output or f"build.{args.arch}"
     try:
-        generate(args.output, args.config, args.manifest)
+        generate(output, args.conf, args.arch, args.manifest)
 
     except BonnibelError as be:
         import sys

definitions/memory_layout.yaml

@@ -1,26 +1,38 @@
 ---
 - name: linear
+  desc: Linearly-mapped physical memory
   size: 64T
   shared: true
 
 - name: bitmap
+  desc: Used/free page tracking bitmap
   size: 1T
   shared: true
 
 - name: heapmap
+  desc: Kernel heap accounting structures
   size: 32G
 
 - name: heap
+  desc: Kernel heap
   size: 32G
 
 - name: capsmap
+  desc: Capabilities accounting structures
   size: 32G
 
 - name: caps
+  desc: Capabilities
   size: 32G
 
 - name: stacks
+  desc: Kernel thread stacks
   size: 64G
 
 - name: buffers
+  desc: Kernel buffers
   size: 64G
+
+- name: logs
+  desc: Kernel logs circular buffer
+  size: 2G

definitions/objects/channel.def

@@ -1,21 +0,0 @@
-object channel : object {
-    uid 3ea38b96aa0e54c8
-
-    capabilities [
-        send
-        receive
-        close
-    ]
-
-    method create [constructor] 
-    method close [destructor cap:close]
-
-    method send [cap:send] {
-        param data buffer [inout]
-    }
-
-    method receive [cap:receive] {
-        param data buffer [out]
-        param flags uint64
-    }
-}

definitions/objects/event.def

@@ -1,3 +1,6 @@
+# An ``event`` is a simple synchronization object. It contains up to 64 signals
+# that threads can wait for and signal in parallel.
+
 object event : object {
     uid f441e03da5516b1a
 

definitions/objects/mailbox.def

@@ -1,5 +1,6 @@
-# Mailboxes are objects that enable synchronous IPC via short message-passing
-# of tagged handles.
+# Mailboxes are objects that enable synchronous IPC via arbitrary
+# message-passing of tagged data and/or handles. Not as efficient
+# as shared memory channels, but more flexible.
 
 object mailbox : object {
     uid 99934ad04ece1e07
@@ -13,13 +14,15 @@ object mailbox : object {
     method create [constructor]
     method close [destructor cap:close]
 
-    # Send a message to the reciever, and block until a
-    # response is sent. Note that getting this response
-    # does not require the receive capability.
+    # Send a message to the reciever, and block until a response is
+    # sent. Note that getting this response does not require the
+    # receive capability.
     method call [cap:send] {
         param tag uint64 [inout]
-        param subtag uint64 [inout]
-        param give_handle ref object [optional inout handle]
+        param data buffer [optional inout]
+        param data_size size  # number of total bytes in data buffer
+        param handles ref object [optional inout handle list]
+        param handles_size size  # total size of handles buffer
     }
 
     # Respond to a message sent using call, and wait for another
@@ -28,8 +31,10 @@ object mailbox : object {
     # to waiting for a new message.
     method respond [cap:receive] {
         param tag uint64 [inout]
-        param subtag uint64 [inout]
-        param give_handle ref object [optional inout handle]
+        param data buffer [optional inout]
+        param data_size size  # number of total bytes in data buffer
+        param handles ref object [optional inout handle list]
+        param handles_size size  # total size of handles buffer
         param reply_tag uint64 [inout]
         param flags uint64
     }

definitions/objects/object.def

@@ -1,4 +1,6 @@
-# The base type of all kernel-exposed objects
+# All kernel-exposed objects inherit from the base ``object`` type, so the
+# ``object`` syscalls can be used with any object's handle.
+
 object object [virtual] {
     uid 667f61fb2cd57bb4
     cname kobject

definitions/objects/process.def

@@ -1,7 +1,7 @@
 import "objects/object.def"
 
-# Processes are a collection of handles and a virtual memory
-# space inside which threads are run.
+# A ``process`` object represents a process running on the system, and allows
+# control over the threads, handles, and virtual memory space of that process.
 
 object process : object {
     uid 0c69ee0b7502ba31
@@ -12,14 +12,16 @@ object process : object {
     ]
 
     # Create a new empty process
-    method create [constructor]
+    method create [constructor] {
+        param name string
+    }
 
     # Stop all threads and exit the given process
     method kill [destructor cap:kill]
 
     # Stop all threads and exit the current process
     method exit [static noreturn] {
-        param result int32    # The result to retrun to the parent process
+        param result int64    # The result to retrun to the parent process
     }
 
     # Give the given process a handle that points to the same

definitions/objects/system.def

@@ -1,5 +1,6 @@
-# The system object represents a handle to kernel functionality
-# needed by drivers and other priviledged services
+# The singular ``system`` object represents a handle to kernel functionality
+# needed by drivers and other priviledged services.
+
 object system : object {
     uid fa72506a2cf71a30
 
@@ -10,8 +11,9 @@ object system : object {
         change_iopl
     ]
 
-    # Get a log line from the kernel log
+    # Get the next log line from the kernel log
     method get_log [cap:get_log] {
+        param seen uint64                 # Last seen log id
         param buffer buffer [out zero_ok] # Buffer for the log message data structure
     }
 

definitions/objects/thread.def

@@ -1,3 +1,7 @@
+# A ``thread`` object represents a thread of execution within a process running
+# on the system. The actual thread does not need to be currently running to
+# hold a handle to it.
+
 object thread : object {
     uid 11f23e593d5761bd
 
@@ -7,9 +11,11 @@ object thread : object {
     ]
 
     method create [constructor] {
-        param process ref process [cap:create_thread]
+        param process ref process [optional cap:create_thread]
         param stack_top address
         param entrypoint address
+        param arg0 uint64
+        param arg1 uint64
     }
 
     method kill [destructor cap:kill]

definitions/objects/vma.def

@@ -1,5 +1,9 @@
 import "objects/process.def"
 
+# A ``vma`` object represents a single virtual memory area, which may be shared
+# between several processes. A process having a handle to a ``vma`` does not
+# necessarily mean that it is mapped into that process' virtual memory space.
+
 object vma : object {
     uid d6a12b63b3ed3937
     cname vm_area
@@ -17,20 +21,21 @@ object vma : object {
 
     method create_map [constructor cap:map] {
         param size size
-        param address address
+        param address address [inout]
         param flags uint32
     }
 
     method map [cap:map] {
-        param process ref process
-        param address address
+        param process ref process [optional]
+        param address address [inout]
+        param flags uint32
     }
 
     method unmap [cap:unmap] {
-        param process ref process
+        param process ref process [optional]
     }
 
     method resize [cap:resize] {
-        param size size [inout]
+        param size size [inout]  # New size for the VMA, or 0 to query the current size without changing
     }
 }

definitions/syscalls.def

@@ -1,6 +1,5 @@
 import "objects/object.def"
 
-import "objects/channel.def"
 import "objects/event.def"
 import "objects/mailbox.def"
 import "objects/process.def"
@@ -12,12 +11,12 @@ interface syscalls [syscall] {
     uid 01d9b6a948961097
 
     expose ref object
-    expose ref system
+
     expose ref event
-    expose ref process
-    expose ref thread
     expose ref mailbox
-    expose ref channel
+    expose ref process
+    expose ref system
+    expose ref thread
     expose ref vma
 
     # Simple no-op syscall for testing
@@ -25,6 +24,8 @@ interface syscalls [syscall] {
 
     # Write a message to the kernel log
     function log {
+        param area uint8
+        param severity uint8
         param message string
     }
 
@@ -43,6 +44,24 @@ interface syscalls [syscall] {
         param mask uint32                        # The capability bitmask
     }
 
+    # Close the handle to an object
+    function handle_close {
+        param hnd ref object [handle] # The handle to close
+    }
+
+    # Block waiting on a futex
+    function futex_wait [static] {
+        param address uint32*  # Address of the futex value
+        param current uint32   # Current value of the futex
+        param timeout uint64   # Wait timeout in nanoseconds
+    }
+
+    # Wake threads waiting on a futex
+    function futex_wake [static] {
+        param address uint32*  # Address of the futex value
+        param count uint64     # Number of threads to wake, or 0 for all
+    }
+
     # Testing mode only: Have the kernel finish and exit QEMU with the given exit code
     function test_finish [test] {
         param exit_code uint32

docs/Makefile

@@ -0,0 +1,23 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+ROOTDIR       = $(SOURCEDIR)/..
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	PYTHONPATH=$(ROOTDIR)/scripts cog -r -D definitions_path=$(ROOTDIR)/definitions -c syscall_interface.rst
+	PYTHONPATH=$(ROOTDIR)/scripts cog -r -D definitions_path=$(ROOTDIR)/definitions -c kernel_memory.rst
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/_static/custom.css

@@ -0,0 +1,254 @@
+/* custom.css - jsix version */
+
+:root {
+    --background-color: #181820;
+    --link-color: #7070e0;
+    --link-hover-color: #9090ff;
+    --text-color: #3d3d3d;
+    --text-literal-color: #d26a98;
+}
+.wy-nav-side {
+  background: var(--background-color);
+}
+
+@media screen and (min-width: 1100px) {
+  .wy-nav-content-wrap {
+      background: var(--background-color);
+  }
+}
+
+a {
+  color: var(--link-color);
+}
+
+a:hover {
+  color: var(--link-hover-color);
+}
+
+a:visited {
+  color: var(--link-color);
+}
+
+.rst-content {
+  color: var(--text-color);
+}
+
+.rst-content code.literal {
+  color: var(--text-literal-color);
+}
+
+.rst-content tt.literal {
+  color: var(--text-literal-color);
+}
+
+.rst-content .note {
+  color: #003274;
+  background: #ccddf3;
+  padding: 1rem;
+  margin-bottom: 1rem;
+}
+
+.rst-content .note .admonition-title {
+  display: none;
+}
+
+.rst-content .warning {
+  color: #605000;
+  background: #fcf4cc;
+  padding: 1rem;
+  margin-bottom: 1rem;
+}
+
+.rst-content .warning .admonition-title {
+  display: none;
+}
+
+.rst-content .highlight {
+  background: #f5f5f5;
+}
+
+.wy-side-scroll {
+  background-color: var(--background-color);
+}
+
+.wy-side-nav-search {
+  background-color: var(--background-color);
+}
+
+.wy-side-nav-search input[type="text"] {
+  width: 100%;
+  border-radius: 0px;
+  padding: 6px 12px;
+  border-color: var(--background-color);
+}
+
+.wy-menu-vertical a {
+  font-size: 100%;
+  color: #d9d9d9;
+  padding-top: 0.6rem;
+  padding-bottom: 0.6rem;
+  background-color: inherit;
+}
+
+.wy-menu-vertical a:hover {
+  background-color: unset;
+  opacity: 1;
+}
+
+.wy-menu-vertical li.current > a {
+  background-color: var(--background-color);
+  color: white;
+}
+
+.wy-menu-vertical li.current > a span.toctree-expand {
+  display: block;
+  font-size: inherit;
+  line-height: inherit;
+  color: inherit;
+}
+
+.wy-menu-vertical li.current > a span.toctree-expand:before {
+  display: block;
+  font-size: inherit;
+  line-height: inherit;
+  color: inherit;
+}
+
+.wy-menu-vertical li.current > a span.toctree-expand:hover {
+  color: white;
+}
+
+.wy-menu-vertical li.current > a:hover {
+  background-color: var(--background-color);
+  color: white;
+}
+
+.wy-menu-vertical li.current > a:hover span.toctree-expand {
+  color: white;
+}
+
+.wy-menu-vertical .toctree-l1 {
+  opacity: 0.5;
+}
+
+.wy-menu-vertical .toctree-l1:hover {
+  opacity: 1;
+  background-color: inherit;
+}
+
+.wy-menu-vertical li.toctree-l1.current {
+  opacity: 1;
+  background-color: inherit;
+}
+
+.wy-menu-vertical li.toctree-l1.current > a {
+  border: 0px;
+}
+
+.wy-menu-vertical .toctree-l2:hover {
+  background-color: #566673;
+}
+
+.wy-menu-vertical li.toctree-l2.current > a {
+  background-color: #566673;
+  color: white;
+}
+
+.wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a {
+  background-color: #e4e7ea;
+  color: #838383;
+}
+
+.wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a:hover {
+  color: var(--text-color);
+}
+
+.wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a:hover span.toctree-expand {
+  color: var(--text-color);
+}
+
+.wy-menu-vertical li.toctree-l2.current li.toctree-l3.current > a {
+  color: var(--text-color);
+}
+
+.wy-menu-vertical li.toctree-l2 a {
+  border: 0px;
+  background-color: #566673;
+  color: #d9d9d9;
+}
+
+.wy-menu-vertical li.toctree-l2 a span.toctree-expand {
+  display: block;
+  font-size: inherit;
+  line-height: inherit;
+  color: inherit;
+}
+
+.wy-menu-vertical li.toctree-l2 a span.toctree-expand:before {
+  display: block;
+  font-size: inherit;
+  line-height: inherit;
+  color: inherit;
+}
+
+.wy-menu-vertical li.toctree-l2 a span.toctree-expand:hover {
+  color: white;
+}
+
+.wy-menu-vertical li.toctree-l2 a:hover {
+  color: white;
+  background-color: #566673;
+}
+
+.wy-menu-vertical li.toctree-l2 a:hover span.toctree-expand {
+  color: white;
+}
+
+.wy-menu-vertical li.toctree-l3.current > a {
+  background-color: #e4e7ea;
+  color: #838383;
+}
+
+.wy-menu-vertical li.toctree-l3.current li.toctree-l4 > a {
+  background-color: #e4e7ea;
+  color: #838383;
+}
+
+.wy-menu-vertical li.toctree-l3.current li.toctree-l4.current > a {
+  color: var(--text-color);
+}
+
+.wy-nav-top {
+  background-color: var(--background-color);
+}
+
+.btn {
+  display: inline-block;
+  font-weight: 400;
+  line-height: 1.5;
+  color: var(--text-color);
+  text-align: center;
+  text-decoration: none;
+  vertical-align: middle;
+  cursor: pointer;
+  -webkit-user-select: none;
+  -ms-user-select: none;
+  user-select: none;
+  background-color: transparent;
+  border: 1px solid transparent;
+  padding: 0.375rem 0.75rem;
+  font-size: 1rem;
+  border-radius: 0;
+  transition: color 0.15s ease-in-out, background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;
+  box-shadow: unset;
+}
+
+.btn-neutral {
+  background: unset !important;
+  color: #838383 !important;
+}
+
+.btn-neutral:active {
+  padding: 0.375rem 0.75rem;
+  box-shadow: unset;
+}

docs/conf.py

@@ -0,0 +1,50 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = 'jsix'
+copyright = '2024, Justin C. Miller'
+author = 'Justin C. Miller'
+release = '0.8'
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = ['sphinx.ext.todo']
+primary_domain = 'cpp'
+todo_include_todos = True
+
+templates_path = ['_templates']
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = 'renku'
+html_title = 'jsix'
+html_logo = 'jsix_transparent.svg'
+html_static_path = ['_static']
+html_css_files = ['custom.css']
+html_theme_options = {
+    "description": "The jsix description",
+    "github_repo": "https://github.com/justinian/jsix",
+    "logo_only": True,
+    "footer_icons": [
+        {
+            "name": "GitHub",
+            "url": "https://github.com/justinian/jsix",
+            "html": """
+                <svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 16 16">
+                    <path fill-rule="evenodd" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path>
+                </svg>
+            """,
+            "class": "",
+        },
+    ],
+}

docs/index.rst

@@ -0,0 +1,81 @@
+.. jsix documentation master file
+.. |amd64| replace:: :abbr:`amd64 (aka x86_64)`
+
+The jsix Operating System
+=========================
+Introduction
+------------
+
+**jsix** is a custom multi-core x64 operating system being built from scratch,
+supporting modern [#]_ Intel or AMD CPUs, and UEFI firmware. It was initially
+created out of a desire to explore UEFI and to explore what's possible with a
+microkernel architecture on modern 64-bit architectures.
+
+Most of jsix is written in C++ (C++17, using `LLVM <https://llvm.org>`_), but
+you'll also find some assembly (in `NASM <https://nasm.us>`_ syntax) and Python
+for development tooling.
+
+jsix can be found `on GitHub <https://github.com/justinian/jsix>`_, and is
+released under the terms of the `MPL 2.0 <https://mozilla.org/MPL/2.0/>`_.
+
+.. admonition:: A note on the name
+
+    This kernel was originally named Popcorn, but I have since discovered that
+    the Popcorn Linux project is also developing a kernel with that name,
+    started around the same time as this project. So I've renamed this kernel
+    jsix as an homage to L4, xv6, and my wonderful wife.
+
+    The name jsix is always styled *jsix* or ``j6``, never capitalized.
+
+.. [#] jsix aims to support amd64 (x86_64) CPUs released in the last 10 years.
+
+Current Features
+----------------
+
+The jsix kernel is quite far along now, but the userland systems are still lacking.
+
+- Platform: |amd64|
+- UEFI bootloader
+- Multi-core & multi-tasking microkernel
+
+  - Work-stealing SMP scheduler
+  - Pluggable panic handler modules
+
+- Capability-style object-oriented syscall API
+
+  - Custom IDL for specifying and documenting syscalls
+
+- Virtual memory based on sharable Virtual Memory Area objects (VMAs)
+- Kernel API library (libj6), also provides features built on kernel primitives:
+
+  - Channels (async stream IPC) built on shared memory and futexes
+  - Ring buffers via doubly-mapped pages
+
+- Custom libc
+- Runtime dynamic linker
+- Init service
+
+  - Built-in VFS service for the initrd
+  - ELF loader
+  - Service-lookup protocol service
+
+- Userland UART driver
+- Userland UEFI framebuffer driver
+- Userland kernel log output service
+- Userland unit test runner
+- Build configuration system (bonnibel)
+
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Site Contents:
+
+   syscall_interface
+   kernel_memory
+   process_initialization
+
+
+* :ref:`genindex`
+* :ref:`search`
+
+

docs/jsix_transparent.svg

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Generator: Gravit.io -->
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" style="isolation:isolate" viewBox="176.562 356.069 211.11 113" width="211.11pt" height="113pt">
+<g>
+    <g>
+        <g>
+            <path d=" M 212.981 372.36 L 219.564 376.16 L 226.147 379.961 L 226.147 387.563 L 226.147 395.164 L 219.564 398.965 L 212.981 402.766 L 206.398 398.965 L 199.815 395.164 L 199.815 387.563 L 199.815 379.961 L 206.398 376.16 L 212.981 372.36 L 212.981 372.36 L 212.981 372.36 Z  M 256.292 397.366 L 262.875 401.166 L 269.458 404.967 L 269.458 412.569 L 269.458 420.17 L 262.875 423.971 L 256.292 427.772 L 249.709 423.971 L 243.126 420.17 L 243.126 412.569 L 243.126 404.967 L 249.709 401.166 L 256.292 397.366 L 256.292 397.366 Z  M 183.622 387.283 L 205.52 374.64 L 227.418 361.997 L 249.316 374.64 L 271.214 387.283 L 271.214 412.569 L 271.214 437.854 L 249.316 450.497 L 227.418 463.14 L 205.52 450.497 L 183.622 437.854 L 183.622 412.569 L 183.622 387.283 L 183.622 387.283 L 183.622 387.283 Z  M 241.855 372.36 L 248.438 376.16 L 255.021 379.961 L 255.021 387.563 L 255.021 395.164 L 248.438 398.965 L 241.855 402.766 L 235.272 398.965 L 228.689 395.164 L 228.689 387.563 L 228.689 379.961 L 235.272 376.16 L 241.855 372.36 Z " fill-rule="evenodd" fill="rgb(49,79,128)"/>
+            <path d=" M 298.642 379.579 L 291.621 379.579 L 291.621 372.558 L 298.642 372.558 L 298.642 379.579 Z  M 285.214 446.718 L 285.214 441.452 L 287.32 441.452 L 287.32 441.452 Q 289.339 441.452 290.524 440.092 L 290.524 440.092 L 290.524 440.092 Q 291.708 438.731 291.708 436.625 L 291.708 436.625 L 291.708 387.039 L 298.729 387.039 L 298.729 436.011 L 298.729 436.011 Q 298.729 440.925 295.921 443.822 L 295.921 443.822 L 295.921 443.822 Q 293.113 446.718 288.286 446.718 L 288.286 446.718 L 285.214 446.718 Z  M 306.628 432.676 L 306.628 427.41 L 314.088 427.41 L 314.088 427.41 Q 317.862 427.41 319.573 425.347 L 319.573 425.347 L 319.573 425.347 Q 321.285 423.285 321.285 419.95 L 321.285 419.95 L 321.285 419.95 Q 321.285 417.317 319.705 415.474 L 319.705 415.474 L 319.705 415.474 Q 318.125 413.631 314.966 411.174 L 314.966 411.174 L 314.966 411.174 Q 312.245 408.98 310.621 407.356 L 310.621 407.356 L 310.621 407.356 Q 308.998 405.732 307.813 403.319 L 307.813 403.319 L 307.813 403.319 Q 306.628 400.905 306.628 397.746 L 306.628 397.746 L 306.628 397.746 Q 306.628 393.095 309.744 390.067 L 309.744 390.067 L 309.744 390.067 Q 312.859 387.039 318.125 387.039 L 318.125 387.039 L 325.76 387.039 L 325.76 392.305 L 319.441 392.305 L 319.441 392.305 Q 313.21 392.305 313.21 398.185 L 313.21 398.185 L 313.21 398.185 Q 313.21 400.467 314.615 402.134 L 314.615 402.134 L 314.615 402.134 Q 316.019 403.802 319.003 406.083 L 319.003 406.083 L 319.003 406.083 Q 321.723 408.19 323.479 409.901 L 323.479 409.901 L 323.479 409.901 Q 325.234 411.613 326.463 414.202 L 326.463 414.202 L 326.463 414.202 Q 327.691 416.791 327.691 420.301 L 327.691 420.301 L 327.691 420.301 Q 327.691 426.532 324.4 429.604 L 324.4 429.604 L 324.4 429.604 Q 321.109 432.676 315.141 432.676 L 315.141 432.676 L 306.628 432.676 Z  M 342.611 379.579 L 335.59 379.579 L 335.59 372.558 L 342.611 372.558 L 342.611 379.579 Z  M 342.611 432.676 L 335.59 432.676 L 335.59 387.039 L 342.611 387.039 L 342.611 432.676 Z  M 356.126 432.676 L 348.754 432.676 L 361.392 409.77 L 349.632 387.039 L 356.39 387.039 L 364.639 403.187 L 372.977 387.039 L 379.735 387.039 L 367.974 409.77 L 380.612 432.676 L 373.24 432.676 L 364.639 416.001 L 356.126 432.676 Z " fill="rgb(49,79,128)"/>
+        </g>
+    </g>
+</g>
+</svg>

docs/kernel_memory.rst

@@ -0,0 +1,178 @@
+.. jsix syscall interface.
+.. Automatically updated from the definition files using cog!
+
+.. [[[cog code generation
+.. from os.path import join
+.. from memory import Layout, unit
+..
+.. layout = Layout(join(definitions_path, "memory_layout.yaml"))
+.. l = max([len(r.name) for r in layout.regions])
+.. ]]]
+.. [[[end]]] (checksum: d41d8cd98f00b204e9800998ecf8427e)
+
+Kernel memory
+=============
+
+While jsix probably should eventually use KASLR to randomize its memory layout,
+currently the layout is mostly fixed. (Kernel code locations are not consistent
+but aren't explicitly randomized.)
+
+.. [[[cog code generation
+.. line_size = 128 * 1024**3  # Each line represents up to 32 GiB
+.. max_lines = 32
+.. totals = sum([r.size for r in layout.regions])
+.. remain = unit((128 * 1024**4) - totals)
+..
+.. def split(val):
+..    return f"0x {val >> 48:04x} {(val >> 32) & 0xffff:04x} {(val >> 16) & 0xffff:04x} {val & 0xffff:04x}"
+..
+.. cog.outl()
+.. cog.outl(f"+-+-----------------------------+----------+---------------------------------------+")
+.. cog.outl(f"| | Address                     | Size     | Use                                   |")
+.. cog.outl(f"+=+=============================+==========+=======================================+")
+..
+.. for region in layout.regions:
+..     cog.outl(f"| | ``{split(region.start)}``  | {unit(region.size):>8} | {region.desc:37} |")
+..     lines = min(max_lines, region.size // line_size)
+..     for i in range(1, lines):
+..         cog.outl(f"+-+                             |          |                                       |")
+..         cog.outl(f"| |                             |          |                                       |")
+..     cog.outl(f"+-+-----------------------------+----------+---------------------------------------+")
+..
+.. cog.outl(f"| |  ...                        |          |                                       |")
+.. cog.outl(f"+-+-----------------------------+----------+---------------------------------------+")
+.. cog.outl(f"| |  ``0x ffff 0000 0000 0000`` |          | Kernel code / headers                 |")
+.. cog.outl(f"+-+-----------------------------+----------+---------------------------------------+")
+.. cog.outl("")
+.. cog.outl("")
+.. cog.outl(f"Un-reserved virtual memory address space in the higher half: {remain}")
+.. cog.outl("")
+..
+.. ]]]
+
++-+-----------------------------+----------+---------------------------------------+
+| | Address                     | Size     | Use                                   |
++=+=============================+==========+=======================================+
+| | ``0x ffff c000 0000 0000``  |   64 TiB | Linearly-mapped physical memory       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+-----------------------------+----------+---------------------------------------+
+| | ``0x ffff bf00 0000 0000``  |    1 TiB | Used/free page tracking bitmap        |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+-----------------------------+----------+---------------------------------------+
+| | ``0x ffff be00 0000 0000``  |    1 TiB | Per-page state tracking structures    |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+                             |          |                                       |
+| |                             |          |                                       |
++-+-----------------------------+----------+---------------------------------------+
+| | ``0x ffff bdf8 0000 0000``  |   32 GiB | Kernel heap accounting structures     |
++-+-----------------------------+----------+---------------------------------------+
+| | ``0x ffff bdf0 0000 0000``  |   32 GiB | Kernel heap                           |
++-+-----------------------------+----------+---------------------------------------+
+| | ``0x ffff bde8 0000 0000``  |   32 GiB | Capabilities accounting structures    |
++-+-----------------------------+----------+---------------------------------------+
+| | ``0x ffff bde0 0000 0000``  |   32 GiB | Capabilities                          |
++-+-----------------------------+----------+---------------------------------------+
+| | ``0x ffff bdd0 0000 0000``  |   64 GiB | Kernel thread stacks                  |
++-+-----------------------------+----------+---------------------------------------+
+| | ``0x ffff bdc0 0000 0000``  |   64 GiB | Kernel buffers                        |
++-+-----------------------------+----------+---------------------------------------+
+| | ``0x ffff bdbf 8000 0000``  |    2 GiB | Kernel logs circular buffer           |
++-+-----------------------------+----------+---------------------------------------+
+| |  ...                        |          |                                       |
++-+-----------------------------+----------+---------------------------------------+
+| |  ``0x ffff 0000 0000 0000`` |          | Kernel code / headers                 |
++-+-----------------------------+----------+---------------------------------------+
+
+
+Un-reserved virtual memory address space in the higher half: 61 TiB
+
+.. [[[end]]] (checksum: 8c336cc8151beba1a79c8d3b653f1109)
+
+* :ref:`genindex`
+* :ref:`search`
+

docs/modd.conf

@@ -0,0 +1,7 @@
+** !_build/** ../definitions/**.def {
+    prep: rm -rf _build; make html
+}
+
+_build/html/** {
+    daemon: devd -m _build/html
+}

docs/process_initialization.rst

@@ -0,0 +1,42 @@
+.. jsix process initialization in userspace
+
+Process Initialization
+======================
+
+jsix follows the `System V ABI`_ on the ``amd64`` architecture. All arguments
+needed for program initialization are passed to the program's initial thread on
+the stack.
+
+Note that jsix adds a number of additional auxiliary vector entry types for
+passing jsix-specific data to a program. The jsix-specific auxiliary vector type
+codes (what the ABI document refers to as ``a_type``) start from ``0xf000``. See
+the header file ``<j6/init.h>`` for more detail.
+
+.. _System V ABI: https://gitlab.com/x86-psABIs/x86-64-ABI
+
+The initial stack frame
+-----------------------
+
+==============  ====================  ============  =======
+Address         Value                 Bytes         Notes
+==============  ====================  ============  =======
+``top``                                             Stack top (out of stack bounds)
+``top`` - 16    0                     16            Stack sentinel
+\               ``envp`` string data  ?
+\               ``argv`` string data  ?
+\               ...                   ?             Possible padding
+\               0, 0 (``AT_NULL``)    16            Aux vector sentinel
+\               Aux vectors           16 * `m`      ``AT_NULL``-terminated array of Aux vectors
+\               0                     8             Environment sentinel
+\                ``envp``             8 * `n`       0-terminated array of environment
+                                                    string pointers
+\               0                     8             Args sentinel
+\               ``argv``              8 * ``argc``  Pointers to argument strings
+``rsp``         ``argc``              8             Number of elements in argv
+==============  ====================  ============  =======
+
+
+* :ref:`genindex`
+* :ref:`search`
+
+

docs/requirements.txt

@@ -0,0 +1,5 @@
+cogapp >= 3
+pyyaml >= 5.4
+lark == 0.12.0
+sphinx
+renku-sphinx-theme

docs/syscall_interface.rst

@@ -0,0 +1,444 @@
+.. jsix syscall interface.
+.. Automatically updated from the definition files using cog!
+
+.. [[[cog code generation
+.. from textwrap import indent
+.. from definitions.context import Context
+.. 
+.. ctx = Context(definitions_path)
+.. ctx.parse("syscalls.def")
+.. syscalls = ctx.interfaces["syscalls"]
+.. 
+.. def caplist(caps):
+..   return ', '.join([f"``{c}``" for c in caps])
+.. ]]]
+.. [[[end]]] (checksum: d41d8cd98f00b204e9800998ecf8427e)
+
+Syscall interface
+=================
+
+The jsix kernel's syscall design is based around object handles. Object handles
+are also a collection of capabilities, encoding certain rights over the object
+they reference.
+
+Very few syscalls in jsix can be made without some handle, and most of them are
+requests to the kernel to create a given kind of object. This is analogous to
+methods on an object in an object-oriented programming language.
+
+.. [[[cog code generation
+.. cog.outl()
+.. for obj in syscalls.exposes:
+..     cog.outl(f"``{obj.name}`` syscalls")
+..     cog.outl(f"-------------------------")
+..     desc = obj.desc or "Undocumented"
+..     cog.outl(desc)
+..     cog.outl()
+..     cog.outl(f":capabilites:  {caplist(obj.caps)}")
+..     cog.outl()
+..     for method in obj.methods:
+..       args = []
+..       if method.constructor:
+..         args.append("j6_handle_t *self")
+..       elif not method.static:
+..         args.append("j6_handle_t self")
+..
+..       for param in method.params:
+..         for type, suffix in param.type.c_names(param.options):
+..           args.append(f"{type} {param.name}{suffix}")
+..
+..       cog.outl(f".. cpp:function:: j6_result_t j6_{obj.name}_{method.name} ({', '.join(args)})")
+..       cog.outl()
+..       desc = method.desc or "Undocumented"
+..       cog.outl(indent(desc, "   "))
+..       cog.outl()
+..       if "cap" in method.options:
+..         cog.outl(f"   :capabilities: {caplist(method.options['cap'])}")
+..         cog.outl()
+..       if method.constructor:
+..         cog.outl(f"   :param self: *[out]* Handle to the new {obj.name} object")
+..       elif not method.static:
+..         cog.outl(f"   :param self: Handle to the {obj.name} object")
+..       for param in method.params:
+..         opts = param.options and f"*[{', '.join(param.options)}]*" or ""
+..         desc = param.desc or 'Undocumented'
+..         cog.outl(f"   :param {param.name}: {opts} {desc}")
+..       cog.outl()
+.. ]]]
+
+``object`` syscalls
+-------------------------
+All kernel-exposed objects inherit from the base ``object`` type, so the
+``object`` syscalls can be used with any object's handle.
+
+:capabilites:  ``clone``
+
+.. cpp:function:: j6_result_t j6_object_koid (j6_handle_t self, uint64_t * koid)
+
+   Get the internal kernel object id of an object
+
+   :param self: Handle to the object object
+   :param koid: *[out]* Undocumented
+
+``event`` syscalls
+-------------------------
+An ``event`` is a simple synchronization object. It contains up to 64 signals
+that threads can wait for and signal in parallel.
+
+:capabilites:  ``signal``, ``wait``
+
+.. cpp:function:: j6_result_t j6_event_create (j6_handle_t *self)
+
+   Undocumented
+
+   :param self: *[out]* Handle to the new event object
+
+.. cpp:function:: j6_result_t j6_event_signal (j6_handle_t self, uint64_t signals)
+
+   Signal events on this object
+
+   :capabilities: ``signal``
+
+   :param self: Handle to the event object
+   :param signals:  A bitset of which events to signal
+
+.. cpp:function:: j6_result_t j6_event_wait (j6_handle_t self, uint64_t * signals, uint64_t timeout)
+
+   Wait for signaled events on this object
+
+   :capabilities: ``wait``
+
+   :param self: Handle to the event object
+   :param signals: *[out]* A bitset of which events were signaled
+   :param timeout:  Wait timeout in nanoseconds
+
+``mailbox`` syscalls
+-------------------------
+Mailboxes are objects that enable synchronous IPC via arbitrary
+message-passing of tagged data and/or handles. Not as efficient
+as shared memory channels, but more flexible.
+
+:capabilites:  ``send``, ``receive``, ``close``
+
+.. cpp:function:: j6_result_t j6_mailbox_create (j6_handle_t *self)
+
+   Undocumented
+
+   :param self: *[out]* Handle to the new mailbox object
+
+.. cpp:function:: j6_result_t j6_mailbox_close (j6_handle_t self)
+
+   Undocumented
+
+   :capabilities: ``close``
+
+   :param self: Handle to the mailbox object
+
+.. cpp:function:: j6_result_t j6_mailbox_call (j6_handle_t self, uint64_t * tag, void * data, size_t * data_len, size_t data_size, j6_handle_t * handles, size_t * handles_count, size_t handles_size)
+
+   Send a message to the reciever, and block until a response is
+   sent. Note that getting this response does not require the
+   receive capability.
+
+   :capabilities: ``send``
+
+   :param self: Handle to the mailbox object
+   :param tag: *[inout]* Undocumented
+   :param data: *[optional, inout]* Undocumented
+   :param data_size:  number of total bytes in data buffer
+   :param handles: *[optional, inout, handle, list]* Undocumented
+   :param handles_size:  total size of handles buffer
+
+.. cpp:function:: j6_result_t j6_mailbox_respond (j6_handle_t self, uint64_t * tag, void * data, size_t * data_len, size_t data_size, j6_handle_t * handles, size_t * handles_count, size_t handles_size, uint64_t * reply_tag, uint64_t flags)
+
+   Respond to a message sent using call, and wait for another
+   message to arrive. Note that this does not require the send
+   capability. A reply tag of 0 skips the reply and goes directly
+   to waiting for a new message.
+
+   :capabilities: ``receive``
+
+   :param self: Handle to the mailbox object
+   :param tag: *[inout]* Undocumented
+   :param data: *[optional, inout]* Undocumented
+   :param data_size:  number of total bytes in data buffer
+   :param handles: *[optional, inout, handle, list]* Undocumented
+   :param handles_size:  total size of handles buffer
+   :param reply_tag: *[inout]* Undocumented
+   :param flags:  Undocumented
+
+``process`` syscalls
+-------------------------
+A ``process`` object represents a process running on the system, and allows
+control over the threads, handles, and virtual memory space of that process.
+
+:capabilites:  ``kill``, ``create_thread``
+
+.. cpp:function:: j6_result_t j6_process_create (j6_handle_t *self)
+
+   Create a new empty process
+
+   :param self: *[out]* Handle to the new process object
+
+.. cpp:function:: j6_result_t j6_process_kill (j6_handle_t self)
+
+   Stop all threads and exit the given process
+
+   :capabilities: ``kill``
+
+   :param self: Handle to the process object
+
+.. cpp:function:: j6_result_t j6_process_exit (int64_t result)
+
+   Stop all threads and exit the current process
+
+   :param result:  The result to retrun to the parent process
+
+.. cpp:function:: j6_result_t j6_process_give_handle (j6_handle_t self, j6_handle_t target)
+
+   Give the given process a handle that points to the same
+   object as the specified handle.
+
+   :param self: Handle to the process object
+   :param target: *[handle]* A handle in the caller process to send
+
+``system`` syscalls
+-------------------------
+The singular ``system`` object represents a handle to kernel functionality
+needed by drivers and other priviledged services.
+
+:capabilites:  ``get_log``, ``bind_irq``, ``map_phys``, ``change_iopl``
+
+.. cpp:function:: j6_result_t j6_system_get_log (j6_handle_t self, uint64_t seen, void * buffer, size_t * buffer_len)
+
+   Get the next log line from the kernel log
+
+   :capabilities: ``get_log``
+
+   :param self: Handle to the system object
+   :param seen:  Last seen log id
+   :param buffer: *[out, zero_ok]* Buffer for the log message data structure
+
+.. cpp:function:: j6_result_t j6_system_bind_irq (j6_handle_t self, j6_handle_t dest, unsigned irq, unsigned signal)
+
+   Ask the kernel to send this process messages whenever
+   the given IRQ fires
+
+   :capabilities: ``bind_irq``
+
+   :param self: Handle to the system object
+   :param dest:  Event object that will receive messages
+   :param irq:  IRQ number to bind
+   :param signal:  Signal number on the event to bind to
+
+.. cpp:function:: j6_result_t j6_system_map_phys (j6_handle_t self, j6_handle_t * area, uintptr_t phys, size_t size, uint32_t flags)
+
+   Create a VMA and map an area of physical memory into it,
+   also mapping that VMA into the current process
+
+   :capabilities: ``map_phys``
+
+   :param self: Handle to the system object
+   :param area: *[out]* Receives a handle to the VMA created
+   :param phys:  The physical address of the area
+   :param size:  Size of the area, in bytes
+   :param flags:  Flags to apply to the created VMA
+
+.. cpp:function:: j6_result_t j6_system_request_iopl (j6_handle_t self, unsigned iopl)
+
+   Request the kernel change the IOPL for this process. The only values
+   that make sense are 0 and 3.
+
+   :capabilities: ``change_iopl``
+
+   :param self: Handle to the system object
+   :param iopl:  The IOPL to set for this process
+
+``thread`` syscalls
+-------------------------
+A ``thread`` object represents a thread of execution within a process running
+on the system. The actual thread does not need to be currently running to
+hold a handle to it.
+
+:capabilites:  ``kill``, ``join``
+
+.. cpp:function:: j6_result_t j6_thread_create (j6_handle_t *self, j6_handle_t process, uintptr_t stack_top, uintptr_t entrypoint, uint64_t arg0, uint64_t arg1)
+
+   Undocumented
+
+   :param self: *[out]* Handle to the new thread object
+   :param process: *[optional, cap]* Undocumented
+   :param stack_top:  Undocumented
+   :param entrypoint:  Undocumented
+   :param arg0:  Undocumented
+   :param arg1:  Undocumented
+
+.. cpp:function:: j6_result_t j6_thread_kill (j6_handle_t self)
+
+   Undocumented
+
+   :capabilities: ``kill``
+
+   :param self: Handle to the thread object
+
+.. cpp:function:: j6_result_t j6_thread_join (j6_handle_t self)
+
+   Undocumented
+
+   :capabilities: ``join``
+
+   :param self: Handle to the thread object
+
+.. cpp:function:: j6_result_t j6_thread_exit ()
+
+   Undocumented
+
+
+.. cpp:function:: j6_result_t j6_thread_sleep (uint64_t duration)
+
+   Undocumented
+
+   :param duration:  Undocumented
+
+``vma`` syscalls
+-------------------------
+A ``vma`` object represents a single virtual memory area, which may be shared
+between several processes. A process having a handle to a ``vma`` does not
+necessarily mean that it is mapped into that process' virtual memory space.
+
+:capabilites:  ``map``, ``unmap``, ``resize``
+
+.. cpp:function:: j6_result_t j6_vma_create (j6_handle_t *self, size_t size, uint32_t flags)
+
+   Undocumented
+
+   :param self: *[out]* Handle to the new vma object
+   :param size:  Undocumented
+   :param flags:  Undocumented
+
+.. cpp:function:: j6_result_t j6_vma_create_map (j6_handle_t *self, size_t size, uintptr_t * address, uint32_t flags)
+
+   Undocumented
+
+   :capabilities: ``map``
+
+   :param self: *[out]* Handle to the new vma object
+   :param size:  Undocumented
+   :param address: *[inout]* Undocumented
+   :param flags:  Undocumented
+
+.. cpp:function:: j6_result_t j6_vma_map (j6_handle_t self, j6_handle_t process, uintptr_t * address, uint32_t flags)
+
+   Undocumented
+
+   :capabilities: ``map``
+
+   :param self: Handle to the vma object
+   :param process: *[optional]* Undocumented
+   :param address: *[inout]* Undocumented
+   :param flags:  Undocumented
+
+.. cpp:function:: j6_result_t j6_vma_unmap (j6_handle_t self, j6_handle_t process)
+
+   Undocumented
+
+   :capabilities: ``unmap``
+
+   :param self: Handle to the vma object
+   :param process: *[optional]* Undocumented
+
+.. cpp:function:: j6_result_t j6_vma_resize (j6_handle_t self, size_t * size)
+
+   Undocumented
+
+   :capabilities: ``resize``
+
+   :param self: Handle to the vma object
+   :param size: *[inout]* New size for the VMA, or 0 to query the current size without changing
+
+.. [[[end]]] (checksum: cb17f54e443d1d3b85995870f3e8dbf2)
+
+Non-object syscalls
+-------------------
+
+The following are the system calls that aren't constructors for objects, and
+either do not require an object handle, or operate generically on handles.
+
+.. [[[cog code generation
+.. cog.outl()
+.. for func in syscalls.functions:
+..   args = []
+..   for param in func.params:
+..     for type, suffix in param.type.c_names(param.options):
+..       args.append(f"{type} {param.name}{suffix}")
+..
+..   cog.outl(f".. cpp:function:: j6_result_t j6_{func.name} ({', '.join(args)})")
+..   cog.outl()
+..   desc = func.desc or "Undocumented"
+..   cog.outl(indent(desc, "   "))
+..   cog.outl()
+..   for param in func.params:
+..     opts = param.options and f"*[{', '.join(param.options)}]*" or ""
+..     desc = param.desc or 'Undocumented'
+..     cog.outl(f"   :param {param.name}: {opts} {desc}")
+..   cog.outl()
+.. ]]]
+
+.. cpp:function:: j6_result_t j6_noop ()
+
+   Simple no-op syscall for testing
+
+
+.. cpp:function:: j6_result_t j6_log (uint8_t area, uint8_t severity, const char * message)
+
+   Write a message to the kernel log
+
+   :param area:  Undocumented
+   :param severity:  Undocumented
+   :param message:  Undocumented
+
+.. cpp:function:: j6_result_t j6_handle_list (struct j6_handle_descriptor * handles, size_t * handles_size)
+
+   Get a list of handles owned by this process. If the
+   supplied list is not big enough, will set the size
+   needed in `size` and return j6_err_insufficient
+
+   :param handles: *[list, inout, zero_ok]* A list of handles to be filled
+
+.. cpp:function:: j6_result_t j6_handle_clone (j6_handle_t orig, j6_handle_t * clone, uint32_t mask)
+
+   Create a clone of an existing handle, possibly with
+   some capabilities masked out.
+
+   :param orig: *[handle, cap]* The handle to clone
+   :param clone: *[out]* The new handle
+   :param mask:  The capability bitmask
+
+.. cpp:function:: j6_result_t j6_handle_close (j6_handle_t hnd)
+
+   Close the handle to an object
+
+   :param hnd: *[handle]* The handle to close
+
+.. cpp:function:: j6_result_t j6_futex_wait (const uint32_t * address, uint32_t current, uint64_t timeout)
+
+   Block waiting on a futex
+
+   :param address:  Address of the futex value
+   :param current:  Current value of the futex
+   :param timeout:  Wait timeout in nanoseconds
+
+.. cpp:function:: j6_result_t j6_futex_wake (const uint32_t * address, uint64_t count)
+
+   Wake threads waiting on a futex
+
+   :param address:  Address of the futex value
+   :param count:  Number of threads to wake, or 0 for all
+
+.. cpp:function:: j6_result_t j6_test_finish (uint32_t exit_code)
+
+   Testing mode only: Have the kernel finish and exit QEMU with the given exit code
+
+   :param exit_code:  Undocumented
+
+.. [[[end]]] (checksum: 0b9d051972abcbb6de408f411331785f)
+

external/zstd.module

@@ -3,8 +3,8 @@
 zstd = module("zstd",
     root = "${source_root}/external/zstd",
     kind = "lib",
-    deps = [ ],
-    output = "libzstd.a",
+    copy_headers = True,
+    deps = [ "libc" ],
     sources = [
         "decompress/zstd_decompress.c",
         "decompress/zstd_ddict.c",

qemu.sh

@@ -1,7 +1,10 @@
 #!/usr/bin/env bash
 
-build="$(dirname $0)/build"
-assets="$(dirname $0)/assets"
+root=$(dirname $0)
+build="${root}/build.amd64"
+assets="${root}/assets"
+
+no_build=""
 debug=""
 isaexit='-device isa-debug-exit,iobase=0xf4,iosize=0x04'
 debugtarget="${build}/jsix.elf"
@@ -9,7 +12,9 @@ gfx="-nographic"
 vga="-vga none"
 log=""
 kvm=""
-cpu="Broadwell,+pdpe1gb"
+debugcon_cmd=""
+cpu_features=",+pdpe1gb,+invtsc,+hypervisor,+x2apic,+xsavec,+xsaves,+xsaveopt"
+cpu="Broadwell"
 smp=4
 
 while true; do
@@ -35,14 +40,14 @@ while true; do
             ;;
         -r | --remote)
             shift
-            vnchost="${1:-${VNCHOST:-localhost}}"
-            gfx="-vnc ${vnchost}:7000,reverse"
+            vnchost="${1:-${VNCHOST:-"localhost:5500"}}"
+            gfx="-vnc ${vnchost},reverse=on"
             vga=""
             shift
             ;;
         -k | --kvm)
             kvm="-enable-kvm"
-            cpu="host"
+            cpu="max"
             shift
             ;;
         -c | --cpus)
@@ -50,7 +55,15 @@ while true; do
             shift 2
             ;;
         -l | --log)
-            log="-d mmu,int,guest_errors -D jsix.log"
+            log="-d mmu,int,guest_errors -D ${root}/jsix.log"
+            shift
+            ;;
+        -x | --debugcon)
+            debugcon_cmd="less --follow-name -R +F debugcon.log"
+            shift
+            ;;
+        --no-build)
+            no_build="yes"
             shift
             ;;
         *)
@@ -67,39 +80,71 @@ if [[ ! -c /dev/kvm ]]; then
     kvm=""
 fi
 
-if ! ninja -C "${build}"; then
+[[ -z "${no_build}" ]] && if ! ninja -C "${build}"; then
     exit 1
 fi
 
 if [[ -n $TMUX ]]; then
+    cols=$(tput cols)
+    log_width=100
+
     if [[ -n $debug ]]; then
-        tmux split-window -h "gdb ${debugtarget}" &
+        log_cols=1
+        if [[ $debugcon_cmd ]]; then
+            log_cols=2
+        fi
+
+        gdb_width=$(($cols - $log_cols * $log_width))
+
+        if (($gdb_width < 150)); then
+            stack=1
+            gdb_width=$(($cols - $log_width))
+            tmux split-window -h -l $gdb_width "gdb ${debugtarget}"
+            if [[ $debugcon_cmd ]]; then
+                tmux select-pane -t .left
+                tmux split-window -v "$debugcon_cmd"
+            fi
         else
-        tmux split-window -h -l 100 "sleep 1; telnet localhost 45455" &
+            if [[ $debugcon_cmd ]]; then
+                tmux split-window -h -l $(($log_width + $gdb_width)) "$debugcon_cmd"
+            fi
+            tmux split-window -h -l $gdb_width "gdb ${debugtarget}"
+            tmux select-pane -t .left
+            tmux select-pane -t .right
+        fi
+    else
+        if [[ $debugcon_cmd ]]; then
+            tmux split-window -h -l $log_width "$debugcon_cmd"
             tmux last-pane
-        tmux split-window -l 10 "sleep 1; telnet localhost 45454" &
+        fi
+        tmux split-window -l 10 "sleep 1; nc localhost 45454"
     fi
 elif [[ $DESKTOP_SESSION = "i3" ]]; then
     if [[ -n $debug ]]; then
         i3-msg exec i3-sensible-terminal -- -e "gdb ${debugtarget}" &
     else
-        i3-msg exec i3-sensible-terminal -- -e 'telnet localhost 45454' &
+        i3-msg exec i3-sensible-terminal -- -e 'nc localhost 45454' &
     fi
 fi
 
+if [[ -n "${debug}" ]]; then
+    (sleep 1; echo "Debugging ready.") &
+fi
+
 qemu-system-x86_64 \
-    -drive "if=pflash,format=raw,readonly,file=${assets}/ovmf/x64/ovmf_code.fd" \
+    -drive "if=pflash,format=raw,readonly=on,file=${assets}/ovmf/x64/ovmf_code.fd" \
     -drive "if=pflash,format=raw,file=${build}/ovmf_vars.fd" \
     -drive "format=raw,file=${build}/jsix.img" \
-    -chardev socket,host=localhost,port=45455,server,nowait,telnet=on,id=jsix_debug -device isa-debugcon,iobase=0x6600,chardev=jsix_debug \
+    -chardev "file,path=${root}/debugcon.log,id=jsix_debug" \
+    -device "isa-debugcon,iobase=0x6600,chardev=jsix_debug" \
     -monitor telnet:localhost:45454,server,nowait \
     -serial stdio \
     -smp "${smp}" \
     -m 4096 \
-    -cpu "${cpu}" \
+    -cpu "${cpu}${cpu_features}" \
     -M q35 \
     -no-reboot \
-    -name "jsix" \
+    -name jsix \
     $isaexit $log $gfx $vga $kvm $debug
 
 ((result = ($? >> 1) - 1))

requirements.txt

@@ -5,3 +5,7 @@ pyyaml >= 5.4
 lark == 0.12.0
 pure-cdb == 4
 pyzstd == 0.15
+pyelftools
+iced-x86
+sphinx
+renku-sphinx-theme

scripts/bonnibel/config.py

@@ -0,0 +1,68 @@
+_config_cache = {}
+
+def _load(filename, depfiles):
+    from . import load_config
+    if not filename in _config_cache:
+        if filename.exists():
+            depfiles.add(filename)
+            data = load_config(filename)
+            _config_cache[filename] = data
+    return _config_cache.get(filename, dict())
+
+
+def _build_config(chain, depfiles):
+    config = {}
+    for d in [_load(c, depfiles) for c in chain]:
+        for k, v in d.items():
+            if isinstance(v, (list, tuple)):
+                config[k] = config.get(k, list()) + list(v)
+            elif isinstance(v, dict):
+                config[k] = config.get(k, dict())
+                config[k].update(v)
+            else:
+                config[k] = v
+
+    return config
+
+
+def _make_ninja_config(outfile, config, files):
+    from os import makedirs
+    from ninja.ninja_syntax import Writer
+
+    makedirs(outfile.parent, exist_ok=True)
+
+    with open(outfile, "w") as buildfile:
+        build = Writer(buildfile)
+        build.comment("This file is automatically generated by bonnibel")
+        build.comment("Source config files:")
+        for f in files:
+            build.comment(f"  - {f}")
+
+        build.newline()
+        for k, v in config.items():
+            build.variable(k, v)
+
+ 
+def generate_configs(root, output, buildconfig, arch, targets, kinds):
+
+    assets = root / "assets" / "build" / arch
+    base = ["global.yaml", f"config.{buildconfig}.yaml"]
+
+    depfiles = set()
+
+    for target in targets:
+        chain = base + [f"target.{target}.yaml"]
+
+        files = [assets / f for f in chain]
+        config = _build_config(files, depfiles)
+        outfile = output / target / f"config.ninja"
+        _make_ninja_config(outfile, config, files)
+
+        for kind in kinds:
+            custom = [f"kind.{kind}.yaml", f"target.{target}.{kind}.yaml"]
+            files = [assets / f for f in chain + custom]
+            config = _build_config(files, depfiles)
+            outfile = output / target / f"config.{kind}.ninja"
+            _make_ninja_config(outfile, config, files)
+
+    return depfiles

scripts/bonnibel/manifest.py

@@ -26,7 +26,7 @@ class Manifest:
                 name="kernel", target="kernel")
 
         self.init = self.__build_entry(modules,
-                config.get("init", None))
+                config.get("init", None), target="init")
 
         self.panics = [self.__build_entry(modules, i, target="kernel")
                 for i in config.get("panic", tuple())]
@@ -37,6 +37,19 @@ class Manifest:
         self.drivers = [self.__build_entry(modules, i)
                 for i in config.get("drivers", tuple())]
 
+        def get_libdeps(names):
+            libmods = modules.get_mods(names)
+            deps = modules.all_deps(libmods, stop_at_static=True)
+            deps = [m.name for m in deps if m.kind == "lib"]
+            return deps
+
+        libs = set(get_libdeps(config.get("libs", tuple())))
+        libs.update(get_libdeps([e.module for e in self.services]))
+        libs.update(get_libdeps([e.module for e in self.drivers]))
+
+        self.libs = [self.__build_entry(modules, i)
+                for i in libs]
+
         self.flags = config.get("flags", tuple())
 
         self.symbols = ""
@@ -71,7 +84,7 @@ class Manifest:
             if not f in Manifest.flags:
                 raise BonnibelError(f"Manifest specifies unknown flag '{f}'")
 
-        return Manifest.Entry(name, target, mod.output, flags)
+        return Manifest.Entry(name, target, mod.get_output(), flags)
 
     def add_data(self, output, desc, flags=tuple()):
         e = Manifest.Entry(None, None, output, flags)

scripts/bonnibel/module.py

@@ -1,4 +1,5 @@
 from . import include_rel, mod_rel, target_rel
+from . import BonnibelError
 
 def resolve(path):
     if path.startswith('/') or path.startswith('$'):
@@ -7,29 +8,43 @@ def resolve(path):
     return str(Path(path).resolve())
 
 class BuildOptions:
-    def __init__(self, includes=tuple(), libs=tuple(), order_only=tuple(), ld_script=None):
+    def __init__(self, includes=tuple(), local=tuple(), late=tuple(), libs=tuple(), order_only=tuple(), ld_script=None):
         self.includes = list(includes)
+        self.local = list(local)
+        self.late = list(late)
         self.libs = list(libs)
         self.order_only = list(order_only)
         self.ld_script = ld_script and str(ld_script)
 
     @property
     def implicit(self):
+        libfiles = list(map(lambda x: x[0], self.libs))
         if self.ld_script is not None:
-            return self.libs + [self.ld_script]
+            return libfiles + [self.ld_script]
         else:
-            return self.libs
+            return libfiles
+
+    @property
+    def linker_args(self):
+        from pathlib import Path
+        def arg(path, static):
+            if static: return path
+            return "-l:" + Path(path).name
+        return [arg(*l) for l in self.libs]
 
 
 class Module:
     __fields = {
         # name: (type, default)
         "kind": (str, "exe"),
-        "output": (str, None),
-        "targets": (set, ()),
+        "outfile": (str, None),
+        "basename": (str, None),
+        "target": (str, None),
         "deps":  (set, ()),
         "public_headers":  (set, ()),
+        "copy_headers": (bool, False),
         "includes": (tuple, ()),
+        "include_phase": (str, "normal"),
         "sources": (tuple, ()),
         "drivers": (tuple, ()),
         "variables": (dict, ()),
@@ -37,13 +52,17 @@ class Module:
         "description": (str, None),
         "no_libc": (bool, False),
         "ld_script": (str, None),
+        "static": (bool, False),
+        "arch_source": (dict, ()),
+        "skip_arches": (tuple, ()),
     }
 
     def __init__(self, name, modfile, root, **kwargs):
-        from .source import make_source
+        from pathlib import Path
+        from .source import make_source, make_copy_source
 
         # Required fields
-        self.root = root
+        self.root = Path(root)
         self.name = name
         self.modfile = modfile
 
@@ -57,106 +76,256 @@ class Module:
 
         for name in kwargs:
             if not name in self.__fields:
-                raise AttributeError(f"No attribute named {name} on Module")
+                raise AttributeError(f"No attribute named {name} on Module ({modfile})")
+
+        if not self.no_libc:
+            self.deps.add("libc_free")
 
         # Turn strings into real Source objects
         self.sources = [make_source(root, f) for f in self.sources]
-        self.public_headers = [make_source(root, f) for f in self.public_headers]
+        for arch in self.arch_source:
+            self.arch_source[arch] = [make_source(root, f) for f in self.arch_source[arch]]
 
-        # Filled by Module.update
-        self.depmods = set()
+        header_source = lambda f: make_source(root, Path("include") / f)
+        if self.copy_headers:
+            header_source = lambda f: make_copy_source(root, f, "include")
+        self.public_headers = [header_source(f) for f in self.public_headers]
 
-    def __str__(self):
-        return "Module {} {}\n\t{}".format(self.kind, self.name, "\n\t".join(map(str, self.sources)))
+    def __repr__(self):
+        return f"<Module {self.kind} {self.name}>"
 
     @property
-    def output(self):
-        if self.__output is not None:
-            return self.__output
-
+    def basename(self):
+        if self.__basename is not None:
+            return self.__basename
         if self.kind == "lib":
-            return f"lib{self.name}.a"
+            return f"lib{self.name}"
         else:
-            return f"{self.name}.elf"
+            return self.name
 
-    @output.setter
-    def output(self, value):
-        self.__output = value
+    @basename.setter
+    def basename(self, value):
+        self.__basename = value
 
-    @classmethod
-    def update(cls, mods):
-        from . import BonnibelError
+    def get_output(self, static=False):
+        if self.outfile is not None:
+            return self.outfile
+        elif self.kind == "headers":
+            return None
+        else:
+            ext = dict(exe=".elf", driver=".drv", lib=(static and ".a" or ".so"))
+            return self.basename + ext.get(self.kind, "")
 
-        def resolve(source, modlist):
-            resolved = set()
-            for dep in modlist:
-                if not dep in mods:
-                    raise BonnibelError(f"module '{source.name}' references unknown module '{dep}'")
-                mod = mods[dep]
-                resolved.add(mod)
-            return resolved
+    def add_input(self, path, **kwargs):
+        from .source import make_source
+        s = make_source(self.root, path, **kwargs)
+        self.sources.append(s)
+        return s.outputs
 
-        for mod in mods.values():
-            mod.depmods = resolve(mod, mod.deps)
+    def add_depends(self, paths, deps):
+        for source in self.sources:
+            if source.path in paths:
+                source.add_deps(deps)
 
-        target_mods = [mod for mod in mods.values() if mod.targets]
-        for mod in target_mods:
-            closed = set()
-            children = set(mod.depmods)
-            while children:
-                child = children.pop()
-                closed.add(child)
-                child.targets |= mod.targets
-                children |= {m for m in child.depmods if not m in closed}
+        for source in self.public_headers:
+            if source.path in paths:
+                source.add_deps(deps)
+
+
+class ModuleList:
+    def __init__(self, arch):
+        self.__arch = arch
+        self.__mods = {}
+        self.__used = {}
+        self.__targets = frozenset()
+        self.__kinds = frozenset()
+
+    def __getitem__(self, name):
+        return self.__mods.get(name)
+
+    def __contains__(self, name):
+        """Return if the module name is known."""
+        return name in self.__mods
+
+    def __iter__(self):
+        """Iterate over _non-skipped_ modules."""
+        return self.used.__iter__()
+
+    def __skip(self, mod):
+        if self.__arch in mod.skip_arches: return True
+        return False
+
+    @property
+    def used(self):
+        if self.__used is None:
+            self.__used = {n:m for n,m in self.__mods.items() if not self.__skip(m)}
+        return self.__used
+
+    @property
+    def targets(self):
+        if self.__targets is None:
+            self.__targets = frozenset([m.target for m in self.used.values() if m.target])
+        return self.__targets
+
+    @property
+    def kinds(self):
+        if self.__kinds is None:
+            self.__kinds = frozenset([m.kind for m in self.used.values()])
+        return self.__kinds
+
+    def get(self, name):
+        return self.__mods.get(name)
+
+    def add(self, mod):
+        if mod.name in self.__mods:
+            raise BonnibelError(f"re-adding module '{mod.name}' to this ModuleList")
+        self.__mods[mod.name] = mod
+        self.__used = None
+        self.__targets = None
+        self.__kinds = None
+
+    def get_mods(self, names, filt=None):
+        return {self[n] for n in names
+                if n in self.used
+                and ((not filt) or filt(self[n]))}
+
+    def all_deps(self, mods, stop_at_static=False):
+        search = set(mods)
+        closed = list()
+
+        while search:
+            mod = search.pop()
+            if mod in closed: continue
+            closed.append(mod)
+
+            if stop_at_static and mod.static:
+                continue
+
+            for dep in mod.deps:
+                if not dep in self.__mods:
+                    raise BonnibelError(f"module '{mod.name}' references unknown module '{dep}'")
+                if dep in self.used:
+                    search.add(self.used[dep])
+
+        return closed
+
+    def target_mods(self, target):
+        return self.all_deps([m for m in self.used.values() if m.target == target])
 
     def generate(self, output):
         from pathlib import Path
         from collections import defaultdict
         from ninja.ninja_syntax import Writer
 
-        def walk_deps(deps):
-            open_set = set(deps)
-            closed_set = set()
-            while open_set:
-                dep = open_set.pop()
-                closed_set.add(dep)
-                open_set |= {m for m in dep.depmods if not m in closed_set}
-            return closed_set
+        def walk_deps(deps, static, results):
+            for modname in deps:
+                mod = self.used.get(modname)
+                if not mod: continue # skipped
+                if static or modname not in results:
+                    results[modname] = (mod, static)
+                walk_deps(mod.deps, static or mod.static, results)
 
-        all_deps = walk_deps(self.depmods)
+        for mod in self.used.values():
+            all_deps = {}
+            walk_deps(mod.deps, mod.static, all_deps)
+            all_deps = all_deps.values()
 
-        def gather_phony(build, deps, child_rel, add_headers=False):
+            def gather_phony(build, deps, child_rel):
                 phony = ".headers.phony"
                 child_phony = [child_rel(phony, module=c.name)
-                    for c in all_deps]
-
-            header_targets = []
-            if add_headers:
-                if not self.no_libc:
-                    header_targets.append(f"${{build_root}}/include/libc/{phony}")
-                if self.public_headers:
-                    header_targets.append(f"${{build_root}}/include/{self.name}/{phony}")
+                        for c, _ in all_deps]
 
                 build.build(
                     rule = "touch",
                     outputs = [mod_rel(phony)],
-                implicit = child_phony + header_targets,
+                    implicit = child_phony,
                     order_only = list(map(mod_rel, deps)),
                 )
 
-        filename = str(output / f"headers.{self.name}.ninja")
+            filename = str(output / f"module.{mod.name}.ninja")
             with open(filename, "w") as buildfile:
                 build = Writer(buildfile)
 
                 build.comment("This file is automatically generated by bonnibel")
                 build.newline()
 
-            build.variable("module_dir", f"${{build_root}}/include/{self.name}")
+                build.variable("module_dir", target_rel(mod.name + ".dir"))
+                build.variable("module_kind", mod.kind)
+                build.newline()
+
+                build.include(f"${{target_dir}}/config.{mod.kind}.ninja")
+                build.newline()
+
+                modopts = BuildOptions(
+                    local = [mod.root, "${module_dir}"],
+                    ld_script = mod.ld_script and mod.root / mod.ld_script,
+                    )
+                if mod.public_headers:
+                    modopts.includes += [
+                        mod.root / "include",
+                        f"${{target_dir}}/{mod.name}.dir/include",
+                    ]
+
+                for key, value in mod.variables.items():
+                    build.variable(key, value)
+                build.newline()
+
+                for include in mod.includes:
+                    p = Path(include)
+                    if p.is_absolute():
+                        if not p in modopts.includes:
+                            modopts.includes.append(str(p.resolve()))
+                    elif include != ".":
+                        incpath = mod.root / p
+                        destpath = mod_rel(p)
+                        for header in incpath.rglob("*.h"):
+                            dest_header = f"{destpath}/" + str(header.relative_to(incpath))
+                        modopts.includes.append(str(incpath))
+                        modopts.includes.append(destpath)
+
+                for dep, static in all_deps:
+                    if dep.public_headers:
+                        if dep.include_phase == "normal":
+                            modopts.includes += [dep.root / "include", f"${{target_dir}}/{dep.name}.dir/include"]
+                        elif dep.include_phase == "late":
+                            modopts.late += [dep.root / "include", f"${{target_dir}}/{dep.name}.dir/include"]
+                        else:
+                            from . import BonnibelError
+                            raise BonnibelError(f"Module {dep.name} has invalid include_phase={dep.include_phase}")
+
+                    if dep.kind == "headers":
+                        continue
+                    elif dep.kind == "lib":
+                        modopts.libs.append((target_rel(dep.get_output(static)), static))
+                    else:
+                        modopts.order_only.append(target_rel(dep.get_output(static)))
+
+                cc_includes = []
+                if modopts.local:
+                    cc_includes += [f"-iquote{i}" for i in modopts.local]
+
+                if modopts.includes:
+                    cc_includes += [f"-I{i}" for i in modopts.includes]
+
+                if modopts.late:
+                    cc_includes += [f"-idirafter{i}" for i in modopts.late]
+
+                if cc_includes:
+                    build.variable("ccflags", ["${ccflags}"] + cc_includes)
+
+                as_includes = [f"-I{d}" for d in modopts.local + modopts.includes + modopts.late]
+                if as_includes:
+                    build.variable("asflags", ["${asflags}"] + as_includes)
+
+                if modopts.libs:
+                    build.variable("libs", ["-L${target_dir}", "${libs}"] + modopts.linker_args)
+
+                if modopts.ld_script:
+                    build.variable("ldflags", ["${ldflags}"] + ["-T", modopts.ld_script])
 
                 header_deps = []
-
                 inputs = []
-            headers = set(self.public_headers)
+                headers = set(mod.public_headers)
                 while headers:
                     source = headers.pop()
                     headers.update(source.next)
@@ -172,65 +341,10 @@ class Module:
                         inputs.extend(map(mod_rel, source.outputs))
 
                 build.newline()
-            gather_phony(build, header_deps, include_rel)
-
-        filename = str(output / f"module.{self.name}.ninja")
-        with open(filename, "w") as buildfile:
-            build = Writer(buildfile)
-
-            build.comment("This file is automatically generated by bonnibel")
-            build.newline()
-
-            build.variable("module_dir", target_rel(self.name + ".dir"))
-
-            modopts = BuildOptions(
-                includes = [self.root, "${module_dir}"],
-                ld_script = self.ld_script and self.root / self.ld_script,
-                )
-            if self.public_headers:
-                modopts.includes += [f"${{build_root}}/include/{self.name}"]
-
-            for key, value in self.variables.items():
-                build.variable(key, value)
-            build.newline()
-
-            for include in self.includes:
-                p = Path(include)
-                if p.is_absolute():
-                    if not p in modopts.includes:
-                        modopts.includes.append(str(p.resolve()))
-                elif include != ".":
-                    incpath = self.root / p
-                    destpath = mod_rel(p)
-                    for header in incpath.rglob("*.h"):
-                        dest_header = f"{destpath}/" + str(header.relative_to(incpath))
-                    modopts.includes.append(str(incpath))
-                    modopts.includes.append(destpath)
-
-            all_deps = walk_deps(self.depmods)
-            for dep in all_deps:
-                if dep.public_headers:
-                    modopts.includes += [f"${{build_root}}/include/{dep.name}"]
-
-                if dep.kind == "lib":
-                    modopts.libs.append(target_rel(dep.output))
-                else:
-                    modopts.order_only.append(target_rel(dep.output))
-
-            if modopts.includes:
-                build.variable("ccflags", ["${ccflags}"] + [f"-I{i}" for i in modopts.includes])
-                build.variable("asflags", ["${asflags}"] + [f"-I{i}" for i in modopts.includes])
-
-            if modopts.libs:
-                build.variable("libs", ["${libs}"] + modopts.libs)
-
-            if modopts.ld_script:
-                build.variable("ldflags", ["${ldflags}"] + ["-T", modopts.ld_script])
-
-            header_deps = []
 
                 inputs = []
-            sources = set(self.sources)
+                sources = set(mod.sources)
+                sources.update(set(mod.arch_source.get(self.__arch, tuple())))
                 while sources:
                     source = sources.pop()
                     sources.update(source.next)
@@ -245,45 +359,56 @@ class Module:
                     if source.input:
                         inputs.extend(map(mod_rel, source.outputs))
 
-            build.newline()
+                gather_phony(build, header_deps, target_rel)
 
-            gather_phony(build, header_deps, target_rel, add_headers=True)
+                if mod.kind == "headers":
+                    # Header-only, don't output a build rule
+                    continue
 
-            output = target_rel(self.output)
+                mod_output = target_rel(mod.get_output())
                 build.newline()
                 build.build(
-                rule = self.kind,
-                outputs = output,
+                    rule = mod.kind,
+                    outputs = mod_output,
                     inputs = inputs,
                     implicit = modopts.implicit,
                     order_only = modopts.order_only,
+                    variables = {"name": mod.name,
+                                 "soname": mod.get_output()},
                 )
 
-            dump = output + ".dump"
+                dump = mod_output + ".dump"
                 build.newline()
                 build.build(
                     rule = "dump",
                     outputs = dump,
-                inputs = output,
-                variables = {"name": self.name},
+                    inputs = mod_output,
+                    variables = {"name": mod.name},
                 )
 
-            if self.default:
+                s_output = target_rel(mod.get_output(static=True))
+                if s_output != mod_output:
                     build.newline()
-                build.default(output)
+                    build.build(
+                        rule = mod.kind + "_static",
+                        outputs = s_output,
+                        inputs = inputs,
+                        order_only = modopts.order_only,
+                        variables = {"name": mod.name},
+                    )
+
+                    dump = s_output + ".dump"
+                    build.newline()
+                    build.build(
+                        rule = "dump",
+                        outputs = dump,
+                        inputs = s_output,
+                        variables = {"name": mod.name},
+                    )
+
+                if mod.default:
+                    build.newline()
+                    build.default(mod_output)
                     build.default(dump)
 
-    def add_input(self, path, **kwargs):
-        from .source import Source
-        s = Source(self.root, path, **kwargs)
-        self.sources.append(s)
-        return s.outputs
 
-    def add_depends(self, paths, deps):
-        for source in self.sources:
-            if source.path in paths:
-                source.add_deps(deps)
-
-        for source in self.public_headers:
-            if source.path in paths:
-                source.add_deps(deps)

scripts/bonnibel/project.py

@@ -10,27 +10,26 @@ class Project:
     def __str__(self):
         return f"{self.name} {self.version.major}.{self.version.minor}.{self.version.patch}-{self.version.sha}"
 
-    def generate(self, root, output, modules, config, manifest_file):
+    def generate(self, root, output, modules, config, arch, manifest_file):
         import sys
-        import bonnibel
         from os.path import join
         from ninja.ninja_syntax import Writer
-        from .target import Target
 
-        targets = set()
-        for mod in modules.values():
-            targets.update({Target.load(root, t, config) for t in mod.targets})
+        from .config import generate_configs
+        config_deps = generate_configs(root, output, config, arch, modules.targets, modules.kinds)
 
         with open(output / "build.ninja", "w") as buildfile:
             build = Writer(buildfile)
+            default_builds = []
 
             build.comment("This file is automatically generated by bonnibel")
             build.variable("ninja_required_version", "1.3")
             build.variable("build_root", output)
             build.variable("source_root", root)
+            build.variable("build_config", config)
             build.newline()
 
-            build.include(root / "assets/build/rules.ninja")
+            build.include(root / "assets" / "build" / arch / "rules.ninja")
             build.newline()
 
             build.variable("version_major", self.version.major)
@@ -45,19 +44,15 @@ class Project:
                 ])
             build.newline()
 
-            for target in targets:
-                build.subninja(output / target.name / "target.ninja")
-            build.newline()
-
-            for mod in modules.values():
-                build.subninja(output / f"headers.{mod.name}.ninja")
+            for target in modules.targets:
+                build.subninja(output / target / "target.ninja")
             build.newline()
 
             build.build(
                 rule = "touch",
                 outputs = "${build_root}/.all_headers",
                 implicit = [f"${{build_root}}/include/{m.name}/.headers.phony"
-                    for m in modules.values() if m.public_headers],
+                    for m in modules.used.values() if m.public_headers],
                 )
             build.build(
                 rule = "phony",
@@ -78,56 +73,24 @@ class Project:
             initrdroot = output / "initrd_root"
             initrdroot.mkdir(exist_ok=True)
 
-            fatroot_content = []
-            initrd_content = []
+            image_content = {'initrd_root': [], 'fatroot': []}
 
-            def add_fatroot(source, name):
-                output = join(manifest.location, name)
-                fatroot_output = f"${{build_root}}/fatroot/{output}"
+            def add_image_content(image, path, name):
+                output = join(path, name)
+                image_output = f"${{build_root}}/{image}/{output}"
 
                 build.build(
                     rule = "cp",
-                    outputs = [fatroot_output],
-                    inputs = [source],
-                    variables = {
-                        "description": f"Installing {output}",
-                    })
-
-                fatroot_content.append(fatroot_output)
-                build.newline()
-
-            def add_fatroot_exe(entry):
-                input_path = f"${{build_root}}/{entry.target}/{entry.output}"
-                intermediary = f"${{build_root}}/{entry.output}"
-
-                build.build(
-                    rule = "strip",
-                    outputs = [intermediary],
-                    inputs = [input_path],
-                    implicit = [f"{input_path}.dump"],
-                    variables = {
-                        "name": f"Stripping {entry.module}",
-                        "debug": f"${{build_root}}/.debug/{entry.output}.debug",
-                    })
-
-                add_fatroot(intermediary, entry.output)
-
-            def add_initrd_content(root, name):
-                output = join(root, name)
-                initrd_output = f"${{build_root}}/initrd_root/{output}"
-
-                build.build(
-                    rule = "cp",
-                    outputs = [initrd_output],
+                    outputs = [image_output],
                     inputs = [f"${{build_root}}/{name}"],
                     variables = {
                         "description": f"Installing {name}",
                     })
 
-                initrd_content.append(initrd_output)
+                image_content[image].append(image_output)
                 build.newline()
 
-            def add_initrd_stripped(root, entry):
+            def add_image_content_exe(image, path, entry):
                 input_path = f"${{build_root}}/{entry.target}/{entry.output}"
                 intermediary = f"${{build_root}}/{entry.output}"
 
@@ -141,18 +104,15 @@ class Project:
                         "debug": f"${{build_root}}/.debug/{entry.output}.debug",
                     })
 
-                add_initrd_content(root, entry.output)
+                build.build(
+                    rule = "phony",
+                    outputs = [entry.output],
+                    inputs = [intermediary])
 
-            add_fatroot_exe(manifest.kernel)
-            add_fatroot_exe(manifest.init)
-            for program in manifest.panics:
-                add_fatroot_exe(program)
+                add_image_content(image, path, entry.output)
 
-            for program in manifest.services:
-                add_initrd_stripped("jsix/services", program)
-
-            for program in manifest.drivers:
-                add_initrd_stripped("jsix/drivers", program)
+            if 'kernel' in modules.used:
+                add_image_content_exe("fatroot", manifest.location, manifest.kernel)
 
                 syms = manifest.add_data("symbol_table.dat",
                         "Symbol table", ("symbols",))
@@ -163,11 +123,37 @@ class Project:
                 build.build(
                     rule = "makest",
                     outputs = [syms_out],
-                inputs = [f"${{build_root}}/kernel/{modules['kernel'].output}"],
+                    inputs = [f"${{build_root}}/kernel/{modules['kernel'].get_output(static=True)}"],
                     )
-            fatroot_content.append(syms_out)
+                image_content['fatroot'].append(syms_out)
                 manifest.symbols = syms_file
 
+                build.newline()
+                default_builds.append("${build_root}/jsix.img")
+
+            def try_add_manifest_module(entry, section, image, path="jsix"):
+                if entry.module in modules.used:
+                    add_image_content_exe(image, path, entry)
+                elif entry.module not in modules:
+                    raise BonnibelError(f'unknown {section} module in manifest: {entry.module}')
+
+            try_add_manifest_module(manifest.init, "init", "fatroot")
+
+            for program in manifest.panics:
+                try_add_manifest_module(program, "panic", "fatroot")
+
+            for program in manifest.services:
+                try_add_manifest_module(program, "services", "initrd_root", "jsix/services")
+
+            for program in manifest.drivers:
+                try_add_manifest_module(program, "drivers", "initrd_root", "jsix/drivers")
+
+            for program in manifest.libs:
+                try_add_manifest_module(program, "libs", "initrd_root", "jsix/lib")
+
+            extra_regen_outputs = []
+
+            if 'boot' in modules.used:
                 bootloader = "${build_root}/fatroot/efi/boot/bootx64.efi"
                 build.build(
                     rule = "cp",
@@ -180,24 +166,25 @@ class Project:
 
                 boot_config = join(fatroot, "jsix", "boot.conf")
                 manifest.write_boot_config(boot_config)
+                extra_regen_outputs.append(boot_config)
 
                 initrd = str(fatroot / manifest.location / manifest.initrd["name"])
                 build.build(
                     rule = "makeinitrd",
                     outputs = [initrd],
                     inputs = [str(initrdroot)],
-                implicit = initrd_content + ["${source_root}/scripts/mkj6romfs.py"],
+                    implicit = image_content['initrd_root'] + ["${source_root}/scripts/mkj6romfs.py"],
                     variables = {"format": manifest.initrd["format"]},
                 )
                 build.newline()
 
-            fatroot_content.append(initrd)
+                image_content['fatroot'].append(initrd)
 
                 build.build(
                     rule = "makefat",
                     outputs = ["${build_root}/jsix.img"],
                     inputs = ["${source_root}/assets/diskbase.img"],
-                implicit = fatroot_content + [bootloader],
+                    implicit = image_content['fatroot'] + [bootloader],
                     variables = {"name": "jsix.img"},
                     )
                 build.newline()
@@ -216,8 +203,8 @@ class Project:
                         inputs = [str(p)],
                         variables = {"name": name},
                         )
-                build.default([out])
                     build.newline()
+                    default_builds.append(out)
 
             compdb = "${source_root}/compile_commands.json"
 
@@ -230,36 +217,35 @@ class Project:
 
             regen_implicits = \
                 [f"{self.root}/configure", str(manifest_file)] + \
-                [str(mod.modfile) for mod in modules.values()]
+                [str(mod.modfile) for mod in modules.used.values()]
 
-            for target in targets:
-                regen_implicits += target.depfiles
+            regen_implicits += list(map(str, config_deps))
 
             build.build(
                 rule = "compdb",
                 outputs = [compdb],
                 implicit = regen_implicits,
                 )
-            build.default([compdb])
             build.newline()
+            default_builds.append(compdb)
 
             build.build(
                 rule = "regen",
                 outputs = ['build.ninja'],
                 implicit = regen_implicits,
                 implicit_outputs = 
-                    [f"module.{mod.name}.ninja" for mod in modules.values()] +
-                    [f"{target.name}/target.ninja" for target in targets] +
-                    [boot_config],
+                    [f"module.{mod.name}.ninja" for mod in modules.used.values()] +
+                    [f"{target}/target.ninja" for target in modules.targets] +
+                    extra_regen_outputs,
                 )
 
             build.newline()
-            build.default(["${build_root}/jsix.img"])
+            build.default(default_builds)
 
-        for target in targets:
-            mods = [m.name for m in modules.values() if target.name in m.targets]
+        for target in modules.targets:
+            mods = modules.target_mods(target)
 
-            targetdir = output / target.name
+            targetdir = output / target
             targetdir.mkdir(exist_ok=True)
 
             buildfilename = str(targetdir / "target.ninja")
@@ -268,17 +254,16 @@ class Project:
                 build.comment("This file is automatically generated by bonnibel")
                 build.newline()
 
-                build.variable("target", target.name)
-                build.variable("target_dir", output / target.name)
+                build.variable("target", target)
+                build.variable("target_dir", output / target)
                 build.newline()
 
-                for name, value in target.items():
-                    build.variable(name, value)
+                build.include(f"{target}/config.ninja")
 
                 build.newline()
                 for kind in ('defs', 'run'):
                     for lang in ('c', 'cpp'):
-                        deffile = str(output / target.name / f"{lang}.{kind}")
+                        deffile = str(output / target / f"{lang}.{kind}")
 
                         build.build(
                             rule = f"dump_{lang}_{kind}",
@@ -289,4 +274,4 @@ class Project:
                         build.newline()
 
                 for mod in mods:
-                    build.subninja(f"module.{mod}.ninja")
+                    build.subninja(f"module.{mod.name}.ninja")

scripts/bonnibel/source.py

@@ -72,22 +72,21 @@ class ParseSource(Source):
             variables = dict(name=self.path),
             )
 
-class HeaderSource(Source):
+class CopySource(ParseSource):
     action = "cp"
-    gather = True
+
+    def __init__(self, path, root = "${module_dir}", deps=tuple(), prefix = ""): 
+        self.path = path
+        self.root = root
+        self.deps = deps
+        self.prefix = prefix
 
     @property
-    def outputs(self):
-        return (self.path,)
+    def output(self):
+        from pathlib import Path
+        return Path(self.prefix) / self.path
 
-    @property
-    def args(self):
-        return dict(
-            outputs = [mod_rel(self.path)],
-            inputs = [join(self.root, self.path)],
-            implicit = list(map(_resolve, self.deps)),
-            variables = dict(name=self.path),
-            )
+class HeaderSource(Source): pass
 
 class CompileSource(Source):
     action = property(_dynamic_action("compile"))
@@ -117,3 +116,6 @@ def make_source(root, path):
         return HeaderSource(path, root)
     else:
         raise RuntimeError(f"{path} has no Source type")
+
+def make_copy_source(root, path, prefix = ""):
+    return CopySource(path, root, prefix=prefix)

scripts/bonnibel/target.py

@@ -1,50 +0,0 @@
-class Target(dict):
-    __targets = {}
-
-    @classmethod
-    def load(cls, root, name, config=None):
-        from . import load_config
-
-        if (name, config) in cls.__targets:
-            return cls.__targets[(name, config)]
-
-        configs = root / "assets/build"
-
-        dicts = []
-        depfiles = []
-        basename = name
-        if config:
-            basename += f"-{config}"
-
-        while basename is not None:
-            filename = str(configs / (basename + ".yaml"))
-            depfiles.append(filename)
-            desc = load_config(filename)
-            basename = desc.get("extends")
-            dicts.append(desc.get("variables", dict()))
-
-        t = Target(name, config, depfiles)
-        for d in reversed(dicts):
-            for k, v in d.items():
-                if isinstance(v, (list, tuple)):
-                    t[k] = t.get(k, list()) + list(v)
-                elif isinstance(v, dict):
-                    t[k] = t.get(k, dict())
-                    t[k].update(v)
-                else:
-                    t[k] = v
-
-        cls.__targets[(name, config)] = t
-        return t
-
-    def __init__(self, name, config, depfiles):
-        self.__name = name
-        self.__config = config
-        self.__depfiles = tuple(depfiles)
-
-    def __hash__(self):
-        return hash((self.__name, self.__config))
-
-    name = property(lambda self: self.__name)
-    config = property(lambda self: self.__config)
-    depfiles = property(lambda self: self.__depfiles)

scripts/codegen/__init__.py

@@ -0,0 +1,21 @@
+import cog
+
+supported_architectures = {
+    "amd64": "__amd64__",
+}
+
+def arch_includes(header, root=""):
+    from pathlib import Path
+    root = Path(root)
+    header = Path(header)
+
+    prefix = "if"
+    for arch, define in supported_architectures.items():
+        path = root / "arch" / arch / header
+        cog.outl(f"#{prefix} defined({define})")
+        cog.outl(f"#include <{path}>")
+        prefix = "elif"
+    cog.outl("#else")
+    cog.outl('#error "Unsupported platform"')
+    cog.outl("#endif")
+

scripts/codegen/int_types.py

@@ -1,18 +1,5 @@
 import cog
 
-supported_architectures = {
-    "amd64": "__amd64__",
-}
-
-def arch_includes(header):
-    prefix = "if"
-    for arch, define in supported_architectures.items():
-        cog.outl(f"#{prefix} defined({define})")
-        cog.outl(f"#include <__j6libc/arch/{arch}/{header}>")
-        prefix = "elif"
-    cog.outl("#endif")
-
-
 int_widths = (8, 16, 32, 64)
 int_mods = ("fast", "least")
 

scripts/debug_buddy_alloc.gdb

@@ -0,0 +1,62 @@
+das -enabled off
+
+break heap_allocator.cpp:200
+commands
+silent
+printf "n %016lx %d\n", m_end, current
+continue
+end
+
+break heap_allocator.cpp:206
+commands
+silent
+printf "N %016lx %d\n", m_end, order
+continue
+end
+
+break heap_allocator::register_free_block
+commands
+silent
+printf "F %016lx %d\n", block, order
+continue
+end
+
+break heap_allocator.cpp:118
+commands
+silent
+printf "f %016lx %d\n", block, info->order
+continue
+end
+
+break heap_allocator.cpp:241
+commands
+silent
+printf "S %016lx %d\n", block, order
+continue
+end
+
+break heap_allocator.cpp:158
+commands
+silent
+printf "P %016lx %d\n", block, order
+continue
+end
+
+break heap_allocator.cpp:180
+commands
+silent
+printf "p %016lx %d\n", block, order
+continue
+end
+
+break heap_allocator.cpp:182
+commands
+silent
+printf "M %016lx %016lx %d\n", block, buddy, order
+continue
+end
+
+set logging file buddy_allocs.txt
+set logging overwrite on
+set logging enabled on
+continue

scripts/debug_frame_alloc.gdb

@@ -0,0 +1,18 @@
+das -enabled off
+break frame_allocator.cpp:62
+commands
+silent
+printf "+ %016lx %3d\n", *address, n
+continue
+end
+break frame_allocator.cpp:95
+commands
+silent
+printf "- %016lx %3d\n", address, count
+continue
+end
+
+set logging file frame_allocs.txt
+set logging overwrite on
+set logging enabled on
+continue

scripts/debug_heap_alloc.gdb

@@ -0,0 +1,29 @@
+das -enabled off
+break heap_allocator.cpp:81
+commands
+silent
+printf "+ %016lx %4d\n", block, length
+continue
+end
+break heap_allocator.cpp:86
+commands
+silent
+printf "+ %016lx %4d\n", block, length
+continue
+end
+break heap_allocator.cpp:120
+commands
+silent
+printf "- %016lx\n", p
+continue
+end
+break heap_allocator.cpp:140
+commands
+silent
+printf "> %016lx %4d %4d\n", p, old_length, new_length
+continue
+end
+set logging file heap_allocs.txt
+set logging overwrite on
+set logging enabled on
+continue

scripts/definitions/types/primitive.py

@@ -42,31 +42,37 @@ class PrimitiveRef(Primitive):
     def cxx_names(self, options):
         return self.c_names(options)
 
+_inttypes = {
+    "int": "int",
+    "uint": "unsigned",
+    "size": "size_t",
+    "address": "uintptr_t",
+
+    "int8": "int8_t",
+    "uint8": "uint8_t",
+    "int16": "int16_t",
+    "uint16": "uint16_t",
+    "int32": "int32_t",
+    "uint32": "uint32_t",
+    "int64": "int64_t",
+    "uint64": "uint64_t",
+}
+
 _primitives = {
     "string": PrimitiveRef("string", "char"),
     "buffer": PrimitiveRef("buffer", "void", counted=True),
-
-    "int": Primitive("int", "int"),
-    "uint": Primitive("uint", "unsigned"),
-    "size": Primitive("size", "size_t"),
-    "address": Primitive("address", "uintptr_t"),
-
-    "int8": Primitive("int8", "int8_t"),
-    "uint8": Primitive("uint8", "uint8_t"),
-
-    "int16": Primitive("int16", "int16_t"),
-    "uint16": Primitive("uint16", "uint16_t"),
-
-    "int32": Primitive("int32", "int32_t"),
-    "uint32": Primitive("uint32", "uint32_t"),
-
-    "int64": Primitive("int64", "int64_t"),
-    "uint64": Primitive("uint64", "uint64_t"),
 }
 
 def get_primitive(name):
     p = _primitives.get(name)
-    if not p:
+    if p: return p
+
+    it = _inttypes.get(name.replace('*', ''))
+    if it:
+        if '*' in name:
+            return PrimitiveRef(name, it)
+        else:
+            return Primitive(name, it)
+
     from ..errors import InvalidType
     raise InvalidType(name)
-    return p

scripts/memory.py

@@ -1,6 +1,18 @@
+
+def unit(size):
+    units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB']
+
+    o = 0
+    while size >= 1024 and o < len(units):
+        o += 1
+        size = size >> 10
+
+    return f"{size} {units[o]}"
+
+
 class Layout:
     from collections import namedtuple
-    Region = namedtuple("Region", ("name", "start", "size", "shared"))
+    Region = namedtuple("Region", ("name", "desc", "start", "size", "shared"))
 
     sizes = {'G': 1024 ** 3, 'T': 1024 ** 4}
 
@@ -26,7 +38,7 @@ class Layout:
             for r in data:
                 size = Layout.get_size(r["size"])
                 addr -= size
-                regions.append(Layout.Region(r["name"], addr, size,
+                regions.append(Layout.Region(r["name"], r["desc"], addr, size,
                     r.get("shared", False)))
 
         self.regions = tuple(regions)

scripts/mkj6romfs.py

@@ -104,7 +104,7 @@ def add_dir(path, files, dirs, inode_data, dir_inodes, output, compress):
         compressed = uncompressed
         comp_size = uncomp_size
 
-    output.write(uncompressed)
+    output.write(compressed)
     inode_data.append((inode_type_dir, offset, comp_size, uncomp_size))
 
 

scripts/parse_buddy_allocs.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+
+class block:
+    def __init__(self, start, order, free=False):
+        self.start = start
+        self.order = order
+        self.free = free
+
+    @property
+    def end(self):
+        return self.start + (1<<self.order)
+
+    def overlaps(self, other):
+        return other.start < self.end and other.end > self.start
+
+    def __str__(self):
+        return f"[{self.start:016x} {self.order:2} {self.free and 'free' or 'used'}]"
+
+def get_block(blocks, addr, order, reason):
+    b = blocks.get(addr)
+    if b is None:
+        print(f"ERROR: {reason} unknown block [{addr:016x}, {order}]")
+    elif b.order != order:
+        print(f"ERROR: {reason} block {b} for order {order}")
+    else:
+        return b
+    return None
+ 
+def new_block(blocks, addr, order):
+    b = block(addr, order)
+    for existing in blocks.values():
+        if b.overlaps(existing):
+            print(f"ERROR: new block {b} overlaps existing {existing}")
+    blocks[addr] = b
+
+def free_block(blocks, addr, order, free):
+    s = free and "freeing" or "popping"
+    b = get_block(blocks, addr, order, s)
+    if b and b.free == free:
+        print(f"ERROR: {s} block {b}")
+    elif b:
+        b.free = free
+
+def split_block(blocks, addr, order):
+    b = get_block(blocks, addr, order+1, "splitting")
+    if b:
+        b.order = order
+        buddy = b.start ^ (1<<order)
+        blocks[buddy] = block(buddy, order)
+
+def merge_blocks(blocks, addr1, addr2, order):
+    b1 = get_block(blocks, addr1, order, "merging")
+    b2 = get_block(blocks, addr2, order, "merging")
+    if b1.start > b2.start:
+        b1, b2 = b2, b1
+    del blocks[b2.start]
+    b1.order = order + 1
+
+def parse_line(blocks, line):
+    args = line.strip().split()
+    match args:
+        case ['N', addr, order]:
+            new_block(blocks, int(addr, base=16), int(order))
+        case ['n', addr, order]:
+            new_block(blocks, int(addr, base=16), int(order))
+        case ['P', addr, order]:
+            free_block(blocks, int(addr, base=16), int(order), False)
+        case ['p', addr, order]:
+            free_block(blocks, int(addr, base=16), int(order), False)
+        case ['F', addr, order]:
+            free_block(blocks, int(addr, base=16), int(order), True)
+        case ['S', addr, order]:
+            split_block(blocks, int(addr, base=16), int(order))
+        case ['M', addr1, addr2, order]:
+            merge_blocks(blocks, int(addr1, base=16), int(addr2, base=16), int(order))
+        case _:
+            pass
+
+def parse_file(f):
+    blocks = {}
+    for line in f.readlines():
+        parse_line(blocks, line)
+
+    #for addr in sorted(blocks.keys()):
+    #    print(f"{addr:09x}: {blocks[addr]}")
+
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) > 1:
+        with open(sys.argv[1]) as f:
+            parse_file(f)
+    else:
+        parse_file(sys.stdin)

scripts/parse_frame_allocs.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+def add_maps(allocs, addr, count):
+    for i in range(count):
+        if addr+i in allocs:
+            print(f"ERROR: frame {addr+i:012x} map collision.")
+        else:
+            #print(f"       frame {addr+i:012x} mapped")
+            allocs.add(addr+i)
+
+def remove_maps(allocs, addr, count):
+    for i in range(count):
+        if addr+i not in allocs:
+            print(f" WARN: removing unmapped frame {addr+i:012x}")
+        else:
+            #print(f"       frame {addr+i:012x} unmapped")
+            allocs.remove(addr+i)
+
+def parse_line(allocs, line):
+    args = line.strip().split()
+    match args:
+        case ['+', addr, count]:
+            add_maps(allocs, int(addr, base=16), int(count))
+        case ['-', addr, count]:
+            remove_maps(allocs, int(addr, base=16), int(count))
+        case _:
+            pass
+
+def parse_file(f):
+    allocs = set()
+    for line in f.readlines():
+        parse_line(allocs, line)
+
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) > 1:
+        with open(sys.argv[1]) as f:
+            parse_file(f)
+    else:
+        parse_file(sys.stdin)

scripts/parse_heap_allocs.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+from bisect import bisect_left, insort
+from operator import attrgetter
+
+by_start = attrgetter('start')
+
+class alloc:
+    def __init__(self, start, size):
+        self.start = start
+        self.size = size
+
+    @property
+    def end(self):
+        return self.start + self.size
+
+    def overlaps(self, other):
+        return other.start < self.end and other.end > self.start
+
+    def __str__(self):
+        return f"[{self.start:012x} - {self.end:012x}]"
+
+    def __gt__(self, other):
+        return self.start > other.start
+
+def add_alloc(allocs, addr, length):
+    a = alloc(addr, length)
+    for existing in allocs:
+        if a.overlaps(existing):
+            print(f"ERROR: allocation {a} overlaps existing {existing}")
+    insort(allocs, a)
+
+def remove_alloc(allocs, addr):
+    a = alloc(addr, 0)
+    i = bisect_left(allocs, a)
+    if len(allocs) > i and allocs[i].start == addr:
+        del allocs[i]
+    else:
+        print(f"ERROR: freeing unallocated {a}")
+
+def parse_line(allocs, line):
+    args = line.strip().split()
+    match args:
+        case ['+', addr, length]:
+            add_alloc(allocs, int(addr, base=16), int(length))
+        case ['-', addr]:
+            remove_alloc(allocs, int(addr, base=16))
+        case _:
+            pass
+
+def parse_file(f):
+    allocs = []
+    for line in f.readlines():
+        parse_line(allocs, line)
+
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) > 1:
+        with open(sys.argv[1]) as f:
+            parse_file(f)
+    else:
+        parse_file(sys.stdin)

scripts/print_got.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+
+
+def section_header(name):
+    print()
+    print(name)
+    print("-" * 40)
+
+
+def print_dyn(elf):
+    section = elf.get_section_by_name(".dynamic")
+    if section is None:
+        return
+
+    section_header(".dynamic")
+    for tag in section.iter_tags():
+        print(tag)
+
+
+def print_got(elf, name):
+    import struct
+
+    section = elf.get_section_by_name(name)
+    if section is None:
+        return
+
+    section_header(name)
+    base_ip = section['sh_addr']
+
+    data = section.data()
+    n = section.data_size // 8
+    for i in range(n):
+        addr = struct.unpack_from("Q", data, i*8)[0]
+        print(f"[{i:2x}]: {base_ip+i*8:6x} {addr:16x}")
+
+
+def print_plt(elf):
+    from iced_x86 import Decoder, Formatter, FormatterSyntax
+
+    section_header(".plt")
+
+    section = elf.get_section_by_name(".plt")
+    n = section.data_size // 16
+    data = section.data()
+
+    frm = Formatter(FormatterSyntax.NASM)
+    frm.digit_separator = "'"
+    frm.first_operand_char_index = 8
+    frm.hex_prefix = "0x"
+    frm.hex_suffix = ""
+    frm.leading_zeros = False
+    frm.rip_relative_addresses = False
+    frm.small_hex_numbers_in_decimal = False
+    frm.uppercase_hex = False
+
+    base_ip = section['sh_addr']
+
+    for i in range(n):
+        entry = data[ i*16 : (i+1)*16 ]
+        d = Decoder(64, entry, ip=base_ip + i*16)
+
+        indent = f"[{i:2x}]:"
+        for instr in d:
+            disasm = frm.format(instr)
+            print(f"{indent:6} {instr.ip:6x} {disasm}")
+            indent = ""
+
+        print()
+
+
+def print_gnu_hash(elf):
+    hash_section = elf.get_section_by_name(".gnu.hash")
+    data = hash_section.data()
+
+    import struct
+    (nbuckets, symoff, bloom_sz, bloom_sh) = struct.unpack_from("IIII", data, 0)
+    blooms = struct.unpack_from(f"{bloom_sz}Q", data, 16)
+    buckets = struct.unpack_from(f"{nbuckets}I", data, 16+(bloom_sz*8))
+
+    p = 16 + (bloom_sz*8) + (nbuckets*4)
+    n = (len(data) - p) // 4
+    chains = struct.unpack_from(f"{n}I", data, p)
+
+    section_header(".gnu.hash")
+    print(f" Bucket Count: {nbuckets}")
+    print(f"Symbol Offset: {symoff}")
+    print(f"      Buckets: {buckets}")
+
+    print("\n  Bloom words:")
+    for i in range(len(blooms)):
+        print(f"          [{i:2}]: {blooms[i]:016x}")
+
+    print("\n       Hashes:")
+    for i in range(len(chains)):
+        h = chains[i]
+        end = ""
+        if (h & 1) == 1:
+            end = "END"
+        bloom_idx = (h>>6) % bloom_sz
+        bloom_msk = ((1<<(h%64)) | (1<<((h>>bloom_sh)%64)))
+        print(f"          [{i+symoff:2}]: {h:08x} {end:5} {bloom_idx:2}/{bloom_msk:016x}")
+
+
+def print_tables(filename):
+    from elftools.elf.elffile import ELFFile
+
+    print(filename)
+    print("=" * 50)
+
+    with open(filename, 'rb') as f:
+        elf = ELFFile(f)
+        print_got(elf, ".got")
+        print_got(elf, ".got.plt")
+        print_plt(elf)
+        print_dyn(elf)
+        print_gnu_hash(elf)
+
+    print()
+
+
+if __name__ == "__main__":
+    import sys
+    for filename in sys.argv[1:]:
+        print_tables(filename)

scripts/wsl_set_vnchost.fish

@@ -0,0 +1 @@
+set -gx VNCHOST (ip -j route list default | jq -r '.[0].gateway'):5500

src/boot/allocator.cpp

@@ -71,10 +71,10 @@ allocator::add_modules()
         allocate_pages(1, alloc_type::init_args, true));
 
     if (m_modules)
-        m_modules->next = reinterpret_cast<uintptr_t>(mods);
+        m_modules->next = mods;
 
     m_modules = mods;
-    m_next_mod = mods->modules;
+    m_next_mod = reinterpret_cast<module*>(mods+1);
     return;
 }
 
@@ -109,9 +109,9 @@ allocator::allocate_pages(size_t count, alloc_type type, bool zero)
 }
 
 module *
-allocator::allocate_module()
+allocator::allocate_module(size_t extra)
 {
-    static constexpr size_t size = sizeof(module);
+    size_t size = sizeof(module) + extra;
 
     size_t remaining =
         reinterpret_cast<uintptr_t>(m_modules) + page_size
@@ -120,8 +120,8 @@ allocator::allocate_module()
     if (size > remaining)
         add_modules();
 
-    ++m_modules->count;
     module *m = m_next_mod;
+    m->bytes = size;
     m_next_mod = util::offset_pointer(m_next_mod, size);
     return m;
 }

src/boot/allocator.h

@@ -32,7 +32,7 @@ public:
 
     void * allocate_pages(size_t count, alloc_type type, bool zero = false);
 
-    module * allocate_module();
+    module * allocate_module(size_t extra = 0);
 
     void memset(void *start, size_t size, uint8_t value);
     void copy(void *to, const void *from, size_t size);

src/boot/boot.module

@@ -2,9 +2,11 @@
 
 boot = module("boot",
     kind = "exe",
-    output = "boot.efi",
-    targets = [ "boot" ],
+    outfile = "boot.efi",
+    target = "boot",
     deps = [ "cpu", "elf", "util", "bootproto" ],
+    static = True,
+    skip_arches = [ "linux" ],
     sources = [
         "allocator.cpp",
         "bootconfig.cpp",

src/boot/bootconfig.cpp

@@ -22,7 +22,7 @@ read_string(util::buffer &data)
 static void
 read_descriptor(descriptor &e, util::buffer &data)
 {
-    e.flags = static_cast<desc_flags>(*util::read<uint16_t>(data));
+    e.flags = util::bitset16 {*util::read<uint16_t>(data)};
     e.path = read_string(data);
 }
 

src/boot/bootconfig.h

@@ -3,6 +3,7 @@
 #pragma once
 
 #include <bootproto/bootconfig.h>
+#include <util/bitset.h>
 #include <util/counted.h>
 
 namespace uefi {
@@ -14,7 +15,7 @@ namespace boot {
 using desc_flags = bootproto::desc_flags;
 
 struct descriptor {
-    desc_flags flags;
+    util::bitset16 flags;
     wchar_t const *path;
 };
 
@@ -27,7 +28,7 @@ public:
     /// Constructor. Loads bootconfig from the given buffer.
     bootconfig(util::buffer data, uefi::boot_services *bs);
 
-    inline uint16_t flags() const { return m_flags; }
+    inline util::bitset16 flags() const { return m_flags; }
     inline const descriptor & kernel() const { return m_kernel; }
     inline const descriptor & init() const { return m_init; }
     inline const wchar_t * initrd() const { return m_initrd; }
@@ -35,7 +36,7 @@ public:
     inline const descriptors & panics() { return m_panics; }
 
 private:
-    uint16_t m_flags;
+    util::bitset16 m_flags;
     descriptor m_kernel;
     descriptor m_init;
     descriptors m_panics;

src/boot/hardware.cpp

@@ -79,10 +79,11 @@ check_cpu_supported()
     status_line status {L"Checking CPU features"};
 
     cpu::cpu_id cpu;
-    cpu::cpu_id::features features = cpu.validate();
+    cpu::features features = cpu.features();
     bool supported = true;
 
 #define CPU_FEATURE_OPT(...)
+#define CPU_FEATURE_WRN(...)
 #define CPU_FEATURE_REQ(name, ...) \
     if (!features[cpu::feature::name]) { \
         status::fail(L"CPU required feature " L ## #name, uefi::status::unsupported); \

src/boot/loader.cpp

@@ -110,7 +110,7 @@ parse_program(const wchar_t *name, util::const_buffer data, bootproto::program &
         section.phys_addr = elf.base() + seg.offset;
         section.virt_addr = seg.vaddr;
         section.size = seg.mem_size;
-        section.type = static_cast<bootproto::section_flags>(seg.flags);
+        section.type = seg.flags;
 
         if (seg.mem_size != seg.file_size)
             section.phys_addr = allocate_bss(seg);
@@ -128,8 +128,6 @@ load_program(
     paging::pager &pager,
     bool verify)
 {
-    using util::bits::has;
-
     status_line status(L"Loading program", name);
 
     elf::file elf {data};
@@ -155,8 +153,8 @@ load_program(
 
         pager.map_pages(phys_addr, seg.vaddr,
             memory::bytes_to_pages(seg.mem_size),
-            has(seg.flags, elf::segment_flags::write),
-            has(seg.flags, elf::segment_flags::exec));
+            seg.flags.get(elf::segment_flags::write),
+            seg.flags.get(elf::segment_flags::exec));
     }
 
     return elf.entrypoint();
@@ -167,15 +165,15 @@ load_module(
     fs::file &disk,
     const wchar_t *name,
     const wchar_t *path,
-    bootproto::module_type type,
-    uint16_t subtype)
+    bootproto::module_type type)
 {
     status_line status(L"Loading module", name);
 
-    bootproto::module *mod = g_alloc.allocate_module();
+    bootproto::module *mod = g_alloc.allocate_module(sizeof(util::buffer));
     mod->type = type;
-    mod->subtype = subtype;
-    mod->data = load_file(disk, path);
+
+    util::buffer *data = mod->data<util::buffer>();
+    *data = load_file(disk, path);
 }
 
 } // namespace loader

src/boot/loader.h

@@ -60,14 +60,12 @@ load_program(
 /// \arg name    The human-readable name of the module
 /// \arg path    The path of the file to load the module from
 /// \arg type    The major type to set on the module
-/// \arg subtype The subtype to set on the module
 void
 load_module(
     fs::file &disk,
     const wchar_t *name,
     const wchar_t *path,
-    bootproto::module_type type,
-    uint16_t subtype);
+    bootproto::module_type type);
 
 } // namespace loader
 } // namespace boot

src/boot/main.cpp

@@ -7,6 +7,7 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include <bootproto/acpi.h>
 #include <bootproto/bootconfig.h>
 #include <bootproto/kernel.h>
 #include <bootproto/memory.h>
@@ -81,9 +82,8 @@ load_resources(
     util::buffer kernel = loader::load_file(disk, bc.kernel().path);
     uintptr_t kentry = loader::load_program(kernel, L"jsix kernel", pager, true);
 
-    args->flags = static_cast<bootproto::boot_flags>(bc.flags());
+    args->flags = bc.flags();
 
-    namespace bits = util::bits;
     using bootproto::desc_flags;
 
     bool has_panic = false;
@@ -95,7 +95,7 @@ load_resources(
         // Find the screen-specific panic handler first to
         // give it priority
         for (const descriptor &d : bc.panics()) {
-            if (bits::has(d.flags, desc_flags::graphical)) {
+            if (d.flags.get(desc_flags::graphical)) {
                 panic = loader::load_file(disk, d.path);
                 has_panic = true;
                 break;
@@ -105,7 +105,7 @@ load_resources(
 
     if (!has_panic) {
         for (const descriptor &d : bc.panics()) {
-            if (!bits::has(d.flags, desc_flags::graphical)) {
+            if (!d.flags.get(desc_flags::graphical)) {
                 panic = loader::load_file(disk, d.path);
                 has_panic = true;
                 break;
@@ -125,7 +125,7 @@ load_resources(
     loader::parse_program(L"init server", init, args->init);
 
     loader::load_module(disk, L"initrd", bc.initrd(),
-            bootproto::module_type::initrd, 0);
+            bootproto::module_type::initrd);
 
     return reinterpret_cast<bootproto::entrypoint>(kentry);
 }
@@ -176,9 +176,23 @@ efi_main(uefi::handle image, uefi::system_table *st)
     bootproto::entrypoint kentry =
         load_resources(args, screen, image, pager, bs);
 
+    bootproto::module *acpi_mod =
+        g_alloc.allocate_module(sizeof(bootproto::acpi));
+    acpi_mod->type = bootproto::module_type::acpi;
+    bootproto::acpi *acpi = acpi_mod->data<bootproto::acpi>();
+    acpi->root = args->acpi_table;
+
     pager.update_kernel_args(args);
     memory::efi_mem_map map = uefi_exit(args, image, st->boot_services);
 
+    for (size_t i = 0; i < args->mem_map.count; ++i) {
+        bootproto::mem_entry &e = args->mem_map.pointer[i];
+        if (e.type == bootproto::mem_type::acpi) {
+            acpi->region = util::buffer::from(e.start, e.pages * memory::page_size);
+            break;
+        }
+    }
+
     args->allocations = allocs;
     args->init_modules = reinterpret_cast<uintptr_t>(modules);
 
@@ -194,6 +208,7 @@ efi_main(uefi::handle image, uefi::system_table *st)
 
     change_pointer(args);
     change_pointer(args->pml4);
+    change_pointer(args->symbol_table.pointer);
     change_pointer(args->init.sections.pointer);
 
     //status.next();

src/boot/memory_map.cpp

@@ -249,13 +249,13 @@ build_frame_blocks(const util::counted<bootproto::mem_entry> &kmap)
                 unsigned i = 0;
 
                 uint64_t b1 = (page_count + 4095) / 4096;
-                blk->map1 = (1 << b1) - 1;
+                blk->map1 = (1ull << b1) - 1;
 
                 uint64_t b2 = (page_count + 63) / 64;
                 uint64_t b2q = b2 / 64;
                 uint64_t b2r = b2 % 64;
                 g_alloc.memset(blk->map2, b2q, 0xff);
-                blk->map2[b2q] = (1 << b2r) - 1;
+                blk->map2[b2q] = (1ull << b2r) - 1;
                 break;
             }
         }
@@ -269,7 +269,7 @@ build_frame_blocks(const util::counted<bootproto::mem_entry> &kmap)
         size_t b = blk.count / 64;
         size_t r = blk.count % 64;
         g_alloc.memset(blk.bitmap, b*8, 0xff);
-        blk.bitmap[b] = (1 << r) - 1;
+        blk.bitmap[b] = (1ull << r) - 1;
 
         bitmap += bitmap_size(blk.count);
     }

src/boot/video.cpp

@@ -45,6 +45,7 @@ pick_mode(uefi::boot_services *bs)
     uint32_t res = info->horizontal_resolution * info->vertical_resolution;
     int pixmode = static_cast<int>(info->pixel_format);
 
+    /*
     const uint32_t modes = gop->mode->max_mode;
     for (uint32_t i = 0; i < modes; ++i) {
         size_t size = 0;
@@ -63,6 +64,7 @@ pick_mode(uefi::boot_services *bs)
             pixmode = new_pixmode;
         }
     }
+    */
 
     screen *s = new screen;
     s->mode = {
@@ -109,20 +111,15 @@ make_module(screen *s)
 {
     using bootproto::module;
     using bootproto::module_type;
-    using bootproto::device_type;
-    using bootproto::devices::uefi_fb;
+    namespace devices = bootproto::devices;
 
-    uefi_fb *fb = new uefi_fb;
+    module *mod = g_alloc.allocate_module(sizeof(devices::uefi_fb));
+    mod->type = module_type::device;
+    mod->type_id = devices::type_id_uefi_fb;
+
+    devices::uefi_fb *fb = mod->data<devices::uefi_fb>();
     fb->framebuffer = s->framebuffer;
     fb->mode = s->mode;
-
-    module *mod = g_alloc.allocate_module();
-    mod->type = module_type::device;
-    mod->subtype = static_cast<uint16_t>(device_type::uefi_fb);
-    mod->data = {
-        .pointer = fb,
-        .count = sizeof(uefi_fb),
-    };
 }
 
 } // namespace video

src/kernel/apic.cpp

@@ -1,5 +1,5 @@
 #include "apic.h"
-#include "assert.h"
+#include "kassert.h"
 #include "clock.h"
 #include "interrupts.h"
 #include "io.h"
@@ -71,21 +71,19 @@ lapic::get_id()
 }
 
 void
-lapic::send_ipi(ipi mode, isr vector, uint8_t dest)
+lapic::send_ipi(util::bitset32 mode, isr vector, uint8_t dest)
 {
     // Wait until the APIC is ready to send
     ipi_wait();
 
-    uint32_t command =
-        static_cast<uint32_t>(vector) |
-        static_cast<uint32_t>(mode);
+    uint32_t command = util::bitset32::from(vector) | mode;
 
     apic_write(m_base, lapic_icr_high, static_cast<uint32_t>(dest) << 24);
     apic_write(m_base, lapic_icr_low, command);
 }
 
 void
-lapic::send_ipi_broadcast(ipi mode, bool self, isr vector)
+lapic::send_ipi_broadcast(util::bitset32 mode, bool self, isr vector)
 {
     // Wait until the APIC is ready to send
     ipi_wait();

src/kernel/apic.h

@@ -3,7 +3,7 @@
 /// Classes to control both local and I/O APICs.
 
 #include <stdint.h>
-#include <util/enum_bitfields.h>
+#include <util/bitset.h>
 
 #include "interrupts.h"
 
@@ -33,33 +33,30 @@ public:
     /// Get the local APIC's ID
     uint8_t get_id();
 
-    enum class ipi : uint32_t
+    enum class ipi_flags
     {
-        // Delivery modes
-        fixed    = 0x0000,
-        smi      = 0x0200,
-        nmi      = 0x0400,
-        init     = 0x0500,
-        startup  = 0x0600,
-
-        // Flags
-        deassert = 0x0000,
-        assert   = 0x4000,
-        edge     = 0x0000, ///< edge-triggered
-        level    = 0x8000, ///< level-triggered
+        logical  = 11,
+        pending  = 12,
+        assert   = 14,
+        level    = 15,
     };
 
+    // IPI flags based on delivery mode (bits 8-10)
+    static constexpr util::bitset32 ipi_fixed = 0;
+    static constexpr util::bitset32 ipi_init  = 0x500;
+    static constexpr util::bitset32 ipi_sipi  = 0x600;
+
     /// Send an inter-processor interrupt.
     /// \arg mode   The sending mode
     /// \arg vector The interrupt vector
     /// \arg dest   The APIC ID of the destination
-    void send_ipi(ipi mode, isr vector, uint8_t dest);
+    void send_ipi(util::bitset32 mode, isr vector, uint8_t dest);
 
     /// Send an inter-processor broadcast interrupt to all other CPUs
     /// \arg mode   The sending mode
     /// \arg self   If true, include this CPU in the broadcast
     /// \arg vector The interrupt vector
-    void send_ipi_broadcast(ipi mode, bool self, isr vector);
+    void send_ipi_broadcast(util::bitset32 mode, bool self, isr vector);
 
     /// Wait for an IPI to finish sending. This is done automatically
     /// before sending another IPI with send_ipi().
@@ -145,6 +142,3 @@ private:
     uint8_t m_id;
     uint8_t m_version;
 };
-
-is_bitfield(lapic::ipi);
-

src/kernel/assert.cpp

@@ -1,21 +0,0 @@
-#include "assert.h"
-#include "idt.h"
-
-namespace panic {
-
-uint32_t *apic_icr = reinterpret_cast<uint32_t*>(0xffffc000fee00300);
-void const *symbol_table = nullptr;
-
-void
-install(uintptr_t entrypoint, util::const_buffer symbol_data)
-{
-    IDT::set_nmi_handler(entrypoint);
-    symbol_table = symbol_data.pointer;
-}
-
-} // namespace panic
-
-extern "C"
-void __assert_fail(const char *message, const char *file, unsigned line, const char *function) {
-    panic::panic(message, nullptr, function, file, line);
-}

src/kernel/capabilities.cpp

@@ -68,6 +68,7 @@ cap_table::derive(j6_handle_t base, j6_cap_t caps)
 capability *
 cap_table::find_without_retain(j6_handle_t id)
 {
+    util::scoped_lock lock {m_lock};
     return m_caps.find(id);
 }
 

src/kernel/cpprt.cpp

@@ -1,4 +1,4 @@
-#include "assert.h"
+#include "kassert.h"
 
 using __exit_func = void (*)(void *);
 

src/kernel/cpu.cpp

@@ -1,58 +1,72 @@
-#include <new>
 #include <stdint.h>
-#include <string.h>
+#include <j6/memutils.h>
 #include <util/bitset.h>
+#include <util/no_construct.h>
 
-#include "assert.h"
+#include "kassert.h"
 #include "cpu.h"
-#include "cpu/cpu_id.h"
 #include "device_manager.h"
 #include "gdt.h"
 #include "idt.h"
 #include "logger.h"
 #include "msr.h"
 #include "objects/thread.h"
-#include "scheduler.h"
 #include "syscall.h"
 #include "tss.h"
+#include "xsave.h"
 
 unsigned g_num_cpus = 1;
 
 panic_data g_panic_data;
 panic_data *g_panic_data_p = &g_panic_data;
 
-cpu_data g_bsp_cpu_data;
+static util::no_construct<cpu_data> __g_bsp_cpu_storage;
+cpu_data &g_bsp_cpu_data = __g_bsp_cpu_storage.value;
+
 cpu_data **g_cpu_data = nullptr;
 
 
+static cpu::features
+get_features()
+{
+    cpu::cpu_id cpuid;
+    return cpuid.features();
+}
+
 // Validate the required CPU features are present. Really, the bootloader already
 // validated the required features, but still iterate the options and log about them.
-void
-cpu_validate()
+static cpu::features
+cpu_validate(cpu_data *c)
 {
-    cpu::cpu_id cpu;
+    cpu::cpu_id cpuid;
 
     char brand_name[50];
-    cpu.brand_name(brand_name);
+    cpuid.brand_name(brand_name);
 
-    cpu::cpu_id::features features = cpu.validate();
+    cpu::features &features = c->features;
 
-    log::info(logs::boot, "CPU: %s", brand_name);
-    log::info(logs::boot, "    Vendor is %s", cpu.vendor_id());
+    log::info(logs::boot, "CPU %2d: %s", c->index, brand_name);
+    log::info(logs::boot, "    Vendor is %s", cpuid.vendor_id());
 
-    log::spam(logs::boot, "    Higest basic CPUID: 0x%02x", cpu.highest_basic());
-    log::spam(logs::boot, "    Higest ext CPUID:   0x%02x", cpu.highest_ext() & ~cpu::cpu_id::cpuid_extended);
+    log::spam(logs::boot, "    Higest basic CPUID: 0x%02x", cpuid.highest_basic());
+    log::spam(logs::boot, "    Higest ext CPUID:   0x%02x", cpuid.highest_ext() & ~cpu::cpu_id::cpuid_extended);
 
 #define CPU_FEATURE_OPT(name, ...) \
-    log::verbose(logs::boot, "    Supports %9s: %s", #name, features[cpu::feature::name] ? "yes" : "no");
+    log::verbose(logs::boot, "    Flag %11s: %s", #name, features[cpu::feature::name] ? "yes" : "no");
+
+#define CPU_FEATURE_WRN(name, feat_leaf, feat_sub, regname, bit) \
+    log::verbose(logs::boot, "    Flag %11s: %s", #name, features[cpu::feature::name] ? "yes" : "no"); \
+    if (!features[cpu::feature::name]) log::warn(logs::boot, "Missing cpu feature %s but continuing", #name);
 
 #define CPU_FEATURE_REQ(name, feat_leaf, feat_sub, regname, bit) \
-    log::verbose(logs::boot, "    Supports %9s: %s", #name, features[cpu::feature::name] ? "yes" : "no"); \
+    log::verbose(logs::boot, "    Flag %11s: %s", #name, features[cpu::feature::name] ? "yes" : "no"); \
     kassert(features[cpu::feature::name], "Missing required CPU feature " #name );
 
 #include "cpu/features.inc"
 #undef CPU_FEATURE_OPT
 #undef CPU_FEATURE_REQ
+
+    return features;
 }
 
 
@@ -64,13 +78,31 @@ cpu_early_init(cpu_data *cpu)
     cpu->idt->install();
     cpu->gdt->install();
 
+    util::bitset64 cr0_val = 0;
+    asm ("mov %%cr0, %0" : "=r"(cr0_val));
+    cr0_val
+        .set(cr0::WP)
+        .clear(cr0::CD);
+    asm volatile ( "mov %0, %%cr0" :: "r" (cr0_val) );
+
+    cpu->features = get_features();
+
+    uintptr_t cr3_val;
+    asm ("mov %%cr3, %0" : "=r"(cr3_val));
+
     util::bitset64 cr4_val = 0;
     asm ("mov %%cr4, %0" : "=r"(cr4_val));
     cr4_val
         .set(cr4::OSXFSR)
         .set(cr4::OSXMMEXCPT)
-        .set(cr4::PCIDE)
         .set(cr4::OSXSAVE);
+
+    // TODO: On KVM setting PCIDE generates a #GP even though
+    // the feature is listed as available in CPUID.
+    /*
+    if (cpu->features[cpu::feature::pcid])
+        cr4_val.set(cr4::PCIDE);
+    */
     asm volatile ( "mov %0, %%cr4" :: "r" (cr4_val) );
 
     // Enable SYSCALL and NX bit
@@ -80,6 +112,23 @@ cpu_early_init(cpu_data *cpu)
         .set(efer::NXE);
     wrmsr(msr::ia32_efer, efer_val);
 
+    util::bitset64 xcr0_val = get_xcr0();
+    xcr0_val
+        .set(xcr0::SSE);
+    set_xcr0(xcr0_val);
+
+    // Set initial floating point state
+    const util::bitset32 mxcsr_val = util::bitset32::of(
+        mxcsr::DAZ,
+        mxcsr::IM,
+        mxcsr::DM,
+        mxcsr::ZM,
+        mxcsr::OM,
+        mxcsr::UM,
+        mxcsr::PM,
+        mxcsr::FTZ);
+    asm ( "ldmxcsr %0" :: "m"(mxcsr_val) );
+
     // Install the GS base pointint to the cpu_data
     wrmsr(msr::ia32_gs_base, reinterpret_cast<uintptr_t>(cpu));
 }
@@ -87,7 +136,6 @@ cpu_early_init(cpu_data *cpu)
 cpu_data *
 bsp_early_init()
 {
-    cpu_validate();
     memset(&g_panic_data, 0, sizeof(g_panic_data));
 
     extern IDT &g_bsp_idt;
@@ -104,6 +152,7 @@ bsp_early_init()
     cpu->gdt = new (&g_bsp_gdt) GDT {cpu->tss};
     cpu->rsp0 = reinterpret_cast<uintptr_t>(&idle_stack_end);
     cpu_early_init(cpu);
+    xsave_init();
 
     return cpu;
 }
@@ -120,8 +169,12 @@ bsp_late_init()
     asm ("mov %%cr0, %0" : "=r"(cr0v));
     asm ("mov %%cr4, %0" : "=r"(cr4v));
 
+    uint32_t mxcsrv = get_mxcsr();
+    uint64_t xcr0v = get_xcr0();
+
     uint64_t efer = rdmsr(msr::ia32_efer);
-    log::spam(logs::boot, "Control regs: cr0:%lx cr4:%lx efer:%lx", cr0v, cr4v, efer);
+    log::spam(logs::boot, "Control regs: cr0:%lx cr4:%lx efer:%lx mxcsr:%x xcr0:%x", cr0v, cr4v, efer, mxcsrv, xcr0v);
+    cpu_validate(&g_bsp_cpu_data);
 }
 
 cpu_data *
@@ -162,7 +215,6 @@ cpu_init(cpu_data *cpu, bool bsp)
 
     obj::thread *idle = obj::thread::create_idle_thread(
             g_kernel_process,
-            scheduler::max_priority,
             cpu->rsp0);
 
     cpu->thread = idle;
@@ -190,4 +242,16 @@ cpu_init(cpu_data *cpu, bool bsp)
         cpu->id = apic->get_id();
         apic->calibrate_timer();
     }
+
+    xsave_enable();
+}
+
+/// Set up initial per-thread CPU state. Called once from initialize_user_cpu in
+/// syscall.s, the first code run after a thread comes out of task_switch for the
+/// very first time.
+extern "C" void
+cpu_initialize_thread_state()
+{
+    const util::bitset32 &mxcsr_val = obj::thread::current().m_mxcsr;
+    asm ( "ldmxcsr %0" :: "m"(mxcsr_val) );
 }

src/kernel/cpu.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <stdint.h>
+#include <cpu/cpu_id.h>
 
 class GDT;
 class IDT;
@@ -17,9 +18,12 @@ enum class cr0
 {
     PE =  0,    // Protected mode enable
     MP =  1,    // Monitor co-processor
+    EM =  2,    // (FPU) Emulation
+    TS =  3,    // Task switched
     ET =  4,    // Extension type
     NE =  5,    // Numeric error
     WP = 16,    // (ring 0) Write protect
+    CD = 30,    // Cache disable
     PG = 31,    // Paging
 };
 
@@ -47,6 +51,8 @@ enum class xcr0
     ZMM_Hi256,
     ZMM_Hi16,
     PKRU = 9,
+
+    J6_SUPPORTED = X87 | SSE | AVX | BINDREG | BINDCSR | OPMASK | ZMM_Hi16 | ZMM_Hi256,
 };
 
 enum class efer
@@ -58,6 +64,26 @@ enum class efer
     FFXSR = 14, // Fast FXSAVE
 };
 
+enum class mxcsr
+{
+    IE  =  0, // Invalid operation flag
+    DE  =  1, // Denormal flag
+    ZE  =  2, // Divide by zero flag
+    OE  =  3, // Overflow flag
+    UE  =  4, // Underflow flag
+    PE  =  5, // Precision flag
+    DAZ =  6, // Denormals are zero
+    IM  =  7, // Invalid operation mask
+    DM  =  8, // Denormal mask
+    ZM  =  9, // Divide by zero mask
+    OM  = 10, // Overflow mask
+    UM  = 11, // Underflow mask
+    PM  = 12, // Precision mask
+    RC0 = 13, // Rounding control bit 0
+    RC1 = 14, // Rounding control bit 1
+    FTZ = 15, // Flush to zero
+};
+
 struct cpu_state
 {
     uint64_t r15, r14, r13, r12, r11, r10, r9,  r8;
@@ -105,9 +131,20 @@ struct cpu_data
     // the assembly version
     lapic *apic;
     panic_data *panic;
+    cpu::features features;
 };
 
-extern "C" cpu_data * _current_gsbase();
+extern "C" {
+    uint32_t get_mxcsr();
+    uint32_t set_mxcsr(uint32_t val);
+
+    uint64_t get_xcr0();
+    uint64_t set_xcr0(uint64_t val);
+
+    cpu_data * _current_gsbase();
+
+    void cpu_initialize_thread_state();
+}
 
 /// Do early initialization of the BSP CPU.
 /// \returns  A pointer to the BSP cpu_data structure

src/kernel/cpu.s

@@ -1,3 +1,34 @@
+global get_mxcsr: function hidden (get_mxcsr.end - get_mxcsr)
+get_mxcsr:
+    push 0
+    stmxcsr [rsp]
+    pop rax
+    ret
+.end:
+
+global set_mxcsr: function hidden (set_mxcsr.end - set_mxcsr)
+set_mxcsr:
+    push rdi
+    ldmxcsr [rsp]
+    pop rax
+    ret
+.end:
+
+global get_xcr0: function hidden (get_xcr0.end - get_xcr0)
+get_xcr0:
+    xor rcx, rcx ; there is no dana there is only xcr0
+    xgetbv
+    ret ; technically edx has the high 32 bits, but bits 10+ are reserved
+.end:
+
+global set_xcr0: function hidden (set_xcr0.end - set_xcr0)
+set_xcr0:
+    xor rcx, rcx ; there is no dana there is only xcr0
+    mov rax, rdi ; technically edx should be or'd into the high bits, but xcr0 bits 10+ are resereved
+    xsetbv
+    ret
+.end:
+
 global get_rsp: function hidden (get_rsp.end - get_rsp)
 get_rsp:
 	mov rax, rsp

src/kernel/debugcon.cpp

@@ -0,0 +1,87 @@
+#include <util/format.h>
+
+#include "debugcon.h"
+#include "interrupts.h"
+#include "logger.h"
+
+namespace debugcon {
+
+namespace {
+    static const uint8_t level_colors[] = {0x00, 0x09, 0x01, 0x0b, 0x0f, 0x07, 0x08};
+    const char *level_names[] = {"", "fatal", "error", "warn", "info", "verbose", "spam"};
+    const char *area_names[] = {
+#define LOG(name, lvl) #name ,
+#include <j6/tables/log_areas.inc>
+#undef LOG
+        nullptr
+    };
+
+    inline void debug_out(const char *msg, size_t size) {
+        asm ( "rep outsb;" :: "c"(size), "d"(port), "S"(msg) );
+    }
+
+    inline void debug_endline() {
+        static const char *newline = "\e[0m\r\n";
+        asm ( "rep outsb;" :: "c"(6), "d"(port), "S"(newline) );
+    }
+} // anon namespace
+
+void
+write(const char *fmt, ...)
+{
+    char buffer[256];
+
+    va_list va;
+    va_start(va, fmt);
+    size_t n = util::vformat({buffer, sizeof(buffer)}, fmt, va);
+    va_end(va);
+
+    debug_out(buffer, n);
+    debug_out("\r\n", 2);
+}
+
+void
+output(j6_log_entry *entry)
+{
+    char buffer [256];
+    size_t dlen = util::format({buffer, sizeof(buffer)}, "\e[38;5;%dm%7s %7s\e[38;5;14m\u2502\e[38;5;%dm ",
+            level_colors[entry->severity],
+            area_names[entry->area],
+            level_names[entry->severity],
+            level_colors[entry->severity]);
+
+    debug_out(buffer, dlen);
+
+    debug_out(entry->message, entry->bytes - sizeof(j6_log_entry));
+    debug_endline();
+}
+
+void
+logger_task()
+{
+    using entry = j6_log_entry;
+
+    uint64_t seen = 0;
+    size_t buf_size = 128;
+    uint8_t *buf = new uint8_t [buf_size];
+
+    while (true) {
+        size_t size = g_logger.get_entry(seen, buf, buf_size);
+        if (size > buf_size) {
+            delete [] buf;
+            buf_size *= 2;
+            buf = new uint8_t [buf_size];
+            continue;
+        }
+
+        entry *header = reinterpret_cast<entry*>(buf);
+        while (size > sizeof(entry)) {
+            output(header);
+            seen = header->id;
+            size -= header->bytes;
+            header = util::offset_pointer(header, header->bytes);
+        }
+    }
+}
+
+} // namespace debugcon

src/kernel/debugcon.h

@@ -0,0 +1,31 @@
+#pragma once
+/// \file debugcon.h
+/// Kernel debugcon log output process
+
+#include "scheduler.h"
+
+namespace debugcon {
+
+static constexpr bool enable =
+#ifdef __jsix_config_debug
+    true;
+#else
+    false;
+#endif
+
+static constexpr uint16_t port = 0x6600;
+
+void logger_task();
+void write(const char *fmt, ...);
+
+inline void
+init_logger()
+{
+    if constexpr (enable) {
+        static constexpr uint8_t pri = scheduler::max_priority - 1;
+        scheduler &s = scheduler::get();
+        s.create_kernel_task(logger_task, pri, true);
+    }
+}
+
+} // namespace debugcon

src/kernel/device_manager.cpp

@@ -1,11 +1,10 @@
-#include <new>
 #include <stddef.h>
 #include <stdint.h>
 #include <util/misc.h> // for checksum
 #include <util/pointers.h>
+#include <acpi/tables.h>
 
-#include "assert.h"
-#include "acpi_tables.h"
+#include "kassert.h"
 #include "apic.h"
 #include "clock.h"
 #include "device_manager.h"
@@ -21,37 +20,6 @@ static const char expected_signature[] = "RSD PTR ";
 
 device_manager device_manager::s_instance;
 
-struct acpi1_rsdp
-{
-    char signature[8];
-    uint8_t checksum;
-    char oem_id[6];
-    uint8_t revision;
-    uint32_t rsdt_address;
-} __attribute__ ((packed));
-
-struct acpi2_rsdp
-{
-    char signature[8];
-    uint8_t checksum10;
-    char oem_id[6];
-    uint8_t revision;
-    uint32_t rsdt_address;
-
-    uint32_t length;
-    acpi_table_header *xsdt_address;
-    uint8_t checksum20;
-    uint8_t reserved[3];
-} __attribute__ ((packed));
-
-bool
-acpi_table_header::validate(uint32_t expected_type) const
-{
-    if (util::checksum(this, length) != 0) return false;
-    return !expected_type || (expected_type == type);
-}
-
-
 device_manager::device_manager() :
     m_lapic_base(0)
 {
@@ -63,37 +31,32 @@ device_manager::device_manager() :
     m_irqs[2] = {ignore_event, 0};
 }
 
-template <typename T> static const T *
-check_get_table(const acpi_table_header *header)
-{
-    kassert(header && header->validate(T::type_id), "Invalid ACPI table.");
-    return reinterpret_cast<const T *>(header);
-}
-
 void
 device_manager::parse_acpi(const void *root_table)
 {
     kassert(root_table != 0, "ACPI root table pointer is null.");
 
-    const acpi1_rsdp *acpi1 = mem::to_virtual(
-        reinterpret_cast<const acpi1_rsdp *>(root_table));
+    const acpi::rsdp1 *acpi1 = mem::to_virtual(
+        reinterpret_cast<const acpi::rsdp1 *>(root_table));
 
     for (int i = 0; i < sizeof(acpi1->signature); ++i)
         kassert(acpi1->signature[i] == expected_signature[i],
                 "ACPI RSDP table signature mismatch");
 
-    uint8_t sum = util::checksum(acpi1, sizeof(acpi1_rsdp), 0);
+    uint8_t sum = util::checksum(acpi1, sizeof(acpi::rsdp1), 0);
     kassert(sum == 0, "ACPI 1.0 RSDP checksum mismatch.");
 
     kassert(acpi1->revision > 1, "ACPI 1.0 not supported.");
 
-    const acpi2_rsdp *acpi2 =
-        reinterpret_cast<const acpi2_rsdp *>(acpi1);
+    const acpi::rsdp2 *acpi2 =
+        reinterpret_cast<const acpi::rsdp2 *>(acpi1);
 
-    sum = util::checksum(acpi2, sizeof(acpi2_rsdp), sizeof(acpi1_rsdp));
+    sum = util::checksum(acpi2, sizeof(acpi::rsdp2), sizeof(acpi::rsdp1));
     kassert(sum == 0, "ACPI 2.0 RSDP checksum mismatch.");
 
-    load_xsdt(mem::to_virtual(acpi2->xsdt_address));
+    const acpi::table_header *xsdt = mem::to_virtual(acpi2->xsdt_address);
+    kassert(xsdt->validate(), "Bad XSDT table");
+    load_xsdt(xsdt);
 }
 
 const device_manager::apic_nmi *
@@ -129,53 +92,50 @@ put_sig(char *into, uint32_t type)
 }
 
 void
-device_manager::load_xsdt(const acpi_table_header *header)
+device_manager::load_xsdt(const acpi::table_header *header)
 {
-    const auto *xsdt = check_get_table<acpi_xsdt>(header);
+    const auto *xsdt = acpi::check_get_table<acpi::xsdt>(header);
 
     char sig[5] = {0,0,0,0,0};
     log::info(logs::device, "ACPI 2.0+ tables loading");
 
     put_sig(sig, xsdt->header.type);
-    log::verbose(logs::device, "  Found table %s", sig);
+    log::verbose(logs::device, "  Loading table %s", sig);
 
-    size_t num_tables = acpi_table_entries(xsdt, sizeof(void*));
+    size_t num_tables = acpi::table_entries(xsdt, sizeof(void*));
     for (size_t i = 0; i < num_tables; ++i) {
-        const acpi_table_header *header =
+        const acpi::table_header *header =
             mem::to_virtual(xsdt->headers[i]);
 
-        put_sig(sig, header->type);
-        log::verbose(logs::device, "  Found table %s", sig);
-
         kassert(header->validate(), "Table failed validation.");
+        put_sig(sig, header->type);
 
         switch (header->type) {
-        case acpi_apic::type_id:
+        case acpi::apic::type_id:
+            log::verbose(logs::device, "  Loading table %s", sig);
             load_apic(header);
             break;
 
-        case acpi_mcfg::type_id:
-            load_mcfg(header);
-            break;
-
-        case acpi_hpet::type_id:
+        case acpi::hpet::type_id:
+            log::verbose(logs::device, "  Loading table %s", sig);
             load_hpet(header);
             break;
 
         default:
+            log::verbose(logs::device, "  Skipping table %s", sig);
             break;
         }
     }
 }
 
 void
-device_manager::load_apic(const acpi_table_header *header)
+device_manager::load_apic(const acpi::table_header *header)
 {
-    const auto *apic = check_get_table<acpi_apic>(header);
+    const auto *apic = acpi::check_get_table<acpi::apic>(header);
 
     m_lapic_base = apic->local_address;
 
-    size_t count = acpi_table_entries(apic, 1);
+    size_t count = acpi::table_entries(apic, 1);
     uint8_t const *p = apic->controller_data;
     uint8_t const *end = p + count;
 
@@ -265,33 +225,9 @@ device_manager::load_apic(const acpi_table_header *header)
 }
 
 void
-device_manager::load_mcfg(const acpi_table_header *header)
+device_manager::load_hpet(const acpi::table_header *header)
 {
-    const auto *mcfg = check_get_table<acpi_mcfg>(header);
-
-    size_t count = acpi_table_entries(mcfg, sizeof(acpi_mcfg_entry));
-    m_pci.set_size(count);
-    m_devices.set_capacity(16);
-
-    for (unsigned i = 0; i < count; ++i) {
-        const acpi_mcfg_entry &mcfge = mcfg->entries[i];
-
-        m_pci[i].group = mcfge.group;
-        m_pci[i].bus_start = mcfge.bus_start;
-        m_pci[i].bus_end = mcfge.bus_end;
-        m_pci[i].base = mem::to_virtual<uint32_t>(mcfge.base);
-
-        log::spam(logs::device, "  Found MCFG entry: base %lx  group %d  bus %d-%d",
-                mcfge.base, mcfge.group, mcfge.bus_start, mcfge.bus_end);
-    }
-
-    probe_pci();
-}
-
-void
-device_manager::load_hpet(const acpi_table_header *header)
-{
-    const auto *hpet = check_get_table<acpi_hpet>(header);
+    const auto *hpet = acpi::check_get_table<acpi::hpet>(header);
 
     log::verbose(logs::device, "  Found HPET device #%3d: base %016lx  pmin %d  attr %02x",
             hpet->index, hpet->base_address.address, hpet->periodic_min, hpet->attributes);
@@ -311,28 +247,6 @@ device_manager::load_hpet(const acpi_table_header *header)
         reinterpret_cast<uint64_t*>(hpet->base_address.address + mem::linear_offset));
 }
 
-void
-device_manager::probe_pci()
-{
-    for (auto &pci : m_pci) {
-        log::verbose(logs::device, "Probing PCI group at base %016lx", pci.base);
-
-        for (int bus = pci.bus_start; bus <= pci.bus_end; ++bus) {
-            for (int dev = 0; dev < 32; ++dev) {
-                if (!pci.has_device(bus, dev, 0)) continue;
-
-                auto &d0 = m_devices.emplace(pci, bus, dev, 0);
-                if (!d0.multi()) continue;
-
-                for (int i = 1; i < 8; ++i) {
-                    if (pci.has_device(bus, dev, i))
-                        m_devices.emplace(pci, bus, dev, i);
-                }
-            }
-        }
-    }
-}
-
 static uint64_t
 tsc_clock_source(void*)
 {
@@ -366,7 +280,7 @@ device_manager::init_drivers()
 
         // becomes the singleton
         master_clock = new clock(h.rate(), hpet_clock_source, &h);
-        log::info(logs::clock, "Created master clock using HPET 0: Rate %d", h.rate());
+        log::info(logs::timer, "Created master clock using HPET 0: Rate %d", h.rate());
     } else {
         //TODO: Other clocks, APIC clock?
         master_clock = new clock(5000, tsc_clock_source, nullptr);
@@ -385,7 +299,7 @@ device_manager::dispatch_irq(unsigned irq)
     if (!binding.target || binding.target == ignore_event)
         return binding.target == ignore_event;
 
-    binding.target->signal(1 << binding.signal);
+    binding.target->signal(1ull << binding.signal);
     return true;
 }
 
@@ -410,10 +324,10 @@ device_manager::unbind_irqs(obj::event *target)
     }
 }
 
+/*
 bool
 device_manager::allocate_msi(const char *name, pci_device &device, irq_callback cb, void *data)
 {
-    /*
     // TODO: find gaps to fill
     uint8_t irq = m_irqs.count();
     isr vector = isr::irq00 + irq;
@@ -424,12 +338,6 @@ device_manager::allocate_msi(const char *name, pci_device &device, irq_callback
     device.write_msi_regs(
             0xFEE00000,
             static_cast<uint16_t>(vector));
-    */
     return true;
 }
-
-void
-device_manager::register_block_device(block_device *blockdev)
-{
-    m_blockdevs.append(blockdev);
-}
+*/

src/kernel/device_manager.h

@@ -6,11 +6,13 @@
 
 #include "apic.h"
 #include "hpet.h"
-#include "pci.h"
 
-struct acpi_table_header;
 class block_device;
 
+namespace acpi {
+    struct table_header;
+}
+
 namespace obj {
     class event;
 }
@@ -53,18 +55,6 @@ public:
     /// \arg target The endpoint to remove
     void unbind_irqs(obj::event *target);
 
-    /// Allocate an MSI IRQ for a device
-    /// \arg name    Name of the interrupt, for display to user
-    /// \arg device  Device this MSI is being allocated for
-    /// \arg cb      Callback to call when the interrupt is received
-    /// \arg data    Data to pass to the callback
-    /// \returns     True if an interrupt was allocated successfully
-    bool allocate_msi(
-            const char *name,
-            pci_device &device,
-            irq_callback cb,
-            void *data);
-
     /// Dispatch an IRQ interrupt
     /// \arg irq  The irq number of the interrupt
     /// \returns  True if the interrupt was handled
@@ -103,23 +93,6 @@ public:
     ///           configuration, or null if no configuration was provided
     const irq_override * get_irq_override(uint8_t irq) const;
 
-    /// Register the existance of a block device.
-    /// \arg blockdev  Pointer to the block device
-    void register_block_device(block_device *blockdev);
-
-    /// Get the number of block devices in the system
-    /// \returns  A count of devices
-    inline unsigned get_num_block_devices() const { return m_blockdevs.count(); }
-
-    /// Get a block device
-    /// \arg i    Index of the device to get
-    /// \returns  A pointer to the requested device, or nullptr
-    inline block_device * get_block_device(unsigned i)
-    {
-        return i < m_blockdevs.count() ?
-            m_blockdevs[i] : nullptr;
-    }
-
     /// Get an HPET device
     /// \arg i    Index of the device to get
     /// \returns  A pointer to the requested device, or nullptr
@@ -132,23 +105,15 @@ public:
 private:
     /// Parse the ACPI XSDT and load relevant sub-tables.
     /// \arg xsdt  Pointer to the XSDT from the firmware
-    void load_xsdt(const acpi_table_header *xsdt);
+    void load_xsdt(const acpi::table_header *xsdt);
 
     /// Parse the ACPI MADT and initialize APICs from it.
     /// \arg apic  Pointer to the MADT from the XSDT
-    void load_apic(const acpi_table_header *apic);
-
-    /// Parse the ACPI MCFG and initialize PCIe from it.
-    /// \arg mcfg  Pointer to the MCFG from the XSDT
-    void load_mcfg(const acpi_table_header *mcfg);
+    void load_apic(const acpi::table_header *apic);
 
     /// Parse the ACPI HPET and initialize an HPET from it.
     /// \arg hpet  Pointer to the HPET from the XSDT
-    void load_hpet(const acpi_table_header *hpet);
-
-    /// Probe the PCIe busses and add found devices to our
-    /// device list. The device list is destroyed and rebuilt.
-    void probe_pci();
+    void load_hpet(const acpi::table_header *hpet);
 
     /// Handle a bad IRQ. Called when an interrupt is dispatched
     /// that has no callback.
@@ -162,9 +127,6 @@ private:
     util::vector<apic_nmi> m_nmis;
     util::vector<irq_override> m_overrides;
 
-    util::vector<pci_group> m_pci;
-    util::vector<pci_device> m_devices;
-
     struct irq_binding
     {
         obj::event *target = nullptr;
@@ -172,8 +134,6 @@ private:
     };
     util::vector<irq_binding> m_irqs;
 
-    util::vector<block_device *> m_blockdevs;
-
     static device_manager s_instance;
 
     device_manager(const device_manager &) = delete;