[test_runner] Add test_runner program

This change introduces test_runner, which runs unit or integration tests
and then tells the kernel to exit QEMU with a status code indicating the
number of failed tests.

The test_runner program is not loaded by default. Use the test manifest
to enable it:

    ./configure --manifest=assets/manifests/test.yml

A number of tests from the old src/tests have moved over. More to come,
as well as moving code from testapp before getting rid of it.

The test.sh script has been repurposed to be a "headless" version of
qemu.sh for running tests, and it exits with the appropriate exit code.
(Though ./qemu.sh gained the ability to exit with the correct exit code
as well.) Exit codes from kernel panics have been updated so that the
bash scripts should exit with code 127.

src/kernel/panic.serial/main.cpp

@@ -63,8 +63,8 @@ void panic_handler(const cpu_state *regs)
     if (__atomic_sub_fetch(&remaining, 1, order) == 0) {
         // No remaining CPUs, if we're running on QEMU,
         // tell it to exit
-        constexpr uint32_t exit_code = 0;
-        asm ( "out %0, %1" :: "a"(exit_code), "Nd"(0xf4) );
+        constexpr uint32_t exit_code = 255;
+        asm ( "outl %%eax, %%dx" :: "a"(exit_code), "d"(0xf4) );
     }
 
     while (1) asm ("hlt");

src/kernel/syscalls/system.cpp

@@ -40,7 +40,7 @@ noop()
 test_finish(uint32_t exit_code)
 {
     // Tell QEMU to exit
-    asm ( "out %0, %1" :: "a"(exit_code), "Nd"(0xf4) );
+    asm ( "out %0, %1" :: "a"(exit_code+1), "Nd"(0xf4) );
     while (1) asm ("hlt");
 }
 

src/tests/constexpr_hash.cpp

@@ -1,21 +0,0 @@
-#include "kutil/constexpr_hash.h"
-#include "catch.hpp"
-
-using namespace kutil;
-
-TEST_CASE( "constexpr hash", "[hash]" )
-{
-    const unsigned hash1 = static_cast<unsigned>("hash1!"_h);
-    CHECK(hash1 == 210);
-
-    const unsigned hash2 = static_cast<unsigned>("hash1!"_h);
-    CHECK(hash1 == hash2);
-
-    const unsigned hash3 = static_cast<unsigned>("not hash1!"_h);
-    CHECK(hash1 != hash3);
-    CHECK(hash3 == 37);
-
-    const unsigned hash4 = static_cast<unsigned>("another thing that's longer"_h);
-    CHECK(hash1 != hash4);
-    CHECK(hash4 == 212);
-}

src/tests/linked_list.cpp

@@ -1,154 +0,0 @@
-#include <chrono>
-#include <iostream>
-#include <limits>
-#include <random>
-#include <vector>
-#include "kutil/linked_list.h"
-#include "catch.hpp"
-#include "container_helpers.h"
-
-using namespace kutil;
-
-const int test_list_size = 100;
-
-template <typename T>
-class ListVectorCompare :
-    public Catch::MatcherBase<std::vector<list_node<T>>>
-{
-public:
-    using item = list_node<T>;
-    using vector = std::vector<item>;
-
-    ListVectorCompare(const linked_list<T> &list, bool reversed) :
-        m_list(list), m_reverse(reversed) {}
-
-    virtual bool match (vector const& vec) const override
-    {
-        size_t index = m_reverse ? vec.size() - 1 : 0;
-        for (const T *i : m_list) {
-            if (i != &vec[index]) return false;
-            index += m_reverse ? -1 : 1;
-        }
-        return true;
-    }
-
-    virtual std::string describe() const override
-    {
-        return "is the same as the given linked list";
-    }
-
-private:
-    const linked_list<T> &m_list;
-    bool m_reverse;
-};
-
-template <typename T>
-class IsSorted :
-    public Catch::MatcherBase<linked_list<T>>
-{
-public:
-    using item = list_node<T>;
-    using list = linked_list<T>;
-
-    IsSorted() {}
-
-    virtual bool match (list const& l) const override
-    {
-        int big = std::numeric_limits<int>::min();
-        for (const T *i : l) {
-            if (i->value < big) return false;
-            big = i->value;
-        }
-        return true;
-    }
-
-    virtual std::string describe() const override
-    {
-        return "is sorted";
-    }
-};
-
-template <typename T>
-class ListContainsMatcher :
-    public Catch::MatcherBase<linked_list<T>>
-{
-public:
-    using item = list_node<T>;
-    using list = linked_list<T>;
-
-    ListContainsMatcher(const item &needle) : m_needle(needle) {}
-
-    virtual bool match (list const& l) const override
-    {
-        for (const T *i : l)
-            if (i == &m_needle) return true;
-        return false;
-    }
-
-    virtual std::string describe() const override
-    {
-        return "contains the given item";
-    }
-
-    const item &m_needle;
-};
-
-template <typename T>
-ListVectorCompare<T> IsSameAsList(const linked_list<T> &list, bool reversed = false)
-{
-    return ListVectorCompare<T>(list, reversed);
-}
-
-template <typename T>
-ListContainsMatcher<T> ListContains(const list_node<T> &item)
-{
-    return ListContainsMatcher<T>(item);
-}
-
-TEST_CASE( "Linked list tests", "[containers] [list]" )
-{
-    linked_list<unsortableT> ulist;
-
-    int value = 0;
-    std::vector<list_node<unsortableT>> unsortables(test_list_size);
-    for (auto &i : unsortables) {
-        i.value = value++;
-        ulist.push_back(&i);
-    }
-    CHECK( ulist.length() == test_list_size );
-    CHECK_THAT( unsortables, IsSameAsList(ulist) );
-
-    linked_list<unsortableT> ulist_reversed;
-
-    for (auto &i : unsortables) {
-        ulist.remove(&i);
-        ulist_reversed.push_front(&i);
-    }
-
-    CHECK( ulist_reversed.length() == test_list_size );
-    CHECK_THAT( unsortables, IsSameAsList(ulist_reversed, true) );
-
-    auto &removed = unsortables[test_list_size / 2];
-    ulist_reversed.remove(&removed);
-    CHECK( ulist_reversed.length() == test_list_size - 1 );
-    CHECK_THAT( ulist_reversed, !ListContains(removed) );
-}
-
-TEST_CASE( "Sorted list tests", "[containers] [list]" )
-{
-    using clock = std::chrono::system_clock;
-    unsigned seed = clock::now().time_since_epoch().count();
-    std::default_random_engine rng(seed);
-    std::uniform_int_distribution<int> gen(1, 1000);
-
-    linked_list<sortableT> slist;
-    CHECK( slist.length() == 0 );
-
-    std::vector<list_node<sortableT>> sortables(test_list_size);
-    for (auto &i : sortables) {
-        i.value = gen(rng);
-        slist.sorted_insert(&i);
-    }
-    CHECK( slist.length() == test_list_size );
-    CHECK_THAT( slist, IsSorted<sortableT>() );
-}

src/tests/map.cpp

@@ -1,95 +0,0 @@
-#include "kutil/map.h"
-#include "catch.hpp"
-
-using Catch::rng;
-std::uniform_int_distribution<int> distrib {0, 10000};
-
-TEST_CASE( "map insertion", "[containers] [map]" )
-{
-    std::vector<int> ints;
-    for (int i = 0; i < 1000; ++i)
-        ints.push_back(i);
-
-    size_t sizes[] = {1, 2, 3, 5, 100};
-    for (size_t s : sizes) {
-        kutil::map<int, int> v;
-        std::shuffle(ints.begin(), ints.end(), rng());
-
-        for (int i = 0; i < s; ++i) {
-            v.insert(ints[i], ints[i]);
-        }
-
-        for (int i = 0; i < s; ++i) {
-            int *p = v.find(ints[i]);
-            CAPTURE( s );
-            CAPTURE( i );
-            CAPTURE( ints[i] );
-            CAPTURE( kutil::hash(ints[i]) );
-            CHECK( p );
-            CHECK( *p == ints[i] );
-        }
-    }
-}
-
-TEST_CASE( "map deletion", "[containers] [map]" )
-{
-    std::vector<int> ints;
-    for (int i = 0; i < 1000; ++i)
-        ints.push_back(i);
-
-    size_t sizes[] = {1, 2, 3, 5, 100};
-    for (size_t s : sizes) {
-        kutil::map<int, int> v;
-        std::shuffle(ints.begin(), ints.end(), rng());
-
-        for (int i = 0; i < s; ++i) {
-            v.insert(ints[i], ints[i]);
-        }
-
-        for (int i = 0; i < s; i += 2) {
-            v.erase(ints[i]);
-        }
-
-        for (int i = 0; i < s; ++i) {
-            int *p = v.find(ints[i]);
-            CAPTURE( s );
-            CAPTURE( i );
-            CAPTURE( ints[i] );
-            CAPTURE( kutil::hash(ints[i]) );
-            if ( i%2 )
-                CHECK( p );
-            else
-                CHECK( !p );
-        }
-    }
-}
-
-TEST_CASE( "map with pointer vals", "[containers] [map]" )
-{
-    kutil::map<int, int*> v;
-    int is[4] = { 0, 0, 0, 0 };
-    for (int i = 0; i < 4; ++i)
-        v.insert(i*7, &is[i]);
-
-    for (int i = 0; i < 4; ++i) {
-        int *p = v.find(i*7);
-        CHECK( p == &is[i] );
-    }
-
-    CHECK( v.find(3) == nullptr );
-}
-
-TEST_CASE( "map with uint64_t keys", "[containers] [map]" )
-{
-    kutil::map<uint64_t, int> v;
-    int is[4] = { 2, 3, 5, 7 };
-    for (uint64_t i = 0; i < 4; ++i)
-        v.insert(i+1, is[i]);
-
-    for (uint64_t i = 0; i < 4; ++i) {
-        int *p = v.find(i+1);
-        CHECK( *p == is[i] );
-    }
-
-    CHECK( v.find(30) == nullptr );
-}

src/user/test_runner/main.cpp

@@ -0,0 +1,12 @@
+#include <stdlib.h>
+#include <j6/syscalls.h>
+
+#include "test_case.h"
+
+extern "C"
+int main()
+{
+    size_t failures = test::registry::run_all_tests();
+    j6_test_finish(failures); // never actually returns
+    return 0;
+}

src/user/test_runner/test_case.cpp

@@ -0,0 +1,32 @@
+#include "test_case.h"
+
+namespace test {
+
+util::vector<fixture*> registry::m_tests;
+
+void
+fixture::_log_failure(const char *test_name, const char *message,
+            const char *function, const char *file, uint64_t line)
+{
+    // TODO: output results
+    ++_test_failure_count;
+}
+
+void
+registry::register_test_case(fixture &test)
+{
+    m_tests.append(&test);
+}
+
+size_t
+registry::run_all_tests()
+{
+    size_t failures = 0;
+    for (auto *test : m_tests) {
+        test->test_execute();
+        failures += test->_test_failure_count;
+    }
+    return failures;
+}
+
+} // namespace test

src/user/test_runner/test_case.h

@@ -0,0 +1,83 @@
+#pragma once
+/// \file test_case.h
+/// Test case definition and helpers
+
+#include <stddef.h>
+#include <util/vector.h>
+
+namespace test {
+
+class fixture
+{
+public:
+    virtual void setup() {}
+    virtual void teardown() {}
+    virtual void test_execute() = 0;
+
+protected:
+    void _log_failure(
+            const char *test_name,
+            const char *message,
+            const char *function = __builtin_FUNCTION(),
+            const char *file = __builtin_FILE(),
+            uint64_t line = __builtin_LINE());
+
+private:
+    friend class registry;
+    size_t _test_failure_count = 0;
+};
+
+class registry
+{
+public:
+    static void register_test_case(fixture &test);
+    static size_t run_all_tests();
+
+private:
+    static util::vector<fixture*> m_tests;
+};
+
+template <typename T>
+class registrar
+{
+public:
+    registrar() { registry::register_test_case(m_test); }
+
+private:
+    T m_test;
+};
+
+template <typename... Args>
+struct comparator_negate;
+
+template <typename... Args>
+struct comparator
+{
+    virtual bool operator()(const Args&... opts) const = 0;
+    virtual comparator_negate<Args...> operator!() const { return comparator_negate<Args...> {*this}; }
+};
+
+template <typename... Args>
+struct comparator_negate :
+    public comparator<Args...>
+{
+    comparator_negate(const comparator<Args...> &in) : inner {in} {}
+    virtual bool operator()(const Args&... opts) const override { return !inner(opts...); }
+    const comparator<Args...> &inner;
+};
+
+} // namespace test
+
+#define TEST_CASE(fixture, name) namespace {                        \
+    struct test_case_ ## name : fixture {                           \
+        static constexpr const char *test_name = #fixture ":" #name;\
+        void test_execute();                                        \
+    };                                                              \
+    test::registrar<test_case_ ## name> name ## _auto_registrar;    \
+}                                                                   \
+void test_case_ ## name::test_execute()
+
+#define CHECK(expr, message) do { if (!(expr)) {_log_failure(test_name,message);} } while (0)
+#define REQUIRE(expr, message) do { if (!(expr)) {_log_failure(test_name,message); return;} } while (0)
+#define CHECK_BARE(expr) do { if (!(expr)) {_log_failure(test_name,#expr);} } while (0)
+#define CHECK_THAT(subject, checker) do { if (!(checker)(subject)) {_log_failure(test_name,#subject #checker);} } while (0)

src/user/test_runner/test_rng.h

@@ -0,0 +1,30 @@
+#pragma once
+/// \file test_rng.h
+/// Simple xorshift-based psuedorandom number generator for tests
+
+#include <stdint.h>
+
+namespace test {
+
+class rng
+{
+public:
+    using result_type = uint64_t;
+
+    rng(uint64_t seed = 1) : a(seed) {}
+
+    uint64_t operator()() {
+        a ^= a << 13;
+        a ^= a >> 7;
+        a ^= a << 17;
+        return a;
+    }
+
+    constexpr static uint64_t max() { return UINT64_MAX; }
+    constexpr static uint64_t min() { return 0; }
+
+private:
+    uint64_t a;
+};
+
+} // namespace test

src/user/test_runner/test_runner.module

@@ -0,0 +1,15 @@
+# vim: ft=python
+
+module("test_runner",
+    targets = [ "user" ],
+    deps = [ "libc", "util" ],
+    description = "Unit test runner",
+    sources = [
+        "main.cpp",
+        "test_case.cpp",
+
+        "tests/constexpr_hash.cpp",
+        "tests/linked_list.cpp",
+        "tests/map.cpp",
+        "tests/vector.cpp",
+    ])

src/user/test_runner/tests/constexpr_hash.cpp

@@ -0,0 +1,24 @@
+#include <util/constexpr_hash.h>
+#include "test_case.h"
+
+class hash_tests :
+    public test::fixture
+{
+};
+
+TEST_CASE( hash_tests, equality_test )
+{
+    const unsigned hash1 = static_cast<unsigned>("hash1!"_h);
+    CHECK( hash1 == 210, "hash gave unexpected value");
+
+    const unsigned hash2 = static_cast<unsigned>("hash1!"_h);
+    CHECK(hash1 == hash2, "hashes of equal strings should be equal");
+
+    const unsigned hash3 = static_cast<unsigned>("not hash1!"_h);
+    CHECK(hash1 != hash3, "hashes of different strings should not be equal");
+    CHECK(hash3 == 37, "hash gave unexpected value");
+
+    const unsigned hash4 = static_cast<unsigned>("another thing that's longer"_h);
+    CHECK(hash1 != hash4, "hashes of different strings should not be equal");
+    CHECK(hash4 == 212, "hash gave unexpected value");
+}

src/user/test_runner/tests/container_helpers.h

@@ -0,0 +1,13 @@
+#pragma once
+
+struct unsortableT {
+    unsigned long value;
+};
+
+struct sortableT {
+    unsigned long value;
+    unsigned long compare(const sortableT &other) const {
+        return value > other.value ? 1 : value == other.value ? 0 : -1;
+    }
+};
+

src/user/test_runner/tests/linked_list.cpp

@@ -0,0 +1,143 @@
+#include <limits>
+#include <vector>
+
+#include <util/linked_list.h>
+
+#include "container_helpers.h"
+#include "test_case.h"
+#include "test_rng.h"
+
+const int test_list_size = 100;
+
+struct linked_list_tests :
+    public test::fixture
+{
+};
+
+template <typename T>
+class list_vector_comparator :
+    public test::comparator<std::vector<util::list_node<T>> const&>
+{
+public:
+    using item = util::list_node<T>;
+    using vector = std::vector<item>;
+
+    list_vector_comparator(const util::linked_list<T> &list, bool reversed) :
+        m_list(list), m_reverse(reversed) {}
+
+    virtual bool operator()(vector const& vec) const override
+    {
+        size_t index = m_reverse ? vec.size() - 1 : 0;
+        for (const T *i : m_list) {
+            if (i != &vec[index]) return false;
+            index += m_reverse ? -1 : 1;
+        }
+        return true;
+    }
+
+private:
+    const util::linked_list<T> &m_list;
+    bool m_reverse;
+};
+
+template <typename T>
+class is_sorted :
+    public test::comparator<util::linked_list<T> const&>
+{
+public:
+    using item = util::list_node<T>;
+    using list = util::linked_list<T>;
+
+    is_sorted() {}
+
+    virtual bool operator()(list const& l) const override
+    {
+        int big = std::numeric_limits<int>::min();
+        for (const T *i : l) {
+            if (i->value < big) return false;
+            big = i->value;
+        }
+        return true;
+    }
+};
+
+template <typename T>
+class list_contains_comparator :
+    public test::comparator<util::linked_list<T>>
+{
+public:
+    using item = util::list_node<T>;
+    using list = util::linked_list<T>;
+
+    list_contains_comparator(const item &needle) : m_needle(needle) {}
+
+    virtual bool operator()(list const& l) const override
+    {
+        for (const T *i : l)
+            if (i == &m_needle) return true;
+        return false;
+    }
+
+    const item &m_needle;
+};
+
+template <typename T>
+list_vector_comparator<T> same_as(const util::linked_list<T> &list, bool reversed = false)
+{
+    return list_vector_comparator<T>(list, reversed);
+}
+
+template <typename T>
+list_contains_comparator<T> contains(const util::list_node<T> &item)
+{
+    return list_contains_comparator<T>(item);
+}
+
+TEST_CASE( linked_list_tests, unsorted )
+{
+    util::linked_list<unsortableT> ulist;
+
+    int value = 0;
+    std::vector<util::list_node<unsortableT>> unsortables(test_list_size);
+    for (auto &i : unsortables) {
+        i.value = value++;
+        ulist.push_back(&i);
+    }
+    CHECK_BARE( ulist.length() == test_list_size );
+    CHECK_THAT( unsortables, same_as(ulist) );
+
+    util::linked_list<unsortableT> ulist_reversed;
+
+    for (auto &i : unsortables) {
+        ulist.remove(&i);
+        ulist_reversed.push_front(&i);
+    }
+
+    CHECK_BARE( ulist_reversed.length() == test_list_size );
+
+    //CHECK_THAT( unsortables, IsSameAsList(ulist_reversed, true) );
+
+    auto &removed = unsortables[test_list_size / 2];
+    ulist_reversed.remove(&removed);
+    CHECK( ulist_reversed.length() == test_list_size - 1,
+            "remove() did not make size 1 less" );
+
+    CHECK_THAT( ulist_reversed, !contains(removed) );
+}
+
+TEST_CASE( linked_list_tests, sorted )
+{
+    test::rng rng {12345};
+    std::uniform_int_distribution<int> gen(1, 1000);
+
+    util::linked_list<sortableT> slist;
+    CHECK( slist.length() == 0, "Newly constructed list should be empty" );
+
+    std::vector<util::list_node<sortableT>> sortables(test_list_size);
+    for (auto &i : sortables) {
+        i.value = gen(rng);
+        slist.sorted_insert(&i);
+    }
+    CHECK_BARE( slist.length() == test_list_size );
+    // CHECK_THAT( slist, is_sorted<sortableT>() );
+}

src/user/test_runner/tests/map.cpp

@@ -0,0 +1,98 @@
+#include <vector>
+#include <util/map.h>
+
+#include "test_case.h"
+#include "test_rng.h"
+
+struct map_tests :
+    public test::fixture
+{
+};
+
+TEST_CASE( map_tests, insert )
+{
+    test::rng rng {12345};
+
+    std::vector<int> ints;
+    for (int i = 0; i < 1000; ++i)
+        ints.push_back(i);
+
+    size_t sizes[] = {1, 2, 10, 50, 100, 100};
+    for (size_t s : sizes) {
+        util::map<int, int> v;
+        std::shuffle(ints.begin(), ints.end(), rng);
+
+        for (int i = 0; i < s; ++i) {
+            v.insert(ints[i], ints[i]);
+        }
+
+        for (int i = 0; i < s; ++i) {
+            int *p = v.find(ints[i]);
+            CHECK( p, "Map did not have expected key" );
+            CHECK( *p == ints[i], "Map did not have expected value" );
+        }
+    }
+}
+
+TEST_CASE( map_tests, deletion )
+{
+    test::rng rng {12345};
+
+    std::vector<int> ints;
+    for (int i = 0; i < 1000; ++i)
+        ints.push_back(i);
+
+    size_t sizes[] = {1, 2, 3, 5, 100};
+    for (size_t s : sizes) {
+        util::map<int, int> v;
+        std::shuffle(ints.begin(), ints.end(), rng);
+
+        for (int i = 0; i < s; ++i) {
+            v.insert(ints[i], ints[i]);
+        }
+
+        for (int i = 0; i < s; i += 2) {
+            v.erase(ints[i]);
+        }
+
+        for (int i = 0; i < s; ++i) {
+            int *p = v.find(ints[i]);
+            if ( i%2 )
+                CHECK( p, "Expected map item did not exist" );
+            else
+                CHECK( !p, "Deleted item should not exist" );
+        }
+    }
+}
+
+TEST_CASE( map_tests, pointer_vals )
+{
+    util::map<int, int*> v;
+    int is[4] = { 0, 0, 0, 0 };
+    for (int i = 0; i < 4; ++i)
+        v.insert(i*7, &is[i]);
+
+    for (int i = 0; i < 4; ++i) {
+        int *p = v.find(i*7);
+        CHECK( p, "Expected pointer did not exist" );
+        CHECK( p == &is[i], "Expected pointer was not correct" );
+    }
+
+    CHECK( v.find(3) == nullptr, "Expected empty slot exists" );
+}
+
+TEST_CASE( map_tests, uint64_keys )
+{
+    util::map<uint64_t, int> v;
+    int is[4] = { 2, 3, 5, 7 };
+    for (uint64_t i = 0; i < 4; ++i)
+        v.insert(i+1, is[i]);
+
+    for (uint64_t i = 0; i < 4; ++i) {
+        int *p = v.find(i+1);
+        CHECK( p, "Expected integer did not exist" );
+        CHECK( *p == is[i], "Expected integer was not correct" );
+    }
+
+    CHECK( v.find(30) == nullptr, "Expected missing intger was found" );
+}

src/user/test_runner/tests/vector.cpp

@@ -0,0 +1,29 @@
+#include <vector>
+#include <util/vector.h>
+
+#include "container_helpers.h"
+#include "test_case.h"
+#include "test_rng.h"
+
+struct vector_tests :
+    public test::fixture
+{
+};
+
+TEST_CASE( vector_tests, sorted_test )
+{
+    test::rng rng {12345};
+
+    util::vector<sortableT> v;
+
+    int sizes[] = {1, 2, 3, 5, 100};
+    for (int s : sizes) {
+        for (int i = 0; i < s; ++i) {
+            sortableT t { rng() };
+            v.sorted_insert(t);
+        }
+
+        for (int i = 1; i < s; ++i)
+            CHECK( v[i].value >= v[i-1].value, "v is not sorted" );
+    }
+}