From d759aae3184d6f1d3056ac886a10c069aea24787 Mon Sep 17 00:00:00 2001
From: "Justin C. Miller" <justin@devjustinian.com>
Date: Sun, 13 Mar 2022 18:07:08 -0700
Subject: [PATCH] [kernel] Add new threads to different CPUs

Previously, when adding a new thread, we only ever added it to the
current CPU and relied on work stealing to balance the CPUs. This commit
has the scheduler schedule new tasks round-robin across CPUs in hopes of
having to steal fewer tasks.

Also adds the run_queue.prev pointer for debugging what task was just
running on the given CPU.
---
 src/kernel/scheduler.cpp | 11 +++++++----
 src/kernel/scheduler.h   |  6 ++++--
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/kernel/scheduler.cpp b/src/kernel/scheduler.cpp
index 34f550d..9485521 100644
--- a/src/kernel/scheduler.cpp
+++ b/src/kernel/scheduler.cpp
@@ -29,6 +29,7 @@ scheduler *scheduler::s_instance = nullptr;
 struct run_queue
 {
     tcb_node *current = nullptr;
+    tcb_node *prev = nullptr;
     tcb_list ready[scheduler::num_priorities];
     tcb_list blocked;
 
@@ -38,7 +39,7 @@ struct run_queue
 };
 
 scheduler::scheduler(unsigned cpus) :
-    m_next_pid {1}
+    m_add_index {0}
 {
     kassert(!s_instance, "Created multiple schedulers!");
     if (!s_instance)
@@ -108,11 +109,11 @@ scheduler::start()
 void
 scheduler::add_thread(TCB *t)
 {
-    cpu_data &cpu = current_cpu();
-    run_queue &queue = m_run_queues[cpu.index];
+    cpu_data *cpu = g_cpu_data[m_add_index++ % g_num_cpus];
+    run_queue &queue = m_run_queues[cpu->index];
     util::scoped_lock lock {queue.lock};
 
-    t->cpu = &cpu;
+    t->cpu = cpu;
     t->time_left = quantum(t->priority);
     queue.blocked.push_back(static_cast<tcb_node*>(t));
 }
@@ -232,6 +233,7 @@ scheduler::steal_work(cpu_data &cpu)
             stolen += balance_lists(my_queue.ready[pri], other_queue.ready[pri], cpu);
 
         stolen += balance_lists(my_queue.blocked, other_queue.blocked, cpu);
+        other_queue_lock.release();
 
         if (stolen)
             log::debug(logs::sched, "CPU%02x stole %2d tasks from CPU%02x",
@@ -313,6 +315,7 @@ scheduler::schedule()
         return;
     }
 
+    queue.prev = queue.current;
     thread *next_thread = next->thread;
 
     cpu.thread = next_thread;
diff --git a/src/kernel/scheduler.h b/src/kernel/scheduler.h
index 140dc0d..df07c07 100644
--- a/src/kernel/scheduler.h
+++ b/src/kernel/scheduler.h
@@ -5,6 +5,8 @@
 #include <stdint.h>
 #include <util/vector.h>
 
+extern cpu_data** g_cpu_data;
+
 namespace kernel {
 namespace args {
     struct program;
@@ -86,14 +88,14 @@ public:
 private:
     friend class obj::process;
 
-    static constexpr uint64_t promote_frequency = 10;
+    static constexpr uint64_t promote_frequency = 100;
     static constexpr uint64_t steal_frequency = 10;
 
     void prune(run_queue &queue, uint64_t now);
     void check_promotions(run_queue &queue, uint64_t now);
     void steal_work(cpu_data &cpu);
 
-    uint32_t m_next_pid;
+    uint32_t m_add_index;
     uint32_t m_tick_count;
 
     obj::process *m_kernel_process;