[kernel] Make IDT per-cpu, not global

Since we modify IST entries while handling interrupts, the IDT cannot be
a global data structure. Allocate new ones for each CPU.

src/kernel/cpu.cpp

@@ -41,7 +41,7 @@ cpu_validate()
 void
 cpu_early_init(cpu_data *cpu)
 {
-	IDT::get().install();
+	cpu->idt->install();
 	cpu->gdt->install();
 
 	// Install the GS base pointint to the cpu_data
@@ -63,4 +63,6 @@ cpu_init(cpu_data *cpu, bool bsp)
 	uint64_t pat = rdmsr(msr::ia32_pat);
 	pat = (pat & 0x00ffffffffffffffull) | (0x01ull << 56); // set PAT 7 to WC
 	wrmsr(msr::ia32_pat, pat);
+
+	cpu->idt->add_ist_entries();
 }

src/kernel/cpu.h

@@ -3,6 +3,7 @@
 #include <stdint.h>
 
 class GDT;
+class IDT;
 class lapic;
 class process;
 struct TCB;
@@ -30,6 +31,7 @@ struct cpu_data
 	TCB *tcb;
 	thread *thread;
 	process *process;
+	IDT *idt;
 	TSS *tss;
 	GDT *gdt;
 

src/kernel/idt.cpp

@@ -1,5 +1,6 @@
 #include "kutil/memory.h"
 #include "kutil/no_construct.h"
+#include "cpu.h"
 #include "idt.h"
 #include "log.h"
 
@@ -15,11 +16,11 @@ extern "C" {
 #undef ISR
 }
 
-// The IDT is initialized _before_ global constructors are called,
+// The BSP's IDT is initialized _before_ global constructors are called,
 // so we don't want it to have a global constructor, lest it overwrite
 // the previous initialization.
-static kutil::no_construct<IDT> __g_idt_storage;
-IDT &g_idt = __g_idt_storage.value;
+static kutil::no_construct<IDT> __g_bsp_idt_storage;
+IDT &g_bsp_idt = __g_bsp_idt_storage.value;
 
 
 IDT::IDT()
@@ -38,9 +39,10 @@ IDT::IDT()
 }
 
 IDT &
-IDT::get()
+IDT::current()
 {
-	return g_idt;
+	cpu_data &cpu = current_cpu();
+	return *cpu.idt;
 }
 
 void

src/kernel/idt.h

@@ -8,6 +8,9 @@ class IDT
 public:
 	IDT();
 
+	/// Get the currently running CPU's IDT
+	static IDT & current();
+
 	/// Install this IDT to the current CPU
 	void install() const;
 
@@ -35,9 +38,6 @@ public:
 	/// \arg index  Which entry to print, or -1 for all entries
 	void dump(unsigned index = -1) const;
 
-	/// Get the global IDT
-	static IDT & get();
-
 private:
 	void set(uint8_t i, void (*handler)(), uint16_t selector, uint8_t flags);
 

src/kernel/interrupts.cpp

@@ -85,9 +85,10 @@ isr_handler(cpu_state *regs)
 
 	// Clear out the IST for this vector so we just keep using
 	// this stack
-	uint8_t old_ist = IDT::get().get_ist(vector);
+	IDT &idt = IDT::current();
+	uint8_t old_ist = idt.get_ist(vector);
 	if (old_ist)
-		IDT::get().set_ist(vector, 0);
+		idt.set_ist(vector, 0);
 
 	switch (static_cast<isr>(vector)) {
 
@@ -152,7 +153,7 @@ isr_handler(cpu_state *regs)
 				case 1:
 				case 3:
 					cons->printf(" IDT[%x]\n", index);
-					IDT::get().dump(index);
+					IDT::current().dump(index);
 					break;
 
 				default:
@@ -274,7 +275,7 @@ isr_handler(cpu_state *regs)
 
 	// Return the IST for this vector to what it was
 	if (old_ist)
-		IDT::get().set_ist(vector, old_ist);
+		idt.set_ist(vector, old_ist);
 	*reinterpret_cast<uint32_t *>(apic_eoi_addr) = 0;
 }
 

src/kernel/main.cpp

@@ -120,18 +120,18 @@ kernel_main(args::header *header)
 		logger_clear_immediate();
 	}
 
-	extern IDT &g_idt;
+	extern IDT &g_bsp_idt;
 	extern TSS &g_bsp_tss;
 	extern GDT &g_bsp_gdt;
 	extern cpu_data g_bsp_cpu_data;
 	extern uintptr_t idle_stack_end;
 
-	IDT *idt = new (&g_idt) IDT;
 
 	cpu_data *cpu = &g_bsp_cpu_data;
 	kutil::memset(cpu, 0, sizeof(cpu_data));
 
 	cpu->self = cpu;
+	cpu->idt = new (&g_bsp_idt) IDT;
 	cpu->tss = new (&g_bsp_tss) TSS;
 	cpu->gdt = new (&g_bsp_gdt) GDT {cpu->tss};
 	cpu->rsp0 = idle_stack_end;
@@ -143,7 +143,7 @@ kernel_main(args::header *header)
 	run_constructors();
 	memory_initialize_post_ctors(*header);
 
-	cpu->tss->create_ist_stacks(idt->used_ist_entries());
+	cpu->tss->create_ist_stacks(cpu->idt->used_ist_entries());
 
 	for (size_t i = 0; i < header->num_modules; ++i) {
 		args::module &mod = header->modules[i];
@@ -180,7 +180,6 @@ kernel_main(args::header *header)
 	const auto &apic_ids = devices.get_apic_ids();
 	unsigned num_cpus = start_aps(*apic, apic_ids, header->pml4);
 
-	idt->add_ist_entries();
 	interrupts_enable();
 
 	/*
@@ -258,7 +257,7 @@ start_aps(lapic &apic, const kutil::vector<uint8_t> &ids, void *kpml4)
 	static constexpr size_t full_stack_bytes = kernel_stack_pages * frame_size;
 	static constexpr size_t idle_stacks_per = full_stack_bytes / idle_stack_bytes;
 
-	uint8_t ist_entries = IDT::get().used_ist_entries();
+	uint8_t ist_entries = IDT::current().used_ist_entries();
 
 	size_t free_stack_count = 0;
 	uintptr_t stack_area_start = 0;
@@ -270,6 +269,7 @@ start_aps(lapic &apic, const kutil::vector<uint8_t> &ids, void *kpml4)
 		if (id == bsp.id) continue;
 
 		// Set up the CPU data structures
+		IDT *idt = new IDT;
 		TSS *tss = new TSS;
 		GDT *gdt = new GDT {tss};
 		cpu_data *cpu = new cpu_data;
@@ -278,8 +278,9 @@ start_aps(lapic &apic, const kutil::vector<uint8_t> &ids, void *kpml4)
 		cpu->self = cpu;
 		cpu->id = id;
 		cpu->index = ++index;
-		cpu->gdt = gdt;
+		cpu->idt = idt;
 		cpu->tss = tss;
+		cpu->gdt = gdt;
 
 		tss->create_ist_stacks(ist_entries);
 

src/kernel/tasking.inc

@@ -15,6 +15,7 @@ struc CPU_DATA
 .tcb:          resq 1
 .thread:       resq 1
 .process:      resq 1
+.idt:          resq 1
 .tss:          resq 1
 .gdt:          resq 1
 endstruc