Implement fast syscall/sysret for sytem calls

src/drivers/nulldrv/main.s

@@ -8,17 +8,17 @@ _start:
 	xor rbp, rbp	; Sentinel rbp
 
 	mov rax, 5		; GETPID syscall
-	int 0xee
+	syscall ; int 0xee
 	mov [mypid], rax
 
 	mov rax, 8		; FORK syscall
-	int 0xee
+	syscall ; int 0xee
 	mov [mychild], rax
 
 	mov r12, [mypid]
 	mov r13, [mychild]
 	mov rax, 1		; DEBUG syscall
-	int 0xee
+	syscall ; int 0xee
 
 	cmp r12, 1
 	je .dosend
@@ -32,7 +32,7 @@ _start:
 	mov rax, 1		; MESSAGE syscall
 	;mov rax, 0		; NOOP syscall
 	;syscall
-	int 0xee
+	syscall ; int 0xee
 
 	inc r11
 	cmp r11, 2
@@ -41,7 +41,7 @@ _start:
 
 	mov rax, 4		; SLEEP syscall
 	; syscall
-	int 0xee
+	syscall ; int 0xee
 
 	add rbx, 20
 
@@ -51,11 +51,11 @@ _start:
 .dosend:
 	mov rax, 6      ; SEND syscall
 	mov rdi, 2      ; target is pid 2
-	int 0xee
+	syscall ; int 0xee
 	jmp .preloop
 
 .doreceive:
 	mov rax, 7      ; RECEIVE syscall
 	mov rdi, 1      ; source is pid 2
-	int 0xee
+	syscall ; int 0xee
 	jmp .preloop

src/kernel/debug.cpp

@@ -6,6 +6,9 @@
 
 #define print_reg(name, value) cons->printf("         %s: %016lx\n", name, (value));
 
+size_t __counter_syscall_enter = 0;
+size_t __counter_syscall_sysret = 0;
+
 void
 print_regs(const cpu_state &regs)
 {

src/kernel/debug.h

@@ -12,6 +12,9 @@ extern "C" {
 
 }
 
+extern size_t __counter_syscall_enter;
+extern size_t __counter_syscall_sysret;
+
 void print_regs(const cpu_state &regs);
 void print_stack(const cpu_state &regs);
 void print_stacktrace(int skip = 0);

src/kernel/interrupts.s

@@ -4,38 +4,37 @@ extern isr_handler
 global isr_handler_prelude
 isr_handler_prelude:
 	push_all_and_segments
+	check_swap_gs
 
 	mov rdi, rsp
 	mov rsi, rsp
 	call isr_handler
 	mov rsp, rax
-
+	jmp isr_handler_return
-	pop_all_and_segments
-
-	add rsp, 16		; because the ISRs added err/num
-	sti
-	iretq
 
 extern irq_handler
 global irq_handler_prelude
 irq_handler_prelude:
 	push_all_and_segments
+	check_swap_gs
 
 	mov rdi, rsp
 	mov rsi, rsp
 	call irq_handler
 	mov rsp, rax
+	; fall through to isr_handler_return
 
+global isr_handler_return
+isr_handler_return:
+	check_swap_gs
 	pop_all_and_segments
 
 	add rsp, 16		; because the ISRs added err/num
-	sti
 	iretq
 
 %macro EMIT_ISR 2
 	global %1
 	%1:
-		cli
 		push 0
 		push %2
 		jmp isr_handler_prelude
@@ -44,7 +43,6 @@ irq_handler_prelude:
 %macro EMIT_EISR 2
 	global %1
 	%1:
-		cli
 		push %2
 		jmp isr_handler_prelude
 %endmacro
@@ -52,7 +50,6 @@ irq_handler_prelude:
 %macro EMIT_IRQ 2
 	global %1
 	%1:
-		cli
 		push 0
 		push %2
 		jmp irq_handler_prelude

src/kernel/loader.s

@@ -18,6 +18,15 @@ ramdisk_process_loader:
 	mov rdx, rcx
 	call load_process
 
+	swapgs
+
+	xor rax, rax
+	mov ax, ss
+	mov ds, ax
+	mov es, ax
+	mov fs, ax
+	mov gs, ax
+
 	pop_all_and_segments
 	add rsp, 16		; because the ISRs add err/num
 	iretq

src/kernel/push_all.inc

@@ -44,4 +44,13 @@
 	pop rax
 %endmacro
 
+%macro check_swap_gs 0
+	mov rax, [rsp+0x90]
+	and rax, 0x03 ; mask out the RPL
+	cmp rax, 0x03
+	jne %%noswapgs
+	swapgs
+%%noswapgs:
+%endmacro
+
 ; vim: ft=asm

src/kernel/scheduler.cpp

@@ -1,6 +1,7 @@
 #include "apic.h"
 #include "console.h"
 #include "cpu.h"
+#include "debug.h"
 #include "gdt.h"
 #include "interrupts.h"
 #include "io.h"
@@ -26,6 +27,14 @@ extern "C" {
 	void load_process(const void *image_start, size_t bytes, process *proc, cpu_state state);
 };
 
+struct cpu_data
+{
+	uintptr_t rsp0;
+	uintptr_t rsp3;
+};
+
+static cpu_data bsp_cpu_data;
+
 scheduler::scheduler(lapic *apic) :
 	m_apic(apic),
 	m_next_pid(1)
@@ -181,6 +190,7 @@ void
 scheduler::start()
 {
 	log::info(logs::task, "Starting scheduler.");
+	wrmsr(msr::ia32_gs_base, reinterpret_cast<uintptr_t>(&bsp_cpu_data));
 	m_tick_count = m_apic->enable_timer(isr::isrTimer, quantum_micros, false);
 }
 
@@ -236,7 +246,6 @@ void scheduler::prune(uint64_t now)
 uintptr_t
 scheduler::schedule(uintptr_t rsp0)
 {
-
 	// TODO: lol a real clock
 	static uint64_t now = 0;
 
@@ -262,7 +271,8 @@ scheduler::schedule(uintptr_t rsp0)
 
 	// Set rsp0 to after the end of the about-to-be-popped cpu state
 	tss_set_stack(0, rsp0 + sizeof(cpu_state));
-	wrmsr(msr::ia32_kernel_gs_base, rsp0);
+	bsp_cpu_data.rsp0 = rsp0;
+
 
 	// Swap page tables
 	page_table *pml4 = m_current->pml4;

src/kernel/syscall.cpp

@@ -52,6 +52,8 @@ syscall_dispatch(uintptr_t return_rsp, cpu_state &regs)
 		cons->printf("\nProcess %u: Received DEBUG syscall\n", p->pid);
 		cons->set_color();
 		print_regs(regs);
+		cons->printf("\n         Syscall enters: %8d\n", __counter_syscall_enter);
+		cons->printf("         Syscall sysret: %8d\n", __counter_syscall_sysret);
 		break;
 
 	case syscall::message:

src/kernel/syscall.s

@@ -1,21 +1,44 @@
 %include "push_all.inc"
 
+extern __counter_syscall_enter
+extern __counter_syscall_sysret
+
 extern syscall_handler
+extern isr_handler_return
 global syscall_handler_prelude
 syscall_handler_prelude:
-	push 0		; ss, doesn't matter here
+	swapgs
+	mov [gs:0x08], rsp
+	mov rsp, [gs:0x00]
+
+	push 0x23	; ss
 	push rsp
 	pushf
-	push 0		; cs, doesn't matter here
+	push 0x2b	; cs
 	push rcx	; user rip
 	push 0		; bogus interrupt
 	push 0		; bogus errorcode
 	push_all_and_segments
 
+	inc qword [rel __counter_syscall_enter]
+
+	mov rax, [gs:0x08]
+	mov [rsp + 0x98], rax
+	mov rax, [rsp + 0x70]
+
 	mov rdi, rsp
 	call syscall_handler
 	mov rsp, rax
 
+	mov rax, [rsp + 0x90]
+	and rax, 0x3
+	cmp rax, 0x3
+	jne isr_handler_return
+
+	inc qword [rel __counter_syscall_sysret]
+
+	swapgs
+
 	pop_all_and_segments
 	add rsp, 16	; ignore bogus interrupt / error
 	pop rcx		; user rip