/*-
 * Copyright (c) 1989, 1990 William F. Jolitz.
 * Copyright (c) 1990 The Regents of the University of California.
 * All rights reserved.
 * Copyright (c) 2014-2018 The FreeBSD Foundation
 * All rights reserved.
 *
 * Portions of this software were developed by
 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
 * the FreeBSD Foundation.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	from: vector.s, 386BSD 0.1 unknown origin
 * $FreeBSD: releng/12.0/sys/amd64/amd64/apic_vector.S 331254 2018-03-20 17:58:51Z emaste $
 */

/*
 * Interrupt entry points for external interrupts triggered by I/O APICs
 * as well as IPI handlers.
 */

#include "opt_smp.h"

#include "assym.inc"

#include <machine/asmacros.h>
#include <machine/specialreg.h>
#include <x86/apicreg.h>

#ifdef SMP
#define LK	lock ;
#else
#define LK
#endif

	.text
	SUPERALIGN_TEXT
	/* End Of Interrupt to APIC */
as_lapic_eoi:
	cmpl	$0,x2apic_mode
	jne	1f
	movq	lapic_map,%rax
	movl	$0,LA_EOI(%rax)
	ret
1:
	movl	$MSR_APIC_EOI,%ecx
	xorl	%eax,%eax
	xorl	%edx,%edx
	wrmsr
	ret

/*
 * I/O Interrupt Entry Point.  Rather than having one entry point for
 * each interrupt source, we use one entry point for each 32-bit word
 * in the ISR.  The handler determines the highest bit set in the ISR,
 * translates that into a vector, and passes the vector to the
 * lapic_handle_intr() function.
 */
	.macro	ISR_VEC	index, vec_name
	INTR_HANDLER	\vec_name
	FAKE_MCOUNT(TF_RIP(%rsp))
	cmpl	$0,x2apic_mode
	je	1f
	movl	$(MSR_APIC_ISR0 + \index),%ecx
	rdmsr
	jmp	2f
1:
	movq	lapic_map, %rdx		/* pointer to local APIC */
	movl	LA_ISR + 16 * (\index)(%rdx), %eax	/* load ISR */
2:
	bsrl	%eax, %eax	/* index of highest set bit in ISR */
	jz	3f
	addl	$(32 * \index),%eax
	movq	%rsp, %rsi
	movl	%eax, %edi	/* pass the IRQ */
	call	lapic_handle_intr
3:
	MEXITCOUNT
	jmp	doreti
	.endm

/*
 * Handle "spurious INTerrupts".
 * Notes:
 *  This is different than the "spurious INTerrupt" generated by an
 *   8259 PIC for missing INTs.  See the APIC documentation for details.
 *  This routine should NOT do an 'EOI' cycle.
 */
	.text
	SUPERALIGN_TEXT
IDTVEC(spuriousint)
	/* No EOI cycle used here */
	jmp	doreti_iret

	ISR_VEC	1, apic_isr1
	ISR_VEC	2, apic_isr2
	ISR_VEC	3, apic_isr3
	ISR_VEC	4, apic_isr4
	ISR_VEC	5, apic_isr5
	ISR_VEC	6, apic_isr6
	ISR_VEC	7, apic_isr7

/*
 * Local APIC periodic timer handler.
 */
	INTR_HANDLER	timerint
	FAKE_MCOUNT(TF_RIP(%rsp))
	movq	%rsp, %rdi
	call	lapic_handle_timer
	MEXITCOUNT
	jmp	doreti

/*
 * Local APIC CMCI handler.
 */
	INTR_HANDLER cmcint
	FAKE_MCOUNT(TF_RIP(%rsp))
	call	lapic_handle_cmc
	MEXITCOUNT
	jmp	doreti

/*
 * Local APIC error interrupt handler.
 */
	INTR_HANDLER errorint
	FAKE_MCOUNT(TF_RIP(%rsp))
	call	lapic_handle_error
	MEXITCOUNT
	jmp	doreti

#ifdef XENHVM
/*
 * Xen event channel upcall interrupt handler.
 * Only used when the hypervisor supports direct vector callbacks.
 */
	INTR_HANDLER xen_intr_upcall
	FAKE_MCOUNT(TF_RIP(%rsp))
	movq	%rsp, %rdi
	call	xen_intr_handle_upcall
	MEXITCOUNT
	jmp	doreti
#endif

#ifdef SMP
/*
 * Global address space TLB shootdown.
 */
	.text

	SUPERALIGN_TEXT
invltlb_ret:
	call	as_lapic_eoi
	jmp	ld_regs

	SUPERALIGN_TEXT
	INTR_HANDLER invltlb
	call	invltlb_handler
	jmp	invltlb_ret

	INTR_HANDLER invltlb_pcid
	call	invltlb_pcid_handler
	jmp	invltlb_ret

	INTR_HANDLER invltlb_invpcid_nopti
	call	invltlb_invpcid_handler
	jmp	invltlb_ret

	INTR_HANDLER invltlb_invpcid_pti
	call	invltlb_invpcid_pti_handler
	jmp	invltlb_ret

/*
 * Single page TLB shootdown
 */
	INTR_HANDLER invlpg
	call	invlpg_handler
	jmp	invltlb_ret

	INTR_HANDLER invlpg_invpcid
	call	invlpg_invpcid_handler
	jmp	invltlb_ret

	INTR_HANDLER invlpg_pcid
	call	invlpg_pcid_handler
	jmp	invltlb_ret

/*
 * Page range TLB shootdown.
 */
	INTR_HANDLER invlrng
	call	invlrng_handler
	jmp	invltlb_ret

	INTR_HANDLER invlrng_invpcid
	call	invlrng_invpcid_handler
	jmp	invltlb_ret

	INTR_HANDLER invlrng_pcid
	call	invlrng_pcid_handler
	jmp	invltlb_ret

/*
 * Invalidate cache.
 */
	INTR_HANDLER invlcache
	call	invlcache_handler
	jmp	invltlb_ret

/*
 * Handler for IPIs sent via the per-cpu IPI bitmap.
 */
	INTR_HANDLER ipi_intr_bitmap_handler
	call	as_lapic_eoi
	FAKE_MCOUNT(TF_RIP(%rsp))
	call	ipi_bitmap_handler
	MEXITCOUNT
	jmp	doreti

/*
 * Executed by a CPU when it receives an IPI_STOP from another CPU.
 */
	INTR_HANDLER cpustop
	call	as_lapic_eoi
	call	cpustop_handler
	jmp	doreti

/*
 * Executed by a CPU when it receives an IPI_SUSPEND from another CPU.
 */
	INTR_HANDLER cpususpend
	call	cpususpend_handler
	call	as_lapic_eoi
	jmp	doreti

/*
 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
 *
 * - Calls the generic rendezvous action function.
 */
	INTR_HANDLER rendezvous
#ifdef COUNT_IPIS
	movl	PCPU(CPUID), %eax
	movq	ipi_rendezvous_counts(,%rax,8), %rax
	incq	(%rax)
#endif
	call	smp_rendezvous_action
	call	as_lapic_eoi
	jmp	doreti

/*
 * IPI handler whose purpose is to interrupt the CPU with minimum overhead.
 * This is used by bhyve to force a host cpu executing in guest context to
 * trap into the hypervisor.
 *
 * This handler is different from other IPI handlers in the following aspects:
 *
 * 1. It doesn't push a trapframe on the stack.
 *
 * This implies that a DDB backtrace involving 'justreturn' will skip the
 * function that was interrupted by this handler.
 *
 * 2. It doesn't 'swapgs' when userspace is interrupted.
 *
 * The 'justreturn' handler does not access any pcpu data so it is not an
 * issue. Moreover the 'justreturn' handler can only be interrupted by an NMI
 * whose handler already doesn't trust GS.base when kernel code is interrupted.
 */
	.text
	SUPERALIGN_TEXT
IDTVEC(justreturn)
	pushq	%rax
	pushq	%rcx
	pushq	%rdx
	call	as_lapic_eoi
	popq	%rdx
	popq	%rcx
	popq	%rax
	jmp	doreti_iret

	INTR_HANDLER	justreturn1
	call	as_lapic_eoi
	jmp	doreti

#endif /* SMP */
