Skip site navigation (1)Skip section navigation (2)
Date:      Thu, 26 Feb 2015 11:02:41 +0000 (UTC)
From:      Konstantin Belousov <kib@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r279319 - in head/sys/x86: include x86
Message-ID:  <201502261102.t1QB2flj026595@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: kib
Date: Thu Feb 26 11:02:40 2015
New Revision: 279319
URL: https://svnweb.freebsd.org/changeset/base/279319

Log:
  Implements EOI suppression mode, where LAPIC on EOI command for
  level-triggered interrupt does not broadcast the EOI message to all
  APICs in the system.  Instead, interrupt handler must follow LAPIC EOI
  with IOAPIC EOI.  For modern IOAPICs, the later is done by writing to
  EOIR register.  Otherwise, Intel provided Linux with a trick of
  temporary switching the pin config to edge and then back to level.
  
  Detect presence of EOIR register by reading IO-APIC version.  The
  summary table in the comments was taken from the Linux kernel.  For
  Intel, newer IO-APICs are only briefly documented as part of the
  ICH/PCH datasheet.  According to the BKDG and chipset documentation,
  AMD LAPICs do not provide EOI suppression, althought IO-APICs do
  declare version 0x21 and implement EOIR.
  
  The trick to temporary switch pin to edge mode to clear IRR was tested
  on modern chipset, by pretending that EOIR is not present, i.e. by
  forcing io_haseoi to zero.
  
  Tunable hw.lapic_eoi_suppression disables the optimization.
  
  Reviewed by:	neel
  Tested by:	pho
  Review:	https://reviews.freebsd.org/D1943
  Sponsored by:	The FreeBSD Foundation
  MFC after:	2 months

Modified:
  head/sys/x86/include/apicvar.h
  head/sys/x86/x86/io_apic.c
  head/sys/x86/x86/local_apic.c

Modified: head/sys/x86/include/apicvar.h
==============================================================================
--- head/sys/x86/include/apicvar.h	Thu Feb 26 09:42:03 2015	(r279318)
+++ head/sys/x86/include/apicvar.h	Thu Feb 26 11:02:40 2015	(r279319)
@@ -426,6 +426,7 @@ void	lapic_handle_timer(struct trapframe
 void	xen_intr_handle_upcall(struct trapframe *frame);
 
 extern int x2apic_mode;
+extern int lapic_eoi_suppression;
 
 #ifdef _SYS_SYSCTL_H_
 SYSCTL_DECL(_hw_apic);

Modified: head/sys/x86/x86/io_apic.c
==============================================================================
--- head/sys/x86/x86/io_apic.c	Thu Feb 26 09:42:03 2015	(r279318)
+++ head/sys/x86/x86/io_apic.c	Thu Feb 26 11:02:40 2015	(r279319)
@@ -97,6 +97,7 @@ struct ioapic {
 	u_int io_apic_id:4;
 	u_int io_intbase:8;		/* System Interrupt base */
 	u_int io_numintr:8;
+	u_int io_haseoi:1;
 	volatile ioapic_t *io_addr;	/* XXX: should use bus_space */
 	vm_paddr_t io_paddr;
 	STAILQ_ENTRY(ioapic) io_next;
@@ -134,10 +135,53 @@ static int enable_extint;
 SYSCTL_INT(_hw_apic, OID_AUTO, enable_extint, CTLFLAG_RDTUN, &enable_extint, 0,
     "Enable the ExtINT pin in the first I/O APIC");
 
-static __inline void
-_ioapic_eoi_source(struct intsrc *isrc)
+static void
+_ioapic_eoi_source(struct intsrc *isrc, int locked)
 {
+	struct ioapic_intsrc *src;
+	struct ioapic *io;
+	volatile uint32_t *apic_eoi;
+	uint32_t low1;
+
 	lapic_eoi();
+	if (!lapic_eoi_suppression)
+		return;
+	src = (struct ioapic_intsrc *)isrc;
+	if (src->io_edgetrigger)
+		return;
+	io = (struct ioapic *)isrc->is_pic;
+
+	/*
+	 * Handle targeted EOI for level-triggered pins, if broadcast
+	 * EOI suppression is supported by LAPICs.
+	 */
+	if (io->io_haseoi) {
+		/*
+		 * If IOAPIC has EOI Register, simply write vector
+		 * number into the reg.
+		 */
+		apic_eoi = (volatile uint32_t *)((volatile char *)
+		    io->io_addr + IOAPIC_EOIR);
+		*apic_eoi = src->io_vector;
+	} else {
+		/*
+		 * Otherwise, if IO-APIC is too old to provide EOIR,
+		 * do what Intel did for the Linux kernel. Temporary
+		 * switch the pin to edge-trigger and back, masking
+		 * the pin during the trick.
+		 */
+		if (!locked)
+			mtx_lock_spin(&icu_lock);
+		low1 = src->io_lowreg;
+		low1 &= ~IOART_TRGRLVL;
+		low1 |= IOART_TRGREDG | IOART_INTMSET;
+		ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(src->io_intpin),
+		    low1);
+		ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(src->io_intpin),
+		    src->io_lowreg);
+		if (!locked)
+			mtx_unlock_spin(&icu_lock);
+	}
 }
 
 static u_int
@@ -230,7 +274,7 @@ ioapic_disable_source(struct intsrc *isr
 	}
 
 	if (eoi == PIC_EOI)
-		_ioapic_eoi_source(isrc);
+		_ioapic_eoi_source(isrc, 1);
 
 	mtx_unlock_spin(&icu_lock);
 }
@@ -239,7 +283,7 @@ static void
 ioapic_eoi_source(struct intsrc *isrc)
 {
 
-	_ioapic_eoi_source(isrc);
+	_ioapic_eoi_source(isrc, 0);
 }
 
 /*
@@ -545,6 +589,22 @@ ioapic_create(vm_paddr_t addr, int32_t a
 	io->io_addr = apic;
 	io->io_paddr = addr;
 
+	if (bootverbose) {
+		printf("ioapic%u: ver 0x%02x maxredir 0x%02x\n", io->io_id,
+		    (value & IOART_VER_VERSION), (value & IOART_VER_MAXREDIR)
+		    >> MAXREDIRSHIFT);
+	}
+	/*
+	 * The  summary information about IO-APIC versions is taken from
+	 * the Linux kernel source:
+	 *     0Xh     82489DX
+	 *     1Xh     I/OAPIC or I/O(x)APIC which are not PCI 2.2 Compliant
+	 *     2Xh     I/O(x)APIC which is PCI 2.2 Compliant
+	 *     30h-FFh Reserved
+	 * IO-APICs with version >= 0x20 have working EOIR register.
+	 */
+	io->io_haseoi = (value & IOART_VER_VERSION) >= 0x20;
+
 	/*
 	 * Initialize pins.  Start off with interrupts disabled.  Default
 	 * to active-hi and edge-triggered for ISA interrupts and active-lo

Modified: head/sys/x86/x86/local_apic.c
==============================================================================
--- head/sys/x86/x86/local_apic.c	Thu Feb 26 09:42:03 2015	(r279318)
+++ head/sys/x86/x86/local_apic.c	Thu Feb 26 11:02:40 2015	(r279319)
@@ -159,11 +159,14 @@ extern inthand_t IDTVEC(rsvd);
 volatile char *lapic_map;
 vm_paddr_t lapic_paddr;
 int x2apic_mode;
+int lapic_eoi_suppression;
 static u_long lapic_timer_divisor;
 static struct eventtimer lapic_et;
 
 SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD, 0, "APIC options");
 SYSCTL_INT(_hw_apic, OID_AUTO, x2apic_mode, CTLFLAG_RD, &x2apic_mode, 0, "");
+SYSCTL_INT(_hw_apic, OID_AUTO, eoi_suppression, CTLFLAG_RD,
+    &lapic_eoi_suppression, 0, "");
 
 static uint32_t
 lapic_read32(enum LAPIC_REGISTERS reg)
@@ -380,6 +383,7 @@ lvt_mode(struct lapic *la, u_int pin, ui
 static void
 native_lapic_init(vm_paddr_t addr)
 {
+	uint32_t ver;
 	u_int regs[4];
 	int i, arat;
 
@@ -443,6 +447,20 @@ native_lapic_init(vm_paddr_t addr)
 		lapic_et.et_priv = NULL;
 		et_register(&lapic_et);
 	}
+
+	/*
+	 * Set lapic_eoi_suppression after lapic_enable(), to not
+	 * enable suppression in the hardware prematurely.  Note that
+	 * we by default enable suppression even when system only has
+	 * one IO-APIC, since EOI is broadcasted to all APIC agents,
+	 * including CPUs, otherwise.
+	 */
+	ver = lapic_read32(LAPIC_VERSION);
+	if ((ver & APIC_VER_EOI_SUPPRESSION) != 0) {
+		lapic_eoi_suppression = 1;
+		TUNABLE_INT_FETCH("hw.lapic_eoi_suppression",
+		    &lapic_eoi_suppression);
+	}
 }
 
 /*
@@ -755,6 +773,8 @@ lapic_enable(void)
 	value = lapic_read32(LAPIC_SVR);
 	value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS);
 	value |= APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT;
+	if (lapic_eoi_suppression)
+		value |= APIC_SVR_EOI_SUPPRESSION;
 	lapic_write32(LAPIC_SVR, value);
 }
 
@@ -1562,8 +1582,10 @@ apic_setup_io(void *dummy __unused)
 	return;
 #endif
 	/*
-	 * Finish setting up the local APIC on the BSP once we know how to
-	 * properly program the LINT pins.
+	 * Finish setting up the local APIC on the BSP once we know
+	 * how to properly program the LINT pins.  In particular, this
+	 * enables the EOI suppression mode, if LAPIC support it and
+	 * user did not disabled the mode.
 	 */
 	lapic_setup(1);
 	if (bootverbose)



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201502261102.t1QB2flj026595>