Skip site navigation (1)Skip section navigation (2)
Date:      Fri, 3 Jan 2014 15:53:52 -0500
From:      Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
To:        Roger Pau Monne <roger.pau@citrix.com>
Cc:        xen-devel@lists.xen.org, julien.grall@citrix.com, freebsd-xen@freebsd.org, freebsd-current@freebsd.org, kib@freebsd.org, gibbs@freebsd.org
Subject:   Re: [Xen-devel] [PATCH v7 12/19] xen: add a hook to perform AP startup
Message-ID:  <20140103205352.GB2732@phenom.dumpdata.com>
In-Reply-To: <1387479296-33389-13-git-send-email-roger.pau@citrix.com>
References:  <1387479296-33389-1-git-send-email-roger.pau@citrix.com> <1387479296-33389-13-git-send-email-roger.pau@citrix.com>

next in thread | previous in thread | raw e-mail | index | archive | help
On Thu, Dec 19, 2013 at 07:54:49PM +0100, Roger Pau Monne wrote:
> AP startup on PVH follows the PV method, so we need to add a hook in
> order to diverge from bare metal.
> ---
>  sys/amd64/amd64/mp_machdep.c |   16 ++++---
>  sys/amd64/include/cpu.h      |    1 +
>  sys/x86/xen/hvm.c            |   17 +++++++-
>  sys/x86/xen/pv.c             |   90 ++++++++++++++++++++++++++++++++++++++++++
>  sys/xen/pv.h                 |    1 +
>  5 files changed, 117 insertions(+), 8 deletions(-)
> 
> diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
> index 4ef4b3d..e302886 100644
> --- a/sys/amd64/amd64/mp_machdep.c
> +++ b/sys/amd64/amd64/mp_machdep.c
> @@ -90,7 +90,7 @@ extern  struct pcpu __pcpu[];
>  
>  /* AP uses this during bootstrap.  Do not staticize.  */
>  char *bootSTK;
> -static int bootAP;
> +int bootAP;
>  
>  /* Free these after use */
>  void *bootstacks[MAXCPU];
> @@ -122,9 +122,12 @@ u_long *ipi_rendezvous_counts[MAXCPU];
>  static u_long *ipi_hardclock_counts[MAXCPU];
>  #endif
>  
> +int native_start_all_aps(void);
> +
>  /* Default cpu_ops implementation. */
>  struct cpu_ops cpu_ops = {
> -	.ipi_vectored = lapic_ipi_vectored
> +	.ipi_vectored = lapic_ipi_vectored,
> +	.start_all_aps = native_start_all_aps,
>  };
>  
>  extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32);
> @@ -138,7 +141,7 @@ extern int pmap_pcid_enabled;
>  static volatile cpuset_t ipi_nmi_pending;
>  
>  /* used to hold the AP's until we are ready to release them */
> -static struct mtx ap_boot_mtx;
> +struct mtx ap_boot_mtx;
>  
>  /* Set to 1 once we're ready to let the APs out of the pen. */
>  static volatile int aps_ready = 0;
> @@ -165,7 +168,6 @@ static int cpu_cores;			/* cores per package */
>  
>  static void	assign_cpu_ids(void);
>  static void	set_interrupt_apic_ids(void);
> -static int	start_all_aps(void);
>  static int	start_ap(int apic_id);
>  static void	release_aps(void *dummy);
>  
> @@ -569,7 +571,7 @@ cpu_mp_start(void)
>  	assign_cpu_ids();
>  
>  	/* Start each Application Processor */
> -	start_all_aps();
> +	cpu_ops.start_all_aps();
>  
>  	set_interrupt_apic_ids();
>  }
> @@ -908,8 +910,8 @@ assign_cpu_ids(void)
>  /*
>   * start each AP in our list
>   */
> -static int
> -start_all_aps(void)
> +int
> +native_start_all_aps(void)
>  {
>  	vm_offset_t va = boot_address + KERNBASE;
>  	u_int64_t *pt4, *pt3, *pt2;
> diff --git a/sys/amd64/include/cpu.h b/sys/amd64/include/cpu.h
> index 3d9ff531..ed9f1db 100644
> --- a/sys/amd64/include/cpu.h
> +++ b/sys/amd64/include/cpu.h
> @@ -64,6 +64,7 @@ struct cpu_ops {
>  	void (*cpu_init)(void);
>  	void (*cpu_resume)(void);
>  	void (*ipi_vectored)(u_int, int);
> +	int  (*start_all_aps)(void);
>  };
>  
>  extern struct	cpu_ops cpu_ops;
> diff --git a/sys/x86/xen/hvm.c b/sys/x86/xen/hvm.c
> index fb1ed79..5ec9f3a 100644
> --- a/sys/x86/xen/hvm.c
> +++ b/sys/x86/xen/hvm.c
> @@ -53,6 +53,9 @@ __FBSDID("$FreeBSD$");
>  #include <xen/hypervisor.h>
>  #include <xen/hvm.h>
>  #include <xen/xen_intr.h>
> +#ifdef __amd64__
> +#include <xen/pv.h>
> +#endif
>  
>  #include <xen/interface/hvm/params.h>
>  #include <xen/interface/vcpu.h>
> @@ -97,6 +100,11 @@ extern void pmap_lazyfix_action(void);
>  /* Variables used by mp_machdep to perform the bitmap IPI */
>  extern volatile u_int cpu_ipi_pending[MAXCPU];
>  
> +#ifdef __amd64__
> +/* Native AP start used on PVHVM */
> +extern int native_start_all_aps(void);
> +#endif
> +
>  /*---------------------------------- Macros ----------------------------------*/
>  #define	IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS)
>  
> @@ -119,7 +127,10 @@ enum xen_domain_type xen_domain_type = XEN_NATIVE;
>  struct cpu_ops xen_hvm_cpu_ops = {
>  	.ipi_vectored	= lapic_ipi_vectored,
>  	.cpu_init	= xen_hvm_cpu_init,
> -	.cpu_resume	= xen_hvm_cpu_resume
> +	.cpu_resume	= xen_hvm_cpu_resume,
> +#ifdef __amd64__
> +	.start_all_aps = native_start_all_aps,
> +#endif
>  };
>  
>  static MALLOC_DEFINE(M_XENHVM, "xen_hvm", "Xen HVM PV Support");
> @@ -698,6 +709,10 @@ xen_hvm_init(enum xen_hvm_init_type init_type)
>  		setup_xen_features();
>  		cpu_ops = xen_hvm_cpu_ops;
>   		vm_guest = VM_GUEST_XEN;
> +#ifdef __amd64__
> +		if (xen_pv_domain())
> +			cpu_ops.start_all_aps = xen_pv_start_all_aps;
> +#endif
>  		break;
>  	case XEN_HVM_INIT_RESUME:
>  		if (error != 0)
> diff --git a/sys/x86/xen/pv.c b/sys/x86/xen/pv.c
> index db576e0..7e45a83 100644
> --- a/sys/x86/xen/pv.c
> +++ b/sys/x86/xen/pv.c
> @@ -34,21 +34,43 @@ __FBSDID("$FreeBSD$");
>  #include <sys/kernel.h>
>  #include <sys/reboot.h>
>  #include <sys/systm.h>
> +#include <sys/malloc.h>
> +#include <sys/lock.h>
> +#include <sys/mutex.h>
> +#include <sys/smp.h>
> +
> +#include <vm/vm.h>
> +#include <vm/pmap.h>
> +#include <vm/vm_extern.h>
> +#include <vm/vm_kern.h>
>  
>  #include <machine/sysarch.h>
>  #include <machine/clock.h>
>  #include <machine/pc/bios.h>
> +#include <machine/smp.h>
>  
>  #include <xen/xen-os.h>
>  #include <xen/pv.h>
>  #include <xen/hypervisor.h>
>  
> +#include <xen/interface/vcpu.h>
> +
>  #define MAX_E820_ENTRIES	128
>  
>  /*--------------------------- Forward Declarations ---------------------------*/
>  static caddr_t xen_pv_parse_preload_data(u_int64_t);
>  static void xen_pv_parse_memmap(caddr_t, vm_paddr_t *, int *);
>  
> +/*---------------------------- Extern Declarations ---------------------------*/
> +/* Variables used by amd64 mp_machdep to start APs */
> +extern struct mtx ap_boot_mtx;
> +extern void *bootstacks[];
> +extern char *doublefault_stack;
> +extern char *nmi_stack;
> +extern void *dpcpu;
> +extern int bootAP;
> +extern char *bootSTK;
> +
>  /*-------------------------------- Global Data -------------------------------*/
>  /* Xen init_ops implementation. */
>  struct init_ops xen_init_ops = {
> @@ -78,6 +100,74 @@ static struct
>  
>  static struct bios_smap xen_smap[MAX_E820_ENTRIES];
>  
> +static int
> +start_xen_ap(int cpu)
> +{
> +	struct vcpu_guest_context *ctxt;
> +	int ms, cpus = mp_naps;
> +
> +	ctxt = malloc(sizeof(*ctxt), M_TEMP, M_NOWAIT | M_ZERO);
> +	if (ctxt == NULL)
> +		panic("unable to allocate memory");
> +
> +	ctxt->flags = VGCF_IN_KERNEL;
> +	ctxt->user_regs.rip = (unsigned long) init_secondary;
> +	ctxt->user_regs.rsp = (unsigned long) bootSTK;
> +
> +	/* Set the AP to use the same page tables */
> +	ctxt->ctrlreg[3] = KPML4phys;
> +
> +	if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt))
> +		panic("unable to initialize AP#%d\n", cpu);
> +
> +	free(ctxt, M_TEMP);
> +
> +	/* Launch the vCPU */
> +	if (HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL))
> +		panic("unable to start AP#%d\n", cpu);

Why the panic? Why not just return 0 (which I presume
would mean we failed to init the CPU?)
> +
> +	/* Wait up to 5 seconds for it to start. */
> +	for (ms = 0; ms < 5000; ms++) {
> +		if (mp_naps > cpus)
> +			return (1);	/* return SUCCESS */
> +		DELAY(1000);
> +	}
> +

Wow. So much simpler than the PV path.

> +	return (0);

There isn't a doorbell mechanism for the booted processor to tell
the BSP it is up?

> +}
> +
> +int
> +xen_pv_start_all_aps(void)
> +{
> +	int cpu;
> +
> +	mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
> +
> +	for (cpu = 1; cpu < mp_ncpus; cpu++) {
> +
> +		/* allocate and set up an idle stack data page */
> +		bootstacks[cpu] = (void *)kmem_malloc(kernel_arena,
> +		    KSTACK_PAGES * PAGE_SIZE, M_WAITOK | M_ZERO);
> +		doublefault_stack = (char *)kmem_malloc(kernel_arena,
> +		    PAGE_SIZE, M_WAITOK | M_ZERO);
> +		nmi_stack = (char *)kmem_malloc(kernel_arena, PAGE_SIZE,
> +		    M_WAITOK | M_ZERO);
> +		dpcpu = (void *)kmem_malloc(kernel_arena, DPCPU_SIZE,
> +		    M_WAITOK | M_ZERO);
> +
> +		bootSTK = (char *)bootstacks[cpu] + KSTACK_PAGES * PAGE_SIZE - 8;
> +		bootAP = cpu;
> +
> +		/* attempt to start the Application Processor */
> +		if (!start_xen_ap(cpu))
> +			panic("AP #%d failed to start!", cpu);
> +
> +		CPU_SET(cpu, &all_cpus);	/* record AP in CPU map */
> +	}
> +
> +	return (mp_naps);
> +}
> +
>  /*
>   * Functions to convert the "extra" parameters passed by Xen
>   * into FreeBSD boot options (from the i386 Xen port).
> diff --git a/sys/xen/pv.h b/sys/xen/pv.h
> index 71b8776..60d9def 100644
> --- a/sys/xen/pv.h
> +++ b/sys/xen/pv.h
> @@ -24,5 +24,6 @@
>  #define	__XEN_PV_H__
>  
>  void	xen_pv_set_init_ops(void);
> +int	xen_pv_start_all_aps(void);
>  
>  #endif	/* __XEN_PV_H__ */
> -- 
> 1.7.7.5 (Apple Git-26)
> 
> 
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xen.org
> http://lists.xen.org/xen-devel



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20140103205352.GB2732>