1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 /*
  26  * Copyright (c) 2010, Intel Corporation.
  27  * All rights reserved.
  28  */
  29 /*
  30  * Copyright (c) 2017, Joyent, Inc.  All rights reserved.
  31  */
  32 
  33 /*
  34  * To understand how the apix module interacts with the interrupt subsystem read
  35  * the theory statement in uts/i86pc/os/intr.c.
  36  */
  37 
  38 /*
  39  * PSMI 1.1 extensions are supported only in 2.6 and later versions.
  40  * PSMI 1.2 extensions are supported only in 2.7 and later versions.
  41  * PSMI 1.3 and 1.4 extensions are supported in Solaris 10.
  42  * PSMI 1.5 extensions are supported in Solaris Nevada.
  43  * PSMI 1.6 extensions are supported in Solaris Nevada.
  44  * PSMI 1.7 extensions are supported in Solaris Nevada.
  45  */
  46 #define PSMI_1_7
  47 
  48 #include <sys/processor.h>
  49 #include <sys/time.h>
  50 #include <sys/psm.h>
  51 #include <sys/smp_impldefs.h>
  52 #include <sys/cram.h>
  53 #include <sys/acpi/acpi.h>
  54 #include <sys/acpica.h>
  55 #include <sys/psm_common.h>
  56 #include <sys/pit.h>
  57 #include <sys/ddi.h>
  58 #include <sys/sunddi.h>
  59 #include <sys/ddi_impldefs.h>
  60 #include <sys/pci.h>
  61 #include <sys/promif.h>
  62 #include <sys/x86_archext.h>
  63 #include <sys/cpc_impl.h>
  64 #include <sys/uadmin.h>
  65 #include <sys/panic.h>
  66 #include <sys/debug.h>
  67 #include <sys/archsystm.h>
  68 #include <sys/trap.h>
  69 #include <sys/machsystm.h>
  70 #include <sys/sysmacros.h>
  71 #include <sys/cpuvar.h>
  72 #include <sys/rm_platter.h>
  73 #include <sys/privregs.h>
  74 #include <sys/note.h>
  75 #include <sys/pci_intr_lib.h>
  76 #include <sys/spl.h>
  77 #include <sys/clock.h>
  78 #include <sys/cyclic.h>
  79 #include <sys/dditypes.h>
  80 #include <sys/sunddi.h>
  81 #include <sys/x_call.h>
  82 #include <sys/reboot.h>
  83 #include <sys/mach_intr.h>
  84 #include <sys/apix.h>
  85 #include <sys/apix_irm_impl.h>
  86 
  87 static int apix_probe();
  88 static void apix_init();
  89 static void apix_picinit(void);
  90 static int apix_intr_enter(int, int *);
  91 static void apix_intr_exit(int, int);
  92 static void apix_setspl(int);
  93 static int apix_disable_intr(processorid_t);
  94 static void apix_enable_intr(processorid_t);
  95 static int apix_get_clkvect(int);
  96 static int apix_get_ipivect(int, int);
  97 static void apix_post_cyclic_setup(void *);
  98 static int apix_post_cpu_start();
  99 static int apix_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *,
 100     psm_intr_op_t, int *);
 101 
 102 /*
 103  * Helper functions for apix_intr_ops()
 104  */
 105 static void apix_redistribute_compute(void);
 106 static int apix_get_pending(apix_vector_t *);
 107 static apix_vector_t *apix_get_req_vector(ddi_intr_handle_impl_t *, ushort_t);
 108 static int apix_get_intr_info(ddi_intr_handle_impl_t *, apic_get_intr_t *);
 109 static char *apix_get_apic_type(void);
 110 static int apix_intx_get_pending(int);
 111 static void apix_intx_set_mask(int irqno);
 112 static void apix_intx_clear_mask(int irqno);
 113 static int apix_intx_get_shared(int irqno);
 114 static void apix_intx_set_shared(int irqno, int delta);
 115 static apix_vector_t *apix_intx_xlate_vector(dev_info_t *, int,
 116     struct intrspec *);
 117 static int apix_intx_alloc_vector(dev_info_t *, int, struct intrspec *);
 118 
 119 extern int apic_clkinit(int);
 120 
 121 /* IRM initialization for APIX PSM module */
 122 extern void apix_irm_init(void);
 123 
 124 extern int irm_enable;
 125 
 126 /*
 127  *      Local static data
 128  */
 129 static struct   psm_ops apix_ops = {
 130         apix_probe,
 131 
 132         apix_init,
 133         apix_picinit,
 134         apix_intr_enter,
 135         apix_intr_exit,
 136         apix_setspl,
 137         apix_addspl,
 138         apix_delspl,
 139         apix_disable_intr,
 140         apix_enable_intr,
 141         NULL,                   /* psm_softlvl_to_irq */
 142         NULL,                   /* psm_set_softintr */
 143 
 144         apic_set_idlecpu,
 145         apic_unset_idlecpu,
 146 
 147         apic_clkinit,
 148         apix_get_clkvect,
 149         NULL,                   /* psm_hrtimeinit */
 150         apic_gethrtime,
 151 
 152         apic_get_next_processorid,
 153         apic_cpu_start,
 154         apix_post_cpu_start,
 155         apic_shutdown,
 156         apix_get_ipivect,
 157         apic_send_ipi,
 158 
 159         NULL,                   /* psm_translate_irq */
 160         NULL,                   /* psm_notify_error */
 161         NULL,                   /* psm_notify_func */
 162         apic_timer_reprogram,
 163         apic_timer_enable,
 164         apic_timer_disable,
 165         apix_post_cyclic_setup,
 166         apic_preshutdown,
 167         apix_intr_ops,          /* Advanced DDI Interrupt framework */
 168         apic_state,             /* save, restore apic state for S3 */
 169         apic_cpu_ops,           /* CPU control interface. */
 170 };
 171 
 172 struct psm_ops *psmops = &apix_ops;
 173 
 174 static struct   psm_info apix_psm_info = {
 175         PSM_INFO_VER01_7,                       /* version */
 176         PSM_OWN_EXCLUSIVE,                      /* ownership */
 177         &apix_ops,                          /* operation */
 178         APIX_NAME,                              /* machine name */
 179         "apix MPv1.4 compatible",
 180 };
 181 
 182 static void *apix_hdlp;
 183 
 184 static int apix_is_enabled = 0;
 185 
 186 /*
 187  * Flag to indicate if APIX is to be enabled only for platforms
 188  * with specific hw feature(s).
 189  */
 190 int apix_hw_chk_enable = 1;
 191 
 192 /*
 193  * Hw features that are checked for enabling APIX support.
 194  */
 195 #define APIX_SUPPORT_X2APIC     0x00000001
 196 uint_t apix_supported_hw = APIX_SUPPORT_X2APIC;
 197 
 198 /*
 199  * apix_lock is used for cpu selection and vector re-binding
 200  */
 201 lock_t apix_lock;
 202 apix_impl_t *apixs[NCPU];
 203 /*
 204  * Mapping between device interrupt and the allocated vector. Indexed
 205  * by major number.
 206  */
 207 apix_dev_vector_t **apix_dev_vector;
 208 /*
 209  * Mapping between device major number and cpu id. It gets used
 210  * when interrupt binding policy round robin with affinity is
 211  * applied. With that policy, devices with the same major number
 212  * will be bound to the same CPU.
 213  */
 214 processorid_t *apix_major_to_cpu;       /* major to cpu mapping */
 215 kmutex_t apix_mutex;    /* for apix_dev_vector & apix_major_to_cpu */
 216 
 217 int apix_nipis = 16;    /* Maximum number of IPIs */
 218 /*
 219  * Maximum number of vectors in a CPU that can be used for interrupt
 220  * allocation (including IPIs and the reserved vectors).
 221  */
 222 int apix_cpu_nvectors = APIX_NVECTOR;
 223 
 224 /* number of CPUs in power-on transition state */
 225 static int apic_poweron_cnt = 0;
 226 
 227 /* gcpu.h */
 228 
 229 extern void apic_do_interrupt(struct regs *rp, trap_trace_rec_t *ttp);
 230 extern void apic_change_eoi();
 231 
 232 /*
 233  *      This is the loadable module wrapper
 234  */
 235 
 236 int
 237 _init(void)
 238 {
 239         if (apic_coarse_hrtime)
 240                 apix_ops.psm_gethrtime = &apic_gettime;
 241         return (psm_mod_init(&apix_hdlp, &apix_psm_info));
 242 }
 243 
 244 int
 245 _fini(void)
 246 {
 247         return (psm_mod_fini(&apix_hdlp, &apix_psm_info));
 248 }
 249 
 250 int
 251 _info(struct modinfo *modinfop)
 252 {
 253         return (psm_mod_info(&apix_hdlp, &apix_psm_info, modinfop));
 254 }
 255 
 256 static int
 257 apix_probe()
 258 {
 259         int rval;
 260 
 261         if (apix_enable == 0)
 262                 return (PSM_FAILURE);
 263 
 264         /*
 265          * FIXME Temporarily disable apix module on Xen HVM platform due to
 266          * known hang during boot (see #3605).
 267          *
 268          * Please remove when/if the issue is resolved.
 269          */
 270         if (get_hwenv() == HW_XEN_HVM)
 271                 return (PSM_FAILURE);
 272 
 273         /* check for hw features if specified  */
 274         if (apix_hw_chk_enable) {
 275                 /* check if x2APIC mode is supported */
 276                 if ((apix_supported_hw & APIX_SUPPORT_X2APIC) ==
 277                     APIX_SUPPORT_X2APIC) {
 278                         if (apic_local_mode() == LOCAL_X2APIC) {
 279                                 /* x2APIC mode activated by BIOS, switch ops */
 280                                 apic_mode = LOCAL_X2APIC;
 281                                 apic_change_ops();
 282                         } else if (!apic_detect_x2apic()) {
 283                                 /* x2APIC mode is not supported in the hw */
 284                                 apix_enable = 0;
 285                         }
 286                 }
 287                 if (apix_enable == 0)
 288                         return (PSM_FAILURE);
 289         }
 290 
 291         rval = apic_probe_common(apix_psm_info.p_mach_idstring);
 292         if (rval == PSM_SUCCESS)
 293                 apix_is_enabled = 1;
 294         else
 295                 apix_is_enabled = 0;
 296         return (rval);
 297 }
 298 
 299 /*
 300  * Initialize the data structures needed by pcplusmpx module.
 301  * Specifically, the data structures used by addspl() and delspl()
 302  * routines.
 303  */
 304 static void
 305 apix_softinit()
 306 {
 307         int i, *iptr;
 308         apix_impl_t *hdlp;
 309         int nproc;
 310 
 311         nproc = max(apic_nproc, apic_max_nproc);
 312 
 313         hdlp = kmem_zalloc(nproc * sizeof (apix_impl_t), KM_SLEEP);
 314         for (i = 0; i < nproc; i++) {
 315                 apixs[i] = &hdlp[i];
 316                 apixs[i]->x_cpuid = i;
 317                 LOCK_INIT_CLEAR(&apixs[i]->x_lock);
 318         }
 319 
 320         /* cpu 0 is always up (for now) */
 321         apic_cpus[0].aci_status = APIC_CPU_ONLINE | APIC_CPU_INTR_ENABLE;
 322 
 323         iptr = (int *)&apic_irq_table[0];
 324         for (i = 0; i <= APIC_MAX_VECTOR; i++) {
 325                 apic_level_intr[i] = 0;
 326                 *iptr++ = NULL;
 327         }
 328         mutex_init(&airq_mutex, NULL, MUTEX_DEFAULT, NULL);
 329 
 330         apix_dev_vector = kmem_zalloc(sizeof (apix_dev_vector_t *) * devcnt,
 331             KM_SLEEP);
 332 
 333         if (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) {
 334                 apix_major_to_cpu = kmem_zalloc(sizeof (int) * devcnt,
 335                     KM_SLEEP);
 336                 for (i = 0; i < devcnt; i++)
 337                         apix_major_to_cpu[i] = IRQ_UNINIT;
 338         }
 339 
 340         mutex_init(&apix_mutex, NULL, MUTEX_DEFAULT, NULL);
 341 }
 342 
 343 static int
 344 apix_get_pending_spl(void)
 345 {
 346         int cpuid = CPU->cpu_id;
 347 
 348         return (bsrw_insn(apixs[cpuid]->x_intr_pending));
 349 }
 350 
 351 static uintptr_t
 352 apix_get_intr_handler(int cpu, short vec)
 353 {
 354         apix_vector_t *apix_vector;
 355 
 356         ASSERT(cpu < apic_nproc && vec < APIX_NVECTOR);
 357         if (cpu >= apic_nproc)
 358                 return (NULL);
 359 
 360         apix_vector = apixs[cpu]->x_vectbl[vec];
 361 
 362         return ((uintptr_t)(apix_vector->v_autovect));
 363 }
 364 
 365 static void
 366 apix_init()
 367 {
 368         extern void (*do_interrupt_common)(struct regs *, trap_trace_rec_t *);
 369 
 370         APIC_VERBOSE(INIT, (CE_CONT, "apix: psm_softinit\n"));
 371 
 372         do_interrupt_common = apix_do_interrupt;
 373         addintr = apix_add_avintr;
 374         remintr = apix_rem_avintr;
 375         get_pending_spl = apix_get_pending_spl;
 376         get_intr_handler = apix_get_intr_handler;
 377         psm_get_localapicid = apic_get_localapicid;
 378         psm_get_ioapicid = apic_get_ioapicid;
 379 
 380         apix_softinit();
 381 
 382 #if !defined(__amd64)
 383         if (cpuid_have_cr8access(CPU))
 384                 apic_have_32bit_cr8 = 1;
 385 #endif
 386 
 387         /*
 388          * Initialize IRM pool parameters
 389          */
 390         if (irm_enable) {
 391                 int     i;
 392                 int     lowest_irq;
 393                 int     highest_irq;
 394 
 395                 /* number of CPUs present */
 396                 apix_irminfo.apix_ncpus = apic_nproc;
 397                 /* total number of entries in all of the IOAPICs present */
 398                 lowest_irq = apic_io_vectbase[0];
 399                 highest_irq = apic_io_vectend[0];
 400                 for (i = 1; i < apic_io_max; i++) {
 401                         if (apic_io_vectbase[i] < lowest_irq)
 402                                 lowest_irq = apic_io_vectbase[i];
 403                         if (apic_io_vectend[i] > highest_irq)
 404                                 highest_irq = apic_io_vectend[i];
 405                 }
 406                 apix_irminfo.apix_ioapic_max_vectors =
 407                     highest_irq - lowest_irq + 1;
 408                 /*
 409                  * Number of available per-CPU vectors excluding
 410                  * reserved vectors for Dtrace, int80, system-call,
 411                  * fast-trap, etc.
 412                  */
 413                 apix_irminfo.apix_per_cpu_vectors = APIX_NAVINTR -
 414                     APIX_SW_RESERVED_VECTORS;
 415 
 416                 /* Number of vectors (pre) allocated (SCI and HPET) */
 417                 apix_irminfo.apix_vectors_allocated = 0;
 418                 if (apic_hpet_vect != -1)
 419                         apix_irminfo.apix_vectors_allocated++;
 420                 if (apic_sci_vect != -1)
 421                         apix_irminfo.apix_vectors_allocated++;
 422         }
 423 }
 424 
 425 static void
 426 apix_init_intr()
 427 {
 428         processorid_t   cpun = psm_get_cpu_id();
 429         uint_t nlvt;
 430         uint32_t svr = AV_UNIT_ENABLE | APIC_SPUR_INTR;
 431         extern void cmi_cmci_trap(void);
 432 
 433         apic_reg_ops->apic_write_task_reg(APIC_MASK_ALL);
 434 
 435         if (apic_mode == LOCAL_APIC) {
 436                 /*
 437                  * We are running APIC in MMIO mode.
 438                  */
 439                 if (apic_flat_model) {
 440                         apic_reg_ops->apic_write(APIC_FORMAT_REG,
 441                             APIC_FLAT_MODEL);
 442                 } else {
 443                         apic_reg_ops->apic_write(APIC_FORMAT_REG,
 444                             APIC_CLUSTER_MODEL);
 445                 }
 446 
 447                 apic_reg_ops->apic_write(APIC_DEST_REG,
 448                     AV_HIGH_ORDER >> cpun);
 449         }
 450 
 451         if (apic_directed_EOI_supported()) {
 452                 /*
 453                  * Setting the 12th bit in the Spurious Interrupt Vector
 454                  * Register suppresses broadcast EOIs generated by the local
 455                  * APIC. The suppression of broadcast EOIs happens only when
 456                  * interrupts are level-triggered.
 457                  */
 458                 svr |= APIC_SVR_SUPPRESS_BROADCAST_EOI;
 459         }
 460 
 461         /* need to enable APIC before unmasking NMI */
 462         apic_reg_ops->apic_write(APIC_SPUR_INT_REG, svr);
 463 
 464         /*
 465          * Presence of an invalid vector with delivery mode AV_FIXED can
 466          * cause an error interrupt, even if the entry is masked...so
 467          * write a valid vector to LVT entries along with the mask bit
 468          */
 469 
 470         /* All APICs have timer and LINT0/1 */
 471         apic_reg_ops->apic_write(APIC_LOCAL_TIMER, AV_MASK|APIC_RESV_IRQ);
 472         apic_reg_ops->apic_write(APIC_INT_VECT0, AV_MASK|APIC_RESV_IRQ);
 473         apic_reg_ops->apic_write(APIC_INT_VECT1, AV_NMI);    /* enable NMI */
 474 
 475         /*
 476          * On integrated APICs, the number of LVT entries is
 477          * 'Max LVT entry' + 1; on 82489DX's (non-integrated
 478          * APICs), nlvt is "3" (LINT0, LINT1, and timer)
 479          */
 480 
 481         if (apic_cpus[cpun].aci_local_ver < APIC_INTEGRATED_VERS) {
 482                 nlvt = 3;
 483         } else {
 484                 nlvt = ((apic_reg_ops->apic_read(APIC_VERS_REG) >> 16) &
 485                     0xFF) + 1;
 486         }
 487 
 488         if (nlvt >= 5) {
 489                 /* Enable performance counter overflow interrupt */
 490 
 491                 if (!is_x86_feature(x86_featureset, X86FSET_MSR))
 492                         apic_enable_cpcovf_intr = 0;
 493                 if (apic_enable_cpcovf_intr) {
 494                         if (apic_cpcovf_vect == 0) {
 495                                 int ipl = APIC_PCINT_IPL;
 496 
 497                                 apic_cpcovf_vect = apix_get_ipivect(ipl, -1);
 498                                 ASSERT(apic_cpcovf_vect);
 499 
 500                                 (void) add_avintr(NULL, ipl,
 501                                     (avfunc)kcpc_hw_overflow_intr,
 502                                     "apic pcint", apic_cpcovf_vect,
 503                                     NULL, NULL, NULL, NULL);
 504                                 kcpc_hw_overflow_intr_installed = 1;
 505                                 kcpc_hw_enable_cpc_intr =
 506                                     apic_cpcovf_mask_clear;
 507                         }
 508                         apic_reg_ops->apic_write(APIC_PCINT_VECT,
 509                             apic_cpcovf_vect);
 510                 }
 511         }
 512 
 513         if (nlvt >= 6) {
 514                 /* Only mask TM intr if the BIOS apparently doesn't use it */
 515 
 516                 uint32_t lvtval;
 517 
 518                 lvtval = apic_reg_ops->apic_read(APIC_THERM_VECT);
 519                 if (((lvtval & AV_MASK) == AV_MASK) ||
 520                     ((lvtval & AV_DELIV_MODE) != AV_SMI)) {
 521                         apic_reg_ops->apic_write(APIC_THERM_VECT,
 522                             AV_MASK|APIC_RESV_IRQ);
 523                 }
 524         }
 525 
 526         /* Enable error interrupt */
 527 
 528         if (nlvt >= 4 && apic_enable_error_intr) {
 529                 if (apic_errvect == 0) {
 530                         int ipl = 0xf;  /* get highest priority intr */
 531                         apic_errvect = apix_get_ipivect(ipl, -1);
 532                         ASSERT(apic_errvect);
 533                         /*
 534                          * Not PSMI compliant, but we are going to merge
 535                          * with ON anyway
 536                          */
 537                         (void) add_avintr(NULL, ipl,
 538                             (avfunc)apic_error_intr, "apic error intr",
 539                             apic_errvect, NULL, NULL, NULL, NULL);
 540                 }
 541                 apic_reg_ops->apic_write(APIC_ERR_VECT, apic_errvect);
 542                 apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
 543                 apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
 544         }
 545 
 546         /* Enable CMCI interrupt */
 547         if (cmi_enable_cmci) {
 548                 mutex_enter(&cmci_cpu_setup_lock);
 549                 if (cmci_cpu_setup_registered == 0) {
 550                         mutex_enter(&cpu_lock);
 551                         register_cpu_setup_func(cmci_cpu_setup, NULL);
 552                         mutex_exit(&cpu_lock);
 553                         cmci_cpu_setup_registered = 1;
 554                 }
 555                 mutex_exit(&cmci_cpu_setup_lock);
 556 
 557                 if (apic_cmci_vect == 0) {
 558                         int ipl = 0x2;
 559                         apic_cmci_vect = apix_get_ipivect(ipl, -1);
 560                         ASSERT(apic_cmci_vect);
 561 
 562                         (void) add_avintr(NULL, ipl,
 563                             (avfunc)cmi_cmci_trap, "apic cmci intr",
 564                             apic_cmci_vect, NULL, NULL, NULL, NULL);
 565                 }
 566                 apic_reg_ops->apic_write(APIC_CMCI_VECT, apic_cmci_vect);
 567         }
 568 
 569         apic_reg_ops->apic_write_task_reg(0);
 570 }
 571 
 572 static void
 573 apix_picinit(void)
 574 {
 575         int i, j;
 576         uint_t isr;
 577 
 578         APIC_VERBOSE(INIT, (CE_CONT, "apix: psm_picinit\n"));
 579 
 580         /*
 581          * initialize interrupt remapping before apic
 582          * hardware initialization
 583          */
 584         apic_intrmap_init(apic_mode);
 585         if (apic_vt_ops == psm_vt_ops)
 586                 apix_mul_ioapic_method = APIC_MUL_IOAPIC_IIR;
 587 
 588         /*
 589          * On UniSys Model 6520, the BIOS leaves vector 0x20 isr
 590          * bit on without clearing it with EOI.  Since softint
 591          * uses vector 0x20 to interrupt itself, so softint will
 592          * not work on this machine.  In order to fix this problem
 593          * a check is made to verify all the isr bits are clear.
 594          * If not, EOIs are issued to clear the bits.
 595          */
 596         for (i = 7; i >= 1; i--) {
 597                 isr = apic_reg_ops->apic_read(APIC_ISR_REG + (i * 4));
 598                 if (isr != 0)
 599                         for (j = 0; ((j < 32) && (isr != 0)); j++)
 600                                 if (isr & (1 << j)) {
 601                                         apic_reg_ops->apic_write(
 602                                             APIC_EOI_REG, 0);
 603                                         isr &= ~(1 << j);
 604                                         apic_error |= APIC_ERR_BOOT_EOI;
 605                                 }
 606         }
 607 
 608         /* set a flag so we know we have run apic_picinit() */
 609         apic_picinit_called = 1;
 610         LOCK_INIT_CLEAR(&apic_gethrtime_lock);
 611         LOCK_INIT_CLEAR(&apic_ioapic_lock);
 612         LOCK_INIT_CLEAR(&apic_error_lock);
 613         LOCK_INIT_CLEAR(&apic_mode_switch_lock);
 614 
 615         picsetup();      /* initialise the 8259 */
 616 
 617         /* add nmi handler - least priority nmi handler */
 618         LOCK_INIT_CLEAR(&apic_nmi_lock);
 619 
 620         if (!psm_add_nmintr(0, (avfunc) apic_nmi_intr,
 621             "apix NMI handler", (caddr_t)NULL))
 622                 cmn_err(CE_WARN, "apix: Unable to add nmi handler");
 623 
 624         apix_init_intr();
 625 
 626         /* enable apic mode if imcr present */
 627         if (apic_imcrp) {
 628                 outb(APIC_IMCR_P1, (uchar_t)APIC_IMCR_SELECT);
 629                 outb(APIC_IMCR_P2, (uchar_t)APIC_IMCR_APIC);
 630         }
 631 
 632         ioapix_init_intr(IOAPIC_MASK);
 633 
 634         /* setup global IRM pool if applicable */
 635         if (irm_enable)
 636                 apix_irm_init();
 637 }
 638 
 639 static __inline__ void
 640 apix_send_eoi(void)
 641 {
 642         if (apic_mode == LOCAL_APIC)
 643                 LOCAL_APIC_WRITE_REG(APIC_EOI_REG, 0);
 644         else
 645                 X2APIC_WRITE(APIC_EOI_REG, 0);
 646 }
 647 
 648 /*
 649  * platform_intr_enter
 650  *
 651  *      Called at the beginning of the interrupt service routine, but unlike
 652  *      pcplusmp, does not mask interrupts. An EOI is given to the interrupt
 653  *      controller to enable other HW interrupts but interrupts are still
 654  *      masked by the IF flag.
 655  *
 656  *      Return -1 for spurious interrupts
 657  *
 658  */
 659 static int
 660 apix_intr_enter(int ipl, int *vectorp)
 661 {
 662         struct cpu *cpu = CPU;
 663         uint32_t cpuid = CPU->cpu_id;
 664         apic_cpus_info_t *cpu_infop;
 665         uchar_t vector;
 666         apix_vector_t *vecp;
 667         int nipl = -1;
 668 
 669         /*
 670          * The real vector delivered is (*vectorp + 0x20), but our caller
 671          * subtracts 0x20 from the vector before passing it to us.
 672          * (That's why APIC_BASE_VECT is 0x20.)
 673          */
 674         vector = *vectorp = (uchar_t)*vectorp + APIC_BASE_VECT;
 675 
 676         cpu_infop = &apic_cpus[cpuid];
 677         if (vector == APIC_SPUR_INTR) {
 678                 cpu_infop->aci_spur_cnt++;
 679                 return (APIC_INT_SPURIOUS);
 680         }
 681 
 682         vecp = xv_vector(cpuid, vector);
 683         if (vecp == NULL) {
 684                 if (APIX_IS_FAKE_INTR(vector))
 685                         nipl = apix_rebindinfo.i_pri;
 686                 apix_send_eoi();
 687                 return (nipl);
 688         }
 689         nipl = vecp->v_pri;
 690 
 691         /* if interrupted by the clock, increment apic_nsec_since_boot */
 692         if (vector == (apic_clkvect + APIC_BASE_VECT)) {
 693                 if (!apic_oneshot) {
 694                         /* NOTE: this is not MT aware */
 695                         apic_hrtime_stamp++;
 696                         apic_nsec_since_boot += apic_nsec_per_intr;
 697                         apic_hrtime_stamp++;
 698                         last_count_read = apic_hertz_count;
 699                         apix_redistribute_compute();
 700                 }
 701 
 702                 apix_send_eoi();
 703 
 704                 return (nipl);
 705         }
 706 
 707         ASSERT(vecp->v_state != APIX_STATE_OBSOLETED);
 708 
 709         /* pre-EOI handling for level-triggered interrupts */
 710         if (!APIX_IS_DIRECTED_EOI(apix_mul_ioapic_method) &&
 711             (vecp->v_type & APIX_TYPE_FIXED) && apic_level_intr[vecp->v_inum])
 712                 apix_level_intr_pre_eoi(vecp->v_inum);
 713 
 714         /* send back EOI */
 715         apix_send_eoi();
 716 
 717         cpu_infop->aci_current[nipl] = vector;
 718         if ((nipl > ipl) && (nipl > cpu->cpu_base_spl)) {
 719                 cpu_infop->aci_curipl = (uchar_t)nipl;
 720                 cpu_infop->aci_ISR_in_progress |= 1 << nipl;
 721         }
 722 
 723 #ifdef  DEBUG
 724         if (vector >= APIX_IPI_MIN)
 725                 return (nipl);  /* skip IPI */
 726 
 727         APIC_DEBUG_BUF_PUT(vector);
 728         APIC_DEBUG_BUF_PUT(vecp->v_inum);
 729         APIC_DEBUG_BUF_PUT(nipl);
 730         APIC_DEBUG_BUF_PUT(psm_get_cpu_id());
 731         if ((apic_stretch_interrupts) && (apic_stretch_ISR & (1 << nipl)))
 732                 drv_usecwait(apic_stretch_interrupts);
 733 #endif /* DEBUG */
 734 
 735         return (nipl);
 736 }
 737 
 738 /*
 739  * Any changes made to this function must also change X2APIC
 740  * version of intr_exit.
 741  */
 742 static void
 743 apix_intr_exit(int prev_ipl, int arg2)
 744 {
 745         int cpuid = psm_get_cpu_id();
 746         apic_cpus_info_t *cpu_infop = &apic_cpus[cpuid];
 747         apix_impl_t *apixp = apixs[cpuid];
 748 
 749         UNREFERENCED_1PARAMETER(arg2);
 750 
 751         cpu_infop->aci_curipl = (uchar_t)prev_ipl;
 752         /* ISR above current pri could not be in progress */
 753         cpu_infop->aci_ISR_in_progress &= (2 << prev_ipl) - 1;
 754 
 755         if (apixp->x_obsoletes != NULL) {
 756                 if (APIX_CPU_LOCK_HELD(cpuid))
 757                         return;
 758 
 759                 APIX_ENTER_CPU_LOCK(cpuid);
 760                 (void) apix_obsolete_vector(apixp->x_obsoletes);
 761                 APIX_LEAVE_CPU_LOCK(cpuid);
 762         }
 763 }
 764 
 765 /*
 766  * The pcplusmp setspl code uses the TPR to mask all interrupts at or below the
 767  * given ipl, but apix never uses the TPR and we never mask a subset of the
 768  * interrupts. They are either all blocked by the IF flag or all can come in.
 769  *
 770  * For setspl, we mask all interrupts for XC_HI_PIL (15), otherwise, interrupts
 771  * can come in if currently enabled by the IF flag. This table shows the state
 772  * of the IF flag when we leave this function.
 773  *
 774  *    curr IF | ipl == 15       ipl != 15
 775  *    --------+---------------------------
 776  *       0    |    0                0
 777  *       1    |    0                1
 778  */
 779 static void
 780 apix_setspl(int ipl)
 781 {
 782         /*
 783          * Interrupts at ipl above this cannot be in progress, so the following
 784          * mask is ok.
 785          */
 786         apic_cpus[psm_get_cpu_id()].aci_ISR_in_progress &= (2 << ipl) - 1;
 787 
 788         if (ipl == XC_HI_PIL)
 789                 cli();
 790 }
 791 
 792 int
 793 apix_addspl(int virtvec, int ipl, int min_ipl, int max_ipl)
 794 {
 795         uint32_t cpuid = APIX_VIRTVEC_CPU(virtvec);
 796         uchar_t vector = (uchar_t)APIX_VIRTVEC_VECTOR(virtvec);
 797         apix_vector_t *vecp = xv_vector(cpuid, vector);
 798 
 799         UNREFERENCED_3PARAMETER(ipl, min_ipl, max_ipl);
 800         ASSERT(vecp != NULL && LOCK_HELD(&apix_lock));
 801 
 802         if (vecp->v_type == APIX_TYPE_FIXED)
 803                 apix_intx_set_shared(vecp->v_inum, 1);
 804 
 805         /* There are more interrupts, so it's already been enabled */
 806         if (vecp->v_share > 1)
 807                 return (PSM_SUCCESS);
 808 
 809         /* return if it is not hardware interrupt */
 810         if (vecp->v_type == APIX_TYPE_IPI)
 811                 return (PSM_SUCCESS);
 812 
 813         /*
 814          * if apix_picinit() has not been called yet, just return.
 815          * At the end of apic_picinit(), we will call setup_io_intr().
 816          */
 817         if (!apic_picinit_called)
 818                 return (PSM_SUCCESS);
 819 
 820         (void) apix_setup_io_intr(vecp);
 821 
 822         return (PSM_SUCCESS);
 823 }
 824 
 825 int
 826 apix_delspl(int virtvec, int ipl, int min_ipl, int max_ipl)
 827 {
 828         uint32_t cpuid = APIX_VIRTVEC_CPU(virtvec);
 829         uchar_t vector = (uchar_t)APIX_VIRTVEC_VECTOR(virtvec);
 830         apix_vector_t *vecp = xv_vector(cpuid, vector);
 831 
 832         UNREFERENCED_3PARAMETER(ipl, min_ipl, max_ipl);
 833         ASSERT(vecp != NULL && LOCK_HELD(&apix_lock));
 834 
 835         if (vecp->v_type == APIX_TYPE_FIXED)
 836                 apix_intx_set_shared(vecp->v_inum, -1);
 837 
 838         /* There are more interrupts */
 839         if (vecp->v_share > 1)
 840                 return (PSM_SUCCESS);
 841 
 842         /* return if it is not hardware interrupt */
 843         if (vecp->v_type == APIX_TYPE_IPI)
 844                 return (PSM_SUCCESS);
 845 
 846         if (!apic_picinit_called) {
 847                 cmn_err(CE_WARN, "apix: delete 0x%x before apic init",
 848                     virtvec);
 849                 return (PSM_SUCCESS);
 850         }
 851 
 852         apix_disable_vector(vecp);
 853 
 854         return (PSM_SUCCESS);
 855 }
 856 
 857 /*
 858  * Try and disable all interrupts. We just assign interrupts to other
 859  * processors based on policy. If any were bound by user request, we
 860  * let them continue and return failure. We do not bother to check
 861  * for cache affinity while rebinding.
 862  */
 863 static int
 864 apix_disable_intr(processorid_t cpun)
 865 {
 866         apix_impl_t *apixp = apixs[cpun];
 867         apix_vector_t *vecp, *newp;
 868         int bindcpu, i, hardbound = 0, errbound = 0, ret, loop, type;
 869 
 870         lock_set(&apix_lock);
 871 
 872         apic_cpus[cpun].aci_status &= ~APIC_CPU_INTR_ENABLE;
 873         apic_cpus[cpun].aci_curipl = 0;
 874 
 875         /* if this is for SUSPEND operation, skip rebinding */
 876         if (apic_cpus[cpun].aci_status & APIC_CPU_SUSPEND) {
 877                 for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) {
 878                         vecp = apixp->x_vectbl[i];
 879                         if (!IS_VECT_ENABLED(vecp))
 880                                 continue;
 881 
 882                         apix_disable_vector(vecp);
 883                 }
 884                 lock_clear(&apix_lock);
 885                 return (PSM_SUCCESS);
 886         }
 887 
 888         for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) {
 889                 vecp = apixp->x_vectbl[i];
 890                 if (!IS_VECT_ENABLED(vecp))
 891                         continue;
 892 
 893                 if (vecp->v_flags & APIX_VECT_USER_BOUND) {
 894                         hardbound++;
 895                         continue;
 896                 }
 897                 type = vecp->v_type;
 898 
 899                 /*
 900                  * If there are bound interrupts on this cpu, then
 901                  * rebind them to other processors.
 902                  */
 903                 loop = 0;
 904                 do {
 905                         bindcpu = apic_find_cpu(APIC_CPU_INTR_ENABLE);
 906 
 907                         if (type != APIX_TYPE_MSI)
 908                                 newp = apix_set_cpu(vecp, bindcpu, &ret);
 909                         else
 910                                 newp = apix_grp_set_cpu(vecp, bindcpu, &ret);
 911                 } while ((newp == NULL) && (loop++ < apic_nproc));
 912 
 913                 if (loop >= apic_nproc) {
 914                         errbound++;
 915                         cmn_err(CE_WARN, "apix: failed to rebind vector %x/%x",
 916                             vecp->v_cpuid, vecp->v_vector);
 917                 }
 918         }
 919 
 920         lock_clear(&apix_lock);
 921 
 922         if (hardbound || errbound) {
 923                 cmn_err(CE_WARN, "Could not disable interrupts on %d"
 924                     "due to user bound interrupts or failed operation",
 925                     cpun);
 926                 return (PSM_FAILURE);
 927         }
 928 
 929         return (PSM_SUCCESS);
 930 }
 931 
 932 /*
 933  * Bind interrupts to specified CPU
 934  */
 935 static void
 936 apix_enable_intr(processorid_t cpun)
 937 {
 938         apix_vector_t *vecp;
 939         int i, ret;
 940         processorid_t n;
 941 
 942         lock_set(&apix_lock);
 943 
 944         apic_cpus[cpun].aci_status |= APIC_CPU_INTR_ENABLE;
 945 
 946         /* interrupt enabling for system resume */
 947         if (apic_cpus[cpun].aci_status & APIC_CPU_SUSPEND) {
 948                 for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) {
 949                         vecp = xv_vector(cpun, i);
 950                         if (!IS_VECT_ENABLED(vecp))
 951                                 continue;
 952 
 953                         apix_enable_vector(vecp);
 954                 }
 955                 apic_cpus[cpun].aci_status &= ~APIC_CPU_SUSPEND;
 956         }
 957 
 958         for (n = 0; n < apic_nproc; n++) {
 959                 if (!apic_cpu_in_range(n) || n == cpun ||
 960                     (apic_cpus[n].aci_status & APIC_CPU_INTR_ENABLE) == 0)
 961                         continue;
 962 
 963                 for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) {
 964                         vecp = xv_vector(n, i);
 965                         if (!IS_VECT_ENABLED(vecp) ||
 966                             vecp->v_bound_cpuid != cpun)
 967                                 continue;
 968 
 969                         if (vecp->v_type != APIX_TYPE_MSI)
 970                                 (void) apix_set_cpu(vecp, cpun, &ret);
 971                         else
 972                                 (void) apix_grp_set_cpu(vecp, cpun, &ret);
 973                 }
 974         }
 975 
 976         lock_clear(&apix_lock);
 977 }
 978 
 979 /*
 980  * Allocate vector for IPI
 981  * type == -1 indicates it is an internal request. Do not change
 982  * resv_vector for these requests.
 983  */
 984 static int
 985 apix_get_ipivect(int ipl, int type)
 986 {
 987         uchar_t vector;
 988 
 989         if ((vector = apix_alloc_ipi(ipl)) > 0) {
 990                 if (type != -1)
 991                         apic_resv_vector[ipl] = vector;
 992                 return (vector);
 993         }
 994         apic_error |= APIC_ERR_GET_IPIVECT_FAIL;
 995         return (-1);    /* shouldn't happen */
 996 }
 997 
 998 static int
 999 apix_get_clkvect(int ipl)
1000 {
1001         int vector;
1002 
1003         if ((vector = apix_get_ipivect(ipl, -1)) == -1)
1004                 return (-1);
1005 
1006         apic_clkvect = vector - APIC_BASE_VECT;
1007         APIC_VERBOSE(IPI, (CE_CONT, "apix: clock vector = %x\n",
1008             apic_clkvect));
1009         return (vector);
1010 }
1011 
1012 static int
1013 apix_post_cpu_start()
1014 {
1015         int cpun;
1016         static int cpus_started = 1;
1017 
1018         /* We know this CPU + BSP  started successfully. */
1019         cpus_started++;
1020 
1021         /*
1022          * On BSP we would have enabled X2APIC, if supported by processor,
1023          * in acpi_probe(), but on AP we do it here.
1024          *
1025          * We enable X2APIC mode only if BSP is running in X2APIC & the
1026          * local APIC mode of the current CPU is MMIO (xAPIC).
1027          */
1028         if (apic_mode == LOCAL_X2APIC && apic_detect_x2apic() &&
1029             apic_local_mode() == LOCAL_APIC) {
1030                 apic_enable_x2apic();
1031         }
1032 
1033         /*
1034          * Switch back to x2apic IPI sending method for performance when target
1035          * CPU has entered x2apic mode.
1036          */
1037         if (apic_mode == LOCAL_X2APIC) {
1038                 apic_switch_ipi_callback(B_FALSE);
1039         }
1040 
1041         splx(ipltospl(LOCK_LEVEL));
1042         apix_init_intr();
1043 
1044         /*
1045          * since some systems don't enable the internal cache on the non-boot
1046          * cpus, so we have to enable them here
1047          */
1048         setcr0(getcr0() & ~(CR0_CD | CR0_NW));
1049 
1050 #ifdef  DEBUG
1051         APIC_AV_PENDING_SET();
1052 #else
1053         if (apic_mode == LOCAL_APIC)
1054                 APIC_AV_PENDING_SET();
1055 #endif  /* DEBUG */
1056 
1057         /*
1058          * We may be booting, or resuming from suspend; aci_status will
1059          * be APIC_CPU_INTR_ENABLE if coming from suspend, so we add the
1060          * APIC_CPU_ONLINE flag here rather than setting aci_status completely.
1061          */
1062         cpun = psm_get_cpu_id();
1063         apic_cpus[cpun].aci_status |= APIC_CPU_ONLINE;
1064 
1065         apic_reg_ops->apic_write(APIC_DIVIDE_REG, apic_divide_reg_init);
1066 
1067         return (PSM_SUCCESS);
1068 }
1069 
1070 /*
1071  * If this module needs a periodic handler for the interrupt distribution, it
1072  * can be added here. The argument to the periodic handler is not currently
1073  * used, but is reserved for future.
1074  */
1075 static void
1076 apix_post_cyclic_setup(void *arg)
1077 {
1078         UNREFERENCED_1PARAMETER(arg);
1079 
1080         cyc_handler_t cyh;
1081         cyc_time_t cyt;
1082 
1083         /* cpu_lock is held */
1084         /* set up a periodic handler for intr redistribution */
1085 
1086         /*
1087          * In peridoc mode intr redistribution processing is done in
1088          * apic_intr_enter during clk intr processing
1089          */
1090         if (!apic_oneshot)
1091                 return;
1092 
1093         /*
1094          * Register a periodical handler for the redistribution processing.
1095          * Though we would generally prefer to use the DDI interface for
1096          * periodic handler invocation, ddi_periodic_add(9F), we are
1097          * unfortunately already holding cpu_lock, which ddi_periodic_add will
1098          * attempt to take for us.  Thus, we add our own cyclic directly:
1099          */
1100         cyh.cyh_func = (void (*)(void *))apix_redistribute_compute;
1101         cyh.cyh_arg = NULL;
1102         cyh.cyh_level = CY_LOW_LEVEL;
1103 
1104         cyt.cyt_when = 0;
1105         cyt.cyt_interval = apic_redistribute_sample_interval;
1106 
1107         apic_cyclic_id = cyclic_add(&cyh, &cyt);
1108 }
1109 
1110 /*
1111  * Called the first time we enable x2apic mode on this cpu.
1112  * Update some of the function pointers to use x2apic routines.
1113  */
1114 void
1115 x2apic_update_psm()
1116 {
1117         struct psm_ops *pops = &apix_ops;
1118 
1119         ASSERT(pops != NULL);
1120 
1121         /*
1122          * The pcplusmp module's version of x2apic_update_psm makes additional
1123          * changes that we do not have to make here. It needs to make those
1124          * changes because pcplusmp relies on the TPR register and the means of
1125          * addressing that changes when using the local apic versus the x2apic.
1126          * It's also worth noting that the apix driver specific function end up
1127          * being apix_foo as opposed to apic_foo and x2apic_foo.
1128          */
1129         pops->psm_send_ipi = x2apic_send_ipi;
1130 
1131         send_dirintf = pops->psm_send_ipi;
1132 
1133         apic_mode = LOCAL_X2APIC;
1134         apic_change_ops();
1135 }
1136 
1137 /*
1138  * This function provides external interface to the nexus for all
1139  * functionalities related to the new DDI interrupt framework.
1140  *
1141  * Input:
1142  * dip     - pointer to the dev_info structure of the requested device
1143  * hdlp    - pointer to the internal interrupt handle structure for the
1144  *           requested interrupt
1145  * intr_op - opcode for this call
1146  * result  - pointer to the integer that will hold the result to be
1147  *           passed back if return value is PSM_SUCCESS
1148  *
1149  * Output:
1150  * return value is either PSM_SUCCESS or PSM_FAILURE
1151  */
1152 static int
1153 apix_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp,
1154     psm_intr_op_t intr_op, int *result)
1155 {
1156         int             cap;
1157         apix_vector_t   *vecp, *newvecp;
1158         struct intrspec *ispec, intr_spec;
1159         processorid_t target;
1160 
1161         ispec = &intr_spec;
1162         ispec->intrspec_pri = hdlp->ih_pri;
1163         ispec->intrspec_vec = hdlp->ih_inum;
1164         ispec->intrspec_func = hdlp->ih_cb_func;
1165 
1166         switch (intr_op) {
1167         case PSM_INTR_OP_ALLOC_VECTORS:
1168                 switch (hdlp->ih_type) {
1169                 case DDI_INTR_TYPE_MSI:
1170                         /* allocate MSI vectors */
1171                         *result = apix_alloc_msi(dip, hdlp->ih_inum,
1172                             hdlp->ih_scratch1,
1173                             (int)(uintptr_t)hdlp->ih_scratch2);
1174                         break;
1175                 case DDI_INTR_TYPE_MSIX:
1176                         /* allocate MSI-X vectors */
1177                         *result = apix_alloc_msix(dip, hdlp->ih_inum,
1178                             hdlp->ih_scratch1,
1179                             (int)(uintptr_t)hdlp->ih_scratch2);
1180                         break;
1181                 case DDI_INTR_TYPE_FIXED:
1182                         /* allocate or share vector for fixed */
1183                         if ((ihdl_plat_t *)hdlp->ih_private == NULL) {
1184                                 return (PSM_FAILURE);
1185                         }
1186                         ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp;
1187                         *result = apix_intx_alloc_vector(dip, hdlp->ih_inum,
1188                             ispec);
1189                         break;
1190                 default:
1191                         return (PSM_FAILURE);
1192                 }
1193                 break;
1194         case PSM_INTR_OP_FREE_VECTORS:
1195                 apix_free_vectors(dip, hdlp->ih_inum, hdlp->ih_scratch1,
1196                     hdlp->ih_type);
1197                 break;
1198         case PSM_INTR_OP_XLATE_VECTOR:
1199                 /*
1200                  * Vectors are allocated by ALLOC and freed by FREE.
1201                  * XLATE finds and returns APIX_VIRTVEC_VECTOR(cpu, vector).
1202                  */
1203                 *result = APIX_INVALID_VECT;
1204                 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1205                 if (vecp != NULL) {
1206                         *result = APIX_VIRTVECTOR(vecp->v_cpuid,
1207                             vecp->v_vector);
1208                         break;
1209                 }
1210 
1211                 /*
1212                  * No vector to device mapping exists. If this is FIXED type
1213                  * then check if this IRQ is already mapped for another device
1214                  * then return the vector number for it (i.e. shared IRQ case).
1215                  * Otherwise, return PSM_FAILURE.
1216                  */
1217                 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) {
1218                         vecp = apix_intx_xlate_vector(dip, hdlp->ih_inum,
1219                             ispec);
1220                         *result = (vecp == NULL) ? APIX_INVALID_VECT :
1221                             APIX_VIRTVECTOR(vecp->v_cpuid, vecp->v_vector);
1222                 }
1223                 if (*result == APIX_INVALID_VECT)
1224                         return (PSM_FAILURE);
1225                 break;
1226         case PSM_INTR_OP_GET_PENDING:
1227                 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1228                 if (vecp == NULL)
1229                         return (PSM_FAILURE);
1230 
1231                 *result = apix_get_pending(vecp);
1232                 break;
1233         case PSM_INTR_OP_CLEAR_MASK:
1234                 if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
1235                         return (PSM_FAILURE);
1236 
1237                 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1238                 if (vecp == NULL)
1239                         return (PSM_FAILURE);
1240 
1241                 apix_intx_clear_mask(vecp->v_inum);
1242                 break;
1243         case PSM_INTR_OP_SET_MASK:
1244                 if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
1245                         return (PSM_FAILURE);
1246 
1247                 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1248                 if (vecp == NULL)
1249                         return (PSM_FAILURE);
1250 
1251                 apix_intx_set_mask(vecp->v_inum);
1252                 break;
1253         case PSM_INTR_OP_GET_SHARED:
1254                 if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
1255                         return (PSM_FAILURE);
1256 
1257                 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1258                 if (vecp == NULL)
1259                         return (PSM_FAILURE);
1260 
1261                 *result = apix_intx_get_shared(vecp->v_inum);
1262                 break;
1263         case PSM_INTR_OP_SET_PRI:
1264                 /*
1265                  * Called prior to adding the interrupt handler or when
1266                  * an interrupt handler is unassigned.
1267                  */
1268                 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
1269                         return (PSM_SUCCESS);
1270 
1271                 if (apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type) == NULL)
1272                         return (PSM_FAILURE);
1273 
1274                 break;
1275         case PSM_INTR_OP_SET_CPU:
1276         case PSM_INTR_OP_GRP_SET_CPU:
1277                 /*
1278                  * The interrupt handle given here has been allocated
1279                  * specifically for this command, and ih_private carries
1280                  * a CPU value.
1281                  */
1282                 *result = EINVAL;
1283                 target = (int)(intptr_t)hdlp->ih_private;
1284                 if (!apic_cpu_in_range(target)) {
1285                         DDI_INTR_IMPLDBG((CE_WARN,
1286                             "[grp_]set_cpu: cpu out of range: %d\n", target));
1287                         return (PSM_FAILURE);
1288                 }
1289 
1290                 lock_set(&apix_lock);
1291 
1292                 vecp = apix_get_req_vector(hdlp, hdlp->ih_flags);
1293                 if (!IS_VECT_ENABLED(vecp)) {
1294                         DDI_INTR_IMPLDBG((CE_WARN,
1295                             "[grp]_set_cpu: invalid vector 0x%x\n",
1296                             hdlp->ih_vector));
1297                         lock_clear(&apix_lock);
1298                         return (PSM_FAILURE);
1299                 }
1300 
1301                 *result = 0;
1302 
1303                 if (intr_op == PSM_INTR_OP_SET_CPU)
1304                         newvecp = apix_set_cpu(vecp, target, result);
1305                 else
1306                         newvecp = apix_grp_set_cpu(vecp, target, result);
1307 
1308                 lock_clear(&apix_lock);
1309 
1310                 if (newvecp == NULL) {
1311                         *result = EIO;
1312                         return (PSM_FAILURE);
1313                 }
1314                 newvecp->v_bound_cpuid = target;
1315                 hdlp->ih_vector = APIX_VIRTVECTOR(newvecp->v_cpuid,
1316                     newvecp->v_vector);
1317                 break;
1318 
1319         case PSM_INTR_OP_GET_INTR:
1320                 /*
1321                  * The interrupt handle given here has been allocated
1322                  * specifically for this command, and ih_private carries
1323                  * a pointer to a apic_get_intr_t.
1324                  */
1325                 if (apix_get_intr_info(hdlp, hdlp->ih_private) != PSM_SUCCESS)
1326                         return (PSM_FAILURE);
1327                 break;
1328 
1329         case PSM_INTR_OP_CHECK_MSI:
1330                 /*
1331                  * Check MSI/X is supported or not at APIC level and
1332                  * masked off the MSI/X bits in hdlp->ih_type if not
1333                  * supported before return.  If MSI/X is supported,
1334                  * leave the ih_type unchanged and return.
1335                  *
1336                  * hdlp->ih_type passed in from the nexus has all the
1337                  * interrupt types supported by the device.
1338                  */
1339                 if (apic_support_msi == 0) {    /* uninitialized */
1340                         /*
1341                          * if apic_support_msi is not set, call
1342                          * apic_check_msi_support() to check whether msi
1343                          * is supported first
1344                          */
1345                         if (apic_check_msi_support() == PSM_SUCCESS)
1346                                 apic_support_msi = 1;   /* supported */
1347                         else
1348                                 apic_support_msi = -1;  /* not-supported */
1349                 }
1350                 if (apic_support_msi == 1) {
1351                         if (apic_msix_enable)
1352                                 *result = hdlp->ih_type;
1353                         else
1354                                 *result = hdlp->ih_type & ~DDI_INTR_TYPE_MSIX;
1355                 } else
1356                         *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI |
1357                             DDI_INTR_TYPE_MSIX);
1358                 break;
1359         case PSM_INTR_OP_GET_CAP:
1360                 cap = DDI_INTR_FLAG_PENDING;
1361                 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
1362                         cap |= DDI_INTR_FLAG_MASKABLE;
1363                 *result = cap;
1364                 break;
1365         case PSM_INTR_OP_APIC_TYPE:
1366                 ((apic_get_type_t *)(hdlp->ih_private))->avgi_type =
1367                     apix_get_apic_type();
1368                 ((apic_get_type_t *)(hdlp->ih_private))->avgi_num_intr =
1369                     APIX_IPI_MIN;
1370                 ((apic_get_type_t *)(hdlp->ih_private))->avgi_num_cpu =
1371                     apic_nproc;
1372                 hdlp->ih_ver = apic_get_apic_version();
1373                 break;
1374         case PSM_INTR_OP_SET_CAP:
1375         default:
1376                 return (PSM_FAILURE);
1377         }
1378 
1379         return (PSM_SUCCESS);
1380 }
1381 
1382 static void
1383 apix_cleanup_busy(void)
1384 {
1385         int i, j;
1386         apix_vector_t *vecp;
1387 
1388         for (i = 0; i < apic_nproc; i++) {
1389                 if (!apic_cpu_in_range(i))
1390                         continue;
1391                 apic_cpus[i].aci_busy = 0;
1392                 for (j = APIX_AVINTR_MIN; j < APIX_AVINTR_MAX; j++) {
1393                         if ((vecp = xv_vector(i, j)) != NULL)
1394                                 vecp->v_busy = 0;
1395                 }
1396         }
1397 }
1398 
1399 static void
1400 apix_redistribute_compute(void)
1401 {
1402         int     i, j, max_busy;
1403 
1404         if (!apic_enable_dynamic_migration)
1405                 return;
1406 
1407         if (++apic_nticks == apic_sample_factor_redistribution) {
1408                 /*
1409                  * Time to call apic_intr_redistribute().
1410                  * reset apic_nticks. This will cause max_busy
1411                  * to be calculated below and if it is more than
1412                  * apic_int_busy, we will do the whole thing
1413                  */
1414                 apic_nticks = 0;
1415         }
1416         max_busy = 0;
1417         for (i = 0; i < apic_nproc; i++) {
1418                 if (!apic_cpu_in_range(i))
1419                         continue;
1420                 /*
1421                  * Check if curipl is non zero & if ISR is in
1422                  * progress
1423                  */
1424                 if (((j = apic_cpus[i].aci_curipl) != 0) &&
1425                     (apic_cpus[i].aci_ISR_in_progress & (1 << j))) {
1426 
1427                         int     vect;
1428                         apic_cpus[i].aci_busy++;
1429                         vect = apic_cpus[i].aci_current[j];
1430                         apixs[i]->x_vectbl[vect]->v_busy++;
1431                 }
1432 
1433                 if (!apic_nticks &&
1434                     (apic_cpus[i].aci_busy > max_busy))
1435                         max_busy = apic_cpus[i].aci_busy;
1436         }
1437         if (!apic_nticks) {
1438                 if (max_busy > apic_int_busy_mark) {
1439                 /*
1440                  * We could make the following check be
1441                  * skipped > 1 in which case, we get a
1442                  * redistribution at half the busy mark (due to
1443                  * double interval). Need to be able to collect
1444                  * more empirical data to decide if that is a
1445                  * good strategy. Punt for now.
1446                  */
1447                         apix_cleanup_busy();
1448                         apic_skipped_redistribute = 0;
1449                 } else
1450                         apic_skipped_redistribute++;
1451         }
1452 }
1453 
1454 /*
1455  * intr_ops() service routines
1456  */
1457 
1458 static int
1459 apix_get_pending(apix_vector_t *vecp)
1460 {
1461         int bit, index, irr, pending;
1462 
1463         /* need to get on the bound cpu */
1464         mutex_enter(&cpu_lock);
1465         affinity_set(vecp->v_cpuid);
1466 
1467         index = vecp->v_vector / 32;
1468         bit = vecp->v_vector % 32;
1469         irr = apic_reg_ops->apic_read(APIC_IRR_REG + index);
1470 
1471         affinity_clear();
1472         mutex_exit(&cpu_lock);
1473 
1474         pending = (irr & (1 << bit)) ? 1 : 0;
1475         if (!pending && vecp->v_type == APIX_TYPE_FIXED)
1476                 pending = apix_intx_get_pending(vecp->v_inum);
1477 
1478         return (pending);
1479 }
1480 
1481 static apix_vector_t *
1482 apix_get_req_vector(ddi_intr_handle_impl_t *hdlp, ushort_t flags)
1483 {
1484         apix_vector_t *vecp;
1485         processorid_t cpuid;
1486         int32_t virt_vec = 0;
1487 
1488         switch (flags & PSMGI_INTRBY_FLAGS) {
1489         case PSMGI_INTRBY_IRQ:
1490                 return (apix_intx_get_vector(hdlp->ih_vector));
1491         case PSMGI_INTRBY_VEC:
1492                 virt_vec = (virt_vec == 0) ? hdlp->ih_vector : virt_vec;
1493 
1494                 cpuid = APIX_VIRTVEC_CPU(virt_vec);
1495                 if (!apic_cpu_in_range(cpuid))
1496                         return (NULL);
1497 
1498                 vecp = xv_vector(cpuid, APIX_VIRTVEC_VECTOR(virt_vec));
1499                 break;
1500         case PSMGI_INTRBY_DEFAULT:
1501                 vecp = apix_get_dev_map(hdlp->ih_dip, hdlp->ih_inum,
1502                     hdlp->ih_type);
1503                 break;
1504         default:
1505                 return (NULL);
1506         }
1507 
1508         return (vecp);
1509 }
1510 
1511 static int
1512 apix_get_intr_info(ddi_intr_handle_impl_t *hdlp,
1513     apic_get_intr_t *intr_params_p)
1514 {
1515         apix_vector_t *vecp;
1516         struct autovec *av_dev;
1517         int i;
1518 
1519         vecp = apix_get_req_vector(hdlp, intr_params_p->avgi_req_flags);
1520         if (IS_VECT_FREE(vecp)) {
1521                 intr_params_p->avgi_num_devs = 0;
1522                 intr_params_p->avgi_cpu_id = 0;
1523                 intr_params_p->avgi_req_flags = 0;
1524                 return (PSM_SUCCESS);
1525         }
1526 
1527         if (intr_params_p->avgi_req_flags & PSMGI_REQ_CPUID) {
1528                 intr_params_p->avgi_cpu_id = vecp->v_cpuid;
1529 
1530                 /* Return user bound info for intrd. */
1531                 if (intr_params_p->avgi_cpu_id & IRQ_USER_BOUND) {
1532                         intr_params_p->avgi_cpu_id &= ~IRQ_USER_BOUND;
1533                         intr_params_p->avgi_cpu_id |= PSMGI_CPU_USER_BOUND;
1534                 }
1535         }
1536 
1537         if (intr_params_p->avgi_req_flags & PSMGI_REQ_VECTOR)
1538                 intr_params_p->avgi_vector = vecp->v_vector;
1539 
1540         if (intr_params_p->avgi_req_flags &
1541             (PSMGI_REQ_NUM_DEVS | PSMGI_REQ_GET_DEVS))
1542                 /* Get number of devices from apic_irq table shared field. */
1543                 intr_params_p->avgi_num_devs = vecp->v_share;
1544 
1545         if (intr_params_p->avgi_req_flags &  PSMGI_REQ_GET_DEVS) {
1546 
1547                 intr_params_p->avgi_req_flags  |= PSMGI_REQ_NUM_DEVS;
1548 
1549                 /* Some devices have NULL dip.  Don't count these. */
1550                 if (intr_params_p->avgi_num_devs > 0) {
1551                         for (i = 0, av_dev = vecp->v_autovect; av_dev;
1552                             av_dev = av_dev->av_link) {
1553                                 if (av_dev->av_vector && av_dev->av_dip)
1554                                         i++;
1555                         }
1556                         intr_params_p->avgi_num_devs =
1557                             (uint8_t)MIN(intr_params_p->avgi_num_devs, i);
1558                 }
1559 
1560                 /* There are no viable dips to return. */
1561                 if (intr_params_p->avgi_num_devs == 0) {
1562                         intr_params_p->avgi_dip_list = NULL;
1563 
1564                 } else {        /* Return list of dips */
1565 
1566                         /* Allocate space in array for that number of devs. */
1567                         intr_params_p->avgi_dip_list = kmem_zalloc(
1568                             intr_params_p->avgi_num_devs *
1569                             sizeof (dev_info_t *),
1570                             KM_NOSLEEP);
1571                         if (intr_params_p->avgi_dip_list == NULL) {
1572                                 DDI_INTR_IMPLDBG((CE_WARN,
1573                                     "apix_get_vector_intr_info: no memory"));
1574                                 return (PSM_FAILURE);
1575                         }
1576 
1577                         /*
1578                          * Loop through the device list of the autovec table
1579                          * filling in the dip array.
1580                          *
1581                          * Note that the autovect table may have some special
1582                          * entries which contain NULL dips.  These will be
1583                          * ignored.
1584                          */
1585                         for (i = 0, av_dev = vecp->v_autovect; av_dev;
1586                             av_dev = av_dev->av_link) {
1587                                 if (av_dev->av_vector && av_dev->av_dip)
1588                                         intr_params_p->avgi_dip_list[i++] =
1589                                             av_dev->av_dip;
1590                         }
1591                 }
1592         }
1593 
1594         return (PSM_SUCCESS);
1595 }
1596 
1597 static char *
1598 apix_get_apic_type(void)
1599 {
1600         return (apix_psm_info.p_mach_idstring);
1601 }
1602 
1603 apix_vector_t *
1604 apix_set_cpu(apix_vector_t *vecp, int new_cpu, int *result)
1605 {
1606         apix_vector_t *newp = NULL;
1607         dev_info_t *dip;
1608         int inum, cap_ptr;
1609         ddi_acc_handle_t handle;
1610         ddi_intr_msix_t *msix_p = NULL;
1611         ushort_t msix_ctrl;
1612         uintptr_t off;
1613         uint32_t mask;
1614 
1615         ASSERT(LOCK_HELD(&apix_lock));
1616         *result = ENXIO;
1617 
1618         /* Fail if this is an MSI intr and is part of a group. */
1619         if (vecp->v_type == APIX_TYPE_MSI) {
1620                 if (i_ddi_intr_get_current_nintrs(APIX_GET_DIP(vecp)) > 1)
1621                         return (NULL);
1622                 else
1623                         return (apix_grp_set_cpu(vecp, new_cpu, result));
1624         }
1625 
1626         /*
1627          * Mask MSI-X. It's unmasked when MSI-X gets enabled.
1628          */
1629         if (vecp->v_type == APIX_TYPE_MSIX && IS_VECT_ENABLED(vecp)) {
1630                 if ((dip = APIX_GET_DIP(vecp)) == NULL)
1631                         return (NULL);
1632                 inum = vecp->v_devp->dv_inum;
1633 
1634                 handle = i_ddi_get_pci_config_handle(dip);
1635                 cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip);
1636                 msix_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSIX_CTRL);
1637                 if ((msix_ctrl & PCI_MSIX_FUNCTION_MASK) == 0) {
1638                         /*
1639                          * Function is not masked, then mask "inum"th
1640                          * entry in the MSI-X table
1641                          */
1642                         msix_p = i_ddi_get_msix(dip);
1643                         off = (uintptr_t)msix_p->msix_tbl_addr + (inum *
1644                             PCI_MSIX_VECTOR_SIZE) + PCI_MSIX_VECTOR_CTRL_OFFSET;
1645                         mask = ddi_get32(msix_p->msix_tbl_hdl, (uint32_t *)off);
1646                         ddi_put32(msix_p->msix_tbl_hdl, (uint32_t *)off,
1647                             mask | 1);
1648                 }
1649         }
1650 
1651         *result = 0;
1652         if ((newp = apix_rebind(vecp, new_cpu, 1)) == NULL)
1653                 *result = EIO;
1654 
1655         /* Restore mask bit */
1656         if (msix_p != NULL)
1657                 ddi_put32(msix_p->msix_tbl_hdl, (uint32_t *)off, mask);
1658 
1659         return (newp);
1660 }
1661 
1662 /*
1663  * Set cpu for MSIs
1664  */
1665 apix_vector_t *
1666 apix_grp_set_cpu(apix_vector_t *vecp, int new_cpu, int *result)
1667 {
1668         apix_vector_t *newp, *vp;
1669         uint32_t orig_cpu = vecp->v_cpuid;
1670         int orig_vect = vecp->v_vector;
1671         int i, num_vectors, cap_ptr, msi_mask_off;
1672         uint32_t msi_pvm;
1673         ushort_t msi_ctrl;
1674         ddi_acc_handle_t handle;
1675         dev_info_t *dip;
1676 
1677         APIC_VERBOSE(INTR, (CE_CONT, "apix_grp_set_cpu: oldcpu: %x, vector: %x,"
1678             " newcpu:%x\n", vecp->v_cpuid, vecp->v_vector, new_cpu));
1679 
1680         ASSERT(LOCK_HELD(&apix_lock));
1681 
1682         *result = ENXIO;
1683 
1684         if (vecp->v_type != APIX_TYPE_MSI) {
1685                 DDI_INTR_IMPLDBG((CE_WARN, "set_grp: intr not MSI\n"));
1686                 return (NULL);
1687         }
1688 
1689         if ((dip = APIX_GET_DIP(vecp)) == NULL)
1690                 return (NULL);
1691 
1692         num_vectors = i_ddi_intr_get_current_nintrs(dip);
1693         if ((num_vectors < 1) || ((num_vectors - 1) & orig_vect)) {
1694                 APIC_VERBOSE(INTR, (CE_WARN,
1695                     "set_grp: base vec not part of a grp or not aligned: "
1696                     "vec:0x%x, num_vec:0x%x\n", orig_vect, num_vectors));
1697                 return (NULL);
1698         }
1699 
1700         if (vecp->v_inum != apix_get_min_dev_inum(dip, vecp->v_type))
1701                 return (NULL);
1702 
1703         *result = EIO;
1704         for (i = 1; i < num_vectors; i++) {
1705                 if ((vp = xv_vector(orig_cpu, orig_vect + i)) == NULL)
1706                         return (NULL);
1707 #ifdef DEBUG
1708                 /*
1709                  * Sanity check: CPU and dip is the same for all entries.
1710                  * May be called when first msi to be enabled, at this time
1711                  * add_avintr() is not called for other msi
1712                  */
1713                 if ((vp->v_share != 0) &&
1714                     ((APIX_GET_DIP(vp) != dip) ||
1715                     (vp->v_cpuid != vecp->v_cpuid))) {
1716                         APIC_VERBOSE(INTR, (CE_WARN,
1717                             "set_grp: cpu or dip for vec 0x%x difft than for "
1718                             "vec 0x%x\n", orig_vect, orig_vect + i));
1719                         APIC_VERBOSE(INTR, (CE_WARN,
1720                             "  cpu: %d vs %d, dip: 0x%p vs 0x%p\n", orig_cpu,
1721                             vp->v_cpuid, (void *)dip,
1722                             (void *)APIX_GET_DIP(vp)));
1723                         return (NULL);
1724                 }
1725 #endif /* DEBUG */
1726         }
1727 
1728         cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip);
1729         handle = i_ddi_get_pci_config_handle(dip);
1730         msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
1731 
1732         /* MSI Per vector masking is supported. */
1733         if (msi_ctrl & PCI_MSI_PVM_MASK) {
1734                 if (msi_ctrl &  PCI_MSI_64BIT_MASK)
1735                         msi_mask_off = cap_ptr + PCI_MSI_64BIT_MASKBITS;
1736                 else
1737                         msi_mask_off = cap_ptr + PCI_MSI_32BIT_MASK;
1738                 msi_pvm = pci_config_get32(handle, msi_mask_off);
1739                 pci_config_put32(handle, msi_mask_off, (uint32_t)-1);
1740                 APIC_VERBOSE(INTR, (CE_CONT,
1741                     "set_grp: pvm supported.  Mask set to 0x%x\n",
1742                     pci_config_get32(handle, msi_mask_off)));
1743         }
1744 
1745         if ((newp = apix_rebind(vecp, new_cpu, num_vectors)) != NULL)
1746                 *result = 0;
1747 
1748         /* Reenable vectors if per vector masking is supported. */
1749         if (msi_ctrl & PCI_MSI_PVM_MASK) {
1750                 pci_config_put32(handle, msi_mask_off, msi_pvm);
1751                 APIC_VERBOSE(INTR, (CE_CONT,
1752                     "set_grp: pvm supported.  Mask restored to 0x%x\n",
1753                     pci_config_get32(handle, msi_mask_off)));
1754         }
1755 
1756         return (newp);
1757 }
1758 
1759 void
1760 apix_intx_set_vector(int irqno, uint32_t cpuid, uchar_t vector)
1761 {
1762         apic_irq_t *irqp;
1763 
1764         mutex_enter(&airq_mutex);
1765         irqp = apic_irq_table[irqno];
1766         irqp->airq_cpu = cpuid;
1767         irqp->airq_vector = vector;
1768         apic_record_rdt_entry(irqp, irqno);
1769         mutex_exit(&airq_mutex);
1770 }
1771 
1772 apix_vector_t *
1773 apix_intx_get_vector(int irqno)
1774 {
1775         apic_irq_t *irqp;
1776         uint32_t cpuid;
1777         uchar_t vector;
1778 
1779         mutex_enter(&airq_mutex);
1780         irqp = apic_irq_table[irqno & 0xff];
1781         if (IS_IRQ_FREE(irqp) || (irqp->airq_cpu == IRQ_UNINIT)) {
1782                 mutex_exit(&airq_mutex);
1783                 return (NULL);
1784         }
1785         cpuid = irqp->airq_cpu;
1786         vector = irqp->airq_vector;
1787         mutex_exit(&airq_mutex);
1788 
1789         return (xv_vector(cpuid, vector));
1790 }
1791 
1792 /*
1793  * Must called with interrupts disabled and apic_ioapic_lock held
1794  */
1795 void
1796 apix_intx_enable(int irqno)
1797 {
1798         uchar_t ioapicindex, intin;
1799         apic_irq_t *irqp = apic_irq_table[irqno];
1800         ioapic_rdt_t irdt;
1801         apic_cpus_info_t *cpu_infop;
1802         apix_vector_t *vecp = xv_vector(irqp->airq_cpu, irqp->airq_vector);
1803 
1804         ASSERT(LOCK_HELD(&apic_ioapic_lock) && !IS_IRQ_FREE(irqp));
1805 
1806         ioapicindex = irqp->airq_ioapicindex;
1807         intin = irqp->airq_intin_no;
1808         cpu_infop =  &apic_cpus[irqp->airq_cpu];
1809 
1810         irdt.ir_lo = AV_PDEST | AV_FIXED | irqp->airq_rdt_entry;
1811         irdt.ir_hi = cpu_infop->aci_local_id;
1812 
1813         apic_vt_ops->apic_intrmap_alloc_entry(&vecp->v_intrmap_private, NULL,
1814             vecp->v_type, 1, ioapicindex);
1815         apic_vt_ops->apic_intrmap_map_entry(vecp->v_intrmap_private,
1816             (void *)&irdt, vecp->v_type, 1);
1817         apic_vt_ops->apic_intrmap_record_rdt(vecp->v_intrmap_private, &irdt);
1818 
1819         /* write RDT entry high dword - destination */
1820         WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin,
1821             irdt.ir_hi);
1822 
1823         /* Write the vector, trigger, and polarity portion of the RDT */
1824         WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin, irdt.ir_lo);
1825 
1826         vecp->v_state = APIX_STATE_ENABLED;
1827 
1828         APIC_VERBOSE_IOAPIC((CE_CONT, "apix_intx_enable: ioapic 0x%x"
1829             " intin 0x%x rdt_low 0x%x rdt_high 0x%x\n",
1830             ioapicindex, intin, irdt.ir_lo, irdt.ir_hi));
1831 }
1832 
1833 /*
1834  * Must called with interrupts disabled and apic_ioapic_lock held
1835  */
1836 void
1837 apix_intx_disable(int irqno)
1838 {
1839         apic_irq_t *irqp = apic_irq_table[irqno];
1840         int ioapicindex, intin;
1841 
1842         ASSERT(LOCK_HELD(&apic_ioapic_lock) && !IS_IRQ_FREE(irqp));
1843         /*
1844          * The assumption here is that this is safe, even for
1845          * systems with IOAPICs that suffer from the hardware
1846          * erratum because all devices have been quiesced before
1847          * they unregister their interrupt handlers.  If that
1848          * assumption turns out to be false, this mask operation
1849          * can induce the same erratum result we're trying to
1850          * avoid.
1851          */
1852         ioapicindex = irqp->airq_ioapicindex;
1853         intin = irqp->airq_intin_no;
1854         ioapic_write(ioapicindex, APIC_RDT_CMD + 2 * intin, AV_MASK);
1855 
1856         APIC_VERBOSE_IOAPIC((CE_CONT, "apix_intx_disable: ioapic 0x%x"
1857             " intin 0x%x\n", ioapicindex, intin));
1858 }
1859 
1860 void
1861 apix_intx_free(int irqno)
1862 {
1863         apic_irq_t *irqp;
1864 
1865         mutex_enter(&airq_mutex);
1866         irqp = apic_irq_table[irqno];
1867 
1868         if (IS_IRQ_FREE(irqp)) {
1869                 mutex_exit(&airq_mutex);
1870                 return;
1871         }
1872 
1873         irqp->airq_mps_intr_index = FREE_INDEX;
1874         irqp->airq_cpu = IRQ_UNINIT;
1875         irqp->airq_vector = APIX_INVALID_VECT;
1876         mutex_exit(&airq_mutex);
1877 }
1878 
1879 #ifdef DEBUG
1880 int apix_intr_deliver_timeouts = 0;
1881 int apix_intr_rirr_timeouts = 0;
1882 int apix_intr_rirr_reset_failure = 0;
1883 #endif
1884 int apix_max_reps_irr_pending = 10;
1885 
1886 #define GET_RDT_BITS(ioapic, intin, bits)       \
1887         (READ_IOAPIC_RDT_ENTRY_LOW_DWORD((ioapic), (intin)) & (bits))
1888 #define APIX_CHECK_IRR_DELAY    drv_usectohz(5000)
1889 
1890 int
1891 apix_intx_rebind(int irqno, processorid_t cpuid, uchar_t vector)
1892 {
1893         apic_irq_t *irqp = apic_irq_table[irqno];
1894         ulong_t iflag;
1895         int waited, ioapic_ix, intin_no, level, repeats, rdt_entry, masked;
1896 
1897         ASSERT(irqp != NULL);
1898 
1899         iflag = intr_clear();
1900         lock_set(&apic_ioapic_lock);
1901 
1902         ioapic_ix = irqp->airq_ioapicindex;
1903         intin_no = irqp->airq_intin_no;
1904         level = apic_level_intr[irqno];
1905 
1906         /*
1907          * Wait for the delivery status bit to be cleared. This should
1908          * be a very small amount of time.
1909          */
1910         repeats = 0;
1911         do {
1912                 repeats++;
1913 
1914                 for (waited = 0; waited < apic_max_reps_clear_pending;
1915                     waited++) {
1916                         if (GET_RDT_BITS(ioapic_ix, intin_no, AV_PENDING) == 0)
1917                                 break;
1918                 }
1919                 if (!level)
1920                         break;
1921 
1922                 /*
1923                  * Mask the RDT entry for level-triggered interrupts.
1924                  */
1925                 irqp->airq_rdt_entry |= AV_MASK;
1926                 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1927                     intin_no);
1928                 if ((masked = (rdt_entry & AV_MASK)) == 0) {
1929                         /* Mask it */
1930                         WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no,
1931                             AV_MASK | rdt_entry);
1932                 }
1933 
1934                 /*
1935                  * If there was a race and an interrupt was injected
1936                  * just before we masked, check for that case here.
1937                  * Then, unmask the RDT entry and try again.  If we're
1938                  * on our last try, don't unmask (because we want the
1939                  * RDT entry to remain masked for the rest of the
1940                  * function).
1941                  */
1942                 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1943                     intin_no);
1944                 if ((masked == 0) && ((rdt_entry & AV_PENDING) != 0) &&
1945                     (repeats < apic_max_reps_clear_pending)) {
1946                         /* Unmask it */
1947                         WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1948                             intin_no, rdt_entry & ~AV_MASK);
1949                         irqp->airq_rdt_entry &= ~AV_MASK;
1950                 }
1951         } while ((rdt_entry & AV_PENDING) &&
1952             (repeats < apic_max_reps_clear_pending));
1953 
1954 #ifdef DEBUG
1955         if (GET_RDT_BITS(ioapic_ix, intin_no, AV_PENDING) != 0)
1956                 apix_intr_deliver_timeouts++;
1957 #endif
1958 
1959         if (!level || !APIX_IS_MASK_RDT(apix_mul_ioapic_method))
1960                 goto done;
1961 
1962         /*
1963          * wait for remote IRR to be cleared for level-triggered
1964          * interrupts
1965          */
1966         repeats = 0;
1967         do {
1968                 repeats++;
1969 
1970                 for (waited = 0; waited < apic_max_reps_clear_pending;
1971                     waited++) {
1972                         if (GET_RDT_BITS(ioapic_ix, intin_no, AV_REMOTE_IRR)
1973                             == 0)
1974                                 break;
1975                 }
1976 
1977                 if (GET_RDT_BITS(ioapic_ix, intin_no, AV_REMOTE_IRR) != 0) {
1978                         lock_clear(&apic_ioapic_lock);
1979                         intr_restore(iflag);
1980 
1981                         delay(APIX_CHECK_IRR_DELAY);
1982 
1983                         iflag = intr_clear();
1984                         lock_set(&apic_ioapic_lock);
1985                 }
1986         } while (repeats < apix_max_reps_irr_pending);
1987 
1988         if (repeats >= apix_max_reps_irr_pending) {
1989 #ifdef DEBUG
1990                 apix_intr_rirr_timeouts++;
1991 #endif
1992 
1993                 /*
1994                  * If we waited and the Remote IRR bit is still not cleared,
1995                  * AND if we've invoked the timeout APIC_REPROGRAM_MAX_TIMEOUTS
1996                  * times for this interrupt, try the last-ditch workaround:
1997                  */
1998                 if (GET_RDT_BITS(ioapic_ix, intin_no, AV_REMOTE_IRR) != 0) {
1999                         /*
2000                          * Trying to clear the bit through normal
2001                          * channels has failed.  So as a last-ditch
2002                          * effort, try to set the trigger mode to
2003                          * edge, then to level.  This has been
2004                          * observed to work on many systems.
2005                          */
2006                         WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
2007                             intin_no,
2008                             READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
2009                             intin_no) & ~AV_LEVEL);
2010                         WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
2011                             intin_no,
2012                             READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
2013                             intin_no) | AV_LEVEL);
2014                 }
2015 
2016                 if (GET_RDT_BITS(ioapic_ix, intin_no, AV_REMOTE_IRR) != 0) {
2017 #ifdef DEBUG
2018                         apix_intr_rirr_reset_failure++;
2019 #endif
2020                         lock_clear(&apic_ioapic_lock);
2021                         intr_restore(iflag);
2022                         prom_printf("apix: Remote IRR still "
2023                             "not clear for IOAPIC %d intin %d.\n"
2024                             "\tInterrupts to this pin may cease "
2025                             "functioning.\n", ioapic_ix, intin_no);
2026                         return (1);     /* return failure */
2027                 }
2028         }
2029 
2030 done:
2031         /* change apic_irq_table */
2032         lock_clear(&apic_ioapic_lock);
2033         intr_restore(iflag);
2034         apix_intx_set_vector(irqno, cpuid, vector);
2035         iflag = intr_clear();
2036         lock_set(&apic_ioapic_lock);
2037 
2038         /* reprogramme IO-APIC RDT entry */
2039         apix_intx_enable(irqno);
2040 
2041         lock_clear(&apic_ioapic_lock);
2042         intr_restore(iflag);
2043 
2044         return (0);
2045 }
2046 
2047 static int
2048 apix_intx_get_pending(int irqno)
2049 {
2050         apic_irq_t *irqp;
2051         int intin, ioapicindex, pending;
2052         ulong_t iflag;
2053 
2054         mutex_enter(&airq_mutex);
2055         irqp = apic_irq_table[irqno];
2056         if (IS_IRQ_FREE(irqp)) {
2057                 mutex_exit(&airq_mutex);
2058                 return (0);
2059         }
2060 
2061         /* check IO-APIC delivery status */
2062         intin = irqp->airq_intin_no;
2063         ioapicindex = irqp->airq_ioapicindex;
2064         mutex_exit(&airq_mutex);
2065 
2066         iflag = intr_clear();
2067         lock_set(&apic_ioapic_lock);
2068 
2069         pending = (READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin) &
2070             AV_PENDING) ? 1 : 0;
2071 
2072         lock_clear(&apic_ioapic_lock);
2073         intr_restore(iflag);
2074 
2075         return (pending);
2076 }
2077 
2078 /*
2079  * This function will mask the interrupt on the I/O APIC
2080  */
2081 static void
2082 apix_intx_set_mask(int irqno)
2083 {
2084         int intin, ioapixindex, rdt_entry;
2085         ulong_t iflag;
2086         apic_irq_t *irqp;
2087 
2088         mutex_enter(&airq_mutex);
2089         irqp = apic_irq_table[irqno];
2090 
2091         ASSERT(irqp->airq_mps_intr_index != FREE_INDEX);
2092 
2093         intin = irqp->airq_intin_no;
2094         ioapixindex = irqp->airq_ioapicindex;
2095         mutex_exit(&airq_mutex);
2096 
2097         iflag = intr_clear();
2098         lock_set(&apic_ioapic_lock);
2099 
2100         rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapixindex, intin);
2101 
2102         /* clear mask */
2103         WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapixindex, intin,
2104             (AV_MASK | rdt_entry));
2105 
2106         lock_clear(&apic_ioapic_lock);
2107         intr_restore(iflag);
2108 }
2109 
2110 /*
2111  * This function will clear the mask for the interrupt on the I/O APIC
2112  */
2113 static void
2114 apix_intx_clear_mask(int irqno)
2115 {
2116         int intin, ioapixindex, rdt_entry;
2117         ulong_t iflag;
2118         apic_irq_t *irqp;
2119 
2120         mutex_enter(&airq_mutex);
2121         irqp = apic_irq_table[irqno];
2122 
2123         ASSERT(irqp->airq_mps_intr_index != FREE_INDEX);
2124 
2125         intin = irqp->airq_intin_no;
2126         ioapixindex = irqp->airq_ioapicindex;
2127         mutex_exit(&airq_mutex);
2128 
2129         iflag = intr_clear();
2130         lock_set(&apic_ioapic_lock);
2131 
2132         rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapixindex, intin);
2133 
2134         /* clear mask */
2135         WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapixindex, intin,
2136             ((~AV_MASK) & rdt_entry));
2137 
2138         lock_clear(&apic_ioapic_lock);
2139         intr_restore(iflag);
2140 }
2141 
2142 /*
2143  * For level-triggered interrupt, mask the IRQ line. Mask means
2144  * new interrupts will not be delivered. The interrupt already
2145  * accepted by a local APIC is not affected
2146  */
2147 void
2148 apix_level_intr_pre_eoi(int irq)
2149 {
2150         apic_irq_t *irqp = apic_irq_table[irq];
2151         int apic_ix, intin_ix;
2152 
2153         if (irqp == NULL)
2154                 return;
2155 
2156         ASSERT(apic_level_intr[irq] == TRIGGER_MODE_LEVEL);
2157 
2158         lock_set(&apic_ioapic_lock);
2159 
2160         intin_ix = irqp->airq_intin_no;
2161         apic_ix = irqp->airq_ioapicindex;
2162 
2163         if (irqp->airq_cpu != CPU->cpu_id) {
2164                 if (!APIX_IS_MASK_RDT(apix_mul_ioapic_method))
2165                         ioapic_write_eoi(apic_ix, irqp->airq_vector);
2166                 lock_clear(&apic_ioapic_lock);
2167                 return;
2168         }
2169 
2170         if (apix_mul_ioapic_method == APIC_MUL_IOAPIC_IOXAPIC) {
2171                 /*
2172                  * This is a IOxAPIC and there is EOI register:
2173                  *      Change the vector to reserved unused vector, so that
2174                  *      the EOI from Local APIC won't clear the Remote IRR for
2175                  *      this level trigger interrupt. Instead, we'll manually
2176                  *      clear it in apix_post_hardint() after ISR handling.
2177                  */
2178                 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(apic_ix, intin_ix,
2179                     (irqp->airq_rdt_entry & (~0xff)) | APIX_RESV_VECTOR);
2180         } else {
2181                 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(apic_ix, intin_ix,
2182                     AV_MASK | irqp->airq_rdt_entry);
2183         }
2184 
2185         lock_clear(&apic_ioapic_lock);
2186 }
2187 
2188 /*
2189  * For level-triggered interrupt, unmask the IRQ line
2190  * or restore the original vector number.
2191  */
2192 void
2193 apix_level_intr_post_dispatch(int irq)
2194 {
2195         apic_irq_t *irqp = apic_irq_table[irq];
2196         int apic_ix, intin_ix;
2197 
2198         if (irqp == NULL)
2199                 return;
2200 
2201         lock_set(&apic_ioapic_lock);
2202 
2203         intin_ix = irqp->airq_intin_no;
2204         apic_ix = irqp->airq_ioapicindex;
2205 
2206         if (APIX_IS_DIRECTED_EOI(apix_mul_ioapic_method)) {
2207                 /*
2208                  * Already sent EOI back to Local APIC.
2209                  * Send EOI to IO-APIC
2210                  */
2211                 ioapic_write_eoi(apic_ix, irqp->airq_vector);
2212         } else {
2213                 /* clear the mask or restore the vector */
2214                 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(apic_ix, intin_ix,
2215                     irqp->airq_rdt_entry);
2216 
2217                 /* send EOI to IOxAPIC */
2218                 if (apix_mul_ioapic_method == APIC_MUL_IOAPIC_IOXAPIC)
2219                         ioapic_write_eoi(apic_ix, irqp->airq_vector);
2220         }
2221 
2222         lock_clear(&apic_ioapic_lock);
2223 }
2224 
2225 static int
2226 apix_intx_get_shared(int irqno)
2227 {
2228         apic_irq_t *irqp;
2229         int share;
2230 
2231         mutex_enter(&airq_mutex);
2232         irqp = apic_irq_table[irqno];
2233         if (IS_IRQ_FREE(irqp) || (irqp->airq_cpu == IRQ_UNINIT)) {
2234                 mutex_exit(&airq_mutex);
2235                 return (0);
2236         }
2237         share = irqp->airq_share;
2238         mutex_exit(&airq_mutex);
2239 
2240         return (share);
2241 }
2242 
2243 static void
2244 apix_intx_set_shared(int irqno, int delta)
2245 {
2246         apic_irq_t *irqp;
2247 
2248         mutex_enter(&airq_mutex);
2249         irqp = apic_irq_table[irqno];
2250         if (IS_IRQ_FREE(irqp)) {
2251                 mutex_exit(&airq_mutex);
2252                 return;
2253         }
2254         irqp->airq_share += delta;
2255         mutex_exit(&airq_mutex);
2256 }
2257 
2258 /*
2259  * Setup IRQ table. Return IRQ no or -1 on failure
2260  */
2261 static int
2262 apix_intx_setup(dev_info_t *dip, int inum, int irqno,
2263     struct apic_io_intr *intrp, struct intrspec *ispec, iflag_t *iflagp)
2264 {
2265         int origirq = ispec->intrspec_vec;
2266         int newirq;
2267         short intr_index;
2268         uchar_t ipin, ioapic, ioapicindex;
2269         apic_irq_t *irqp;
2270 
2271         UNREFERENCED_1PARAMETER(inum);
2272 
2273         if (intrp != NULL) {
2274                 intr_index = (short)(intrp - apic_io_intrp);
2275                 ioapic = intrp->intr_destid;
2276                 ipin = intrp->intr_destintin;
2277 
2278                 /* Find ioapicindex. If destid was ALL, we will exit with 0. */
2279                 for (ioapicindex = apic_io_max - 1; ioapicindex; ioapicindex--)
2280                         if (apic_io_id[ioapicindex] == ioapic)
2281                                 break;
2282                 ASSERT((ioapic == apic_io_id[ioapicindex]) ||
2283                     (ioapic == INTR_ALL_APIC));
2284 
2285                 /* check whether this intin# has been used by another irqno */
2286                 if ((newirq = apic_find_intin(ioapicindex, ipin)) != -1)
2287                         return (newirq);
2288 
2289         } else if (iflagp != NULL) {    /* ACPI */
2290                 intr_index = ACPI_INDEX;
2291                 ioapicindex = acpi_find_ioapic(irqno);
2292                 ASSERT(ioapicindex != 0xFF);
2293                 ioapic = apic_io_id[ioapicindex];
2294                 ipin = irqno - apic_io_vectbase[ioapicindex];
2295 
2296                 if (apic_irq_table[irqno] &&
2297                     apic_irq_table[irqno]->airq_mps_intr_index == ACPI_INDEX) {
2298                         ASSERT(apic_irq_table[irqno]->airq_intin_no == ipin &&
2299                             apic_irq_table[irqno]->airq_ioapicindex ==
2300                             ioapicindex);
2301                         return (irqno);
2302                 }
2303 
2304         } else {        /* default configuration */
2305                 intr_index = DEFAULT_INDEX;
2306                 ioapicindex = 0;
2307                 ioapic = apic_io_id[ioapicindex];
2308                 ipin = (uchar_t)irqno;
2309         }
2310 
2311         /* allocate a new IRQ no */
2312         if ((irqp = apic_irq_table[irqno]) == NULL) {
2313                 irqp = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
2314                 apic_irq_table[irqno] = irqp;
2315         } else {
2316                 if (irqp->airq_mps_intr_index != FREE_INDEX) {
2317                         newirq = apic_allocate_irq(apic_first_avail_irq);
2318                         if (newirq == -1) {
2319                                 return (-1);
2320                         }
2321                         irqno = newirq;
2322                         irqp = apic_irq_table[irqno];
2323                         ASSERT(irqp != NULL);
2324                 }
2325         }
2326         apic_max_device_irq = max(irqno, apic_max_device_irq);
2327         apic_min_device_irq = min(irqno, apic_min_device_irq);
2328 
2329         irqp->airq_mps_intr_index = intr_index;
2330         irqp->airq_ioapicindex = ioapicindex;
2331         irqp->airq_intin_no = ipin;
2332         irqp->airq_dip = dip;
2333         irqp->airq_origirq = (uchar_t)origirq;
2334         if (iflagp != NULL)
2335                 irqp->airq_iflag = *iflagp;
2336         irqp->airq_cpu = IRQ_UNINIT;
2337         irqp->airq_vector = 0;
2338 
2339         return (irqno);
2340 }
2341 
2342 /*
2343  * Setup IRQ table for non-pci devices. Return IRQ no or -1 on error
2344  */
2345 static int
2346 apix_intx_setup_nonpci(dev_info_t *dip, int inum, int bustype,
2347     struct intrspec *ispec)
2348 {
2349         int irqno = ispec->intrspec_vec;
2350         int newirq, i;
2351         iflag_t intr_flag;
2352         ACPI_SUBTABLE_HEADER    *hp;
2353         ACPI_MADT_INTERRUPT_OVERRIDE *isop;
2354         struct apic_io_intr *intrp;
2355 
2356         if (!apic_enable_acpi || apic_use_acpi_madt_only) {
2357                 int busid;
2358 
2359                 if (bustype == 0)
2360                         bustype = eisa_level_intr_mask ? BUS_EISA : BUS_ISA;
2361 
2362                 /* loop checking BUS_ISA/BUS_EISA */
2363                 for (i = 0; i < 2; i++) {
2364                         if (((busid = apic_find_bus_id(bustype)) != -1) &&
2365                             ((intrp = apic_find_io_intr_w_busid(irqno, busid))
2366                             != NULL)) {
2367                                 return (apix_intx_setup(dip, inum, irqno,
2368                                     intrp, ispec, NULL));
2369                         }
2370                         bustype = (bustype == BUS_EISA) ? BUS_ISA : BUS_EISA;
2371                 }
2372 
2373                 /* fall back to default configuration */
2374                 return (-1);
2375         }
2376 
2377         /* search iso entries first */
2378         if (acpi_iso_cnt != 0) {
2379                 hp = (ACPI_SUBTABLE_HEADER *)acpi_isop;
2380                 i = 0;
2381                 while (i < acpi_iso_cnt) {
2382                         if (hp->Type == ACPI_MADT_TYPE_INTERRUPT_OVERRIDE) {
2383                                 isop = (ACPI_MADT_INTERRUPT_OVERRIDE *) hp;
2384                                 if (isop->Bus == 0 &&
2385                                     isop->SourceIrq == irqno) {
2386                                         newirq = isop->GlobalIrq;
2387                                         intr_flag.intr_po = isop->IntiFlags &
2388                                             ACPI_MADT_POLARITY_MASK;
2389                                         intr_flag.intr_el = (isop->IntiFlags &
2390                                             ACPI_MADT_TRIGGER_MASK) >> 2;
2391                                         intr_flag.bustype = BUS_ISA;
2392 
2393                                         return (apix_intx_setup(dip, inum,
2394                                             newirq, NULL, ispec, &intr_flag));
2395                                 }
2396                                 i++;
2397                         }
2398                         hp = (ACPI_SUBTABLE_HEADER *)(((char *)hp) +
2399                             hp->Length);
2400                 }
2401         }
2402         intr_flag.intr_po = INTR_PO_ACTIVE_HIGH;
2403         intr_flag.intr_el = INTR_EL_EDGE;
2404         intr_flag.bustype = BUS_ISA;
2405         return (apix_intx_setup(dip, inum, irqno, NULL, ispec, &intr_flag));
2406 }
2407 
2408 
2409 /*
2410  * Setup IRQ table for pci devices. Return IRQ no or -1 on error
2411  */
2412 static int
2413 apix_intx_setup_pci(dev_info_t *dip, int inum, int bustype,
2414     struct intrspec *ispec)
2415 {
2416         int busid, devid, pci_irq;
2417         ddi_acc_handle_t cfg_handle;
2418         uchar_t ipin;
2419         iflag_t intr_flag;
2420         struct apic_io_intr *intrp;
2421 
2422         if (acpica_get_bdf(dip, &busid, &devid, NULL) != 0)
2423                 return (-1);
2424 
2425         if (busid == 0 && apic_pci_bus_total == 1)
2426                 busid = (int)apic_single_pci_busid;
2427 
2428         if (pci_config_setup(dip, &cfg_handle) != DDI_SUCCESS)
2429                 return (-1);
2430         ipin = pci_config_get8(cfg_handle, PCI_CONF_IPIN) - PCI_INTA;
2431         pci_config_teardown(&cfg_handle);
2432 
2433         if (apic_enable_acpi && !apic_use_acpi_madt_only) {     /* ACPI */
2434                 if (apic_acpi_translate_pci_irq(dip, busid, devid,
2435                     ipin, &pci_irq, &intr_flag) != ACPI_PSM_SUCCESS)
2436                         return (-1);
2437 
2438                 intr_flag.bustype = (uchar_t)bustype;
2439                 return (apix_intx_setup(dip, inum, pci_irq, NULL, ispec,
2440                     &intr_flag));
2441         }
2442 
2443         /* MP configuration table */
2444         pci_irq = ((devid & 0x1f) << 2) | (ipin & 0x3);
2445         if ((intrp = apic_find_io_intr_w_busid(pci_irq, busid)) == NULL) {
2446                 pci_irq = apic_handle_pci_pci_bridge(dip, devid, ipin, &intrp);
2447                 if (pci_irq == -1)
2448                         return (-1);
2449         }
2450 
2451         return (apix_intx_setup(dip, inum, pci_irq, intrp, ispec, NULL));
2452 }
2453 
2454 /*
2455  * Translate and return IRQ no
2456  */
2457 static int
2458 apix_intx_xlate_irq(dev_info_t *dip, int inum, struct intrspec *ispec)
2459 {
2460         int newirq, irqno = ispec->intrspec_vec;
2461         int parent_is_pci_or_pciex = 0, child_is_pciex = 0;
2462         int bustype = 0, dev_len;
2463         char dev_type[16];
2464 
2465         if (apic_defconf) {
2466                 mutex_enter(&airq_mutex);
2467                 goto defconf;
2468         }
2469 
2470         if ((dip == NULL) || (!apic_irq_translate && !apic_enable_acpi)) {
2471                 mutex_enter(&airq_mutex);
2472                 goto nonpci;
2473         }
2474 
2475         /*
2476          * use ddi_getlongprop_buf() instead of ddi_prop_lookup_string()
2477          * to avoid extra buffer allocation.
2478          */
2479         dev_len = sizeof (dev_type);
2480         if (ddi_getlongprop_buf(DDI_DEV_T_ANY, ddi_get_parent(dip),
2481             DDI_PROP_DONTPASS, "device_type", (caddr_t)dev_type,
2482             &dev_len) == DDI_PROP_SUCCESS) {
2483                 if ((strcmp(dev_type, "pci") == 0) ||
2484                     (strcmp(dev_type, "pciex") == 0))
2485                         parent_is_pci_or_pciex = 1;
2486         }
2487 
2488         if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip,
2489             DDI_PROP_DONTPASS, "compatible", (caddr_t)dev_type,
2490             &dev_len) == DDI_PROP_SUCCESS) {
2491                 if (strstr(dev_type, "pciex"))
2492                         child_is_pciex = 1;
2493         }
2494 
2495         mutex_enter(&airq_mutex);
2496 
2497         if (parent_is_pci_or_pciex) {
2498                 bustype = child_is_pciex ? BUS_PCIE : BUS_PCI;
2499                 newirq = apix_intx_setup_pci(dip, inum, bustype, ispec);
2500                 if (newirq != -1)
2501                         goto done;
2502                 bustype = 0;
2503         } else if (strcmp(dev_type, "isa") == 0)
2504                 bustype = BUS_ISA;
2505         else if (strcmp(dev_type, "eisa") == 0)
2506                 bustype = BUS_EISA;
2507 
2508 nonpci:
2509         newirq = apix_intx_setup_nonpci(dip, inum, bustype, ispec);
2510         if (newirq != -1)
2511                 goto done;
2512 
2513 defconf:
2514         newirq = apix_intx_setup(dip, inum, irqno, NULL, ispec, NULL);
2515         if (newirq == -1) {
2516                 mutex_exit(&airq_mutex);
2517                 return (-1);
2518         }
2519 done:
2520         ASSERT(apic_irq_table[newirq]);
2521         mutex_exit(&airq_mutex);
2522         return (newirq);
2523 }
2524 
2525 static int
2526 apix_intx_alloc_vector(dev_info_t *dip, int inum, struct intrspec *ispec)
2527 {
2528         int irqno;
2529         apix_vector_t *vecp;
2530 
2531         if ((irqno = apix_intx_xlate_irq(dip, inum, ispec)) == -1)
2532                 return (0);
2533 
2534         if ((vecp = apix_alloc_intx(dip, inum, irqno)) == NULL)
2535                 return (0);
2536 
2537         DDI_INTR_IMPLDBG((CE_CONT, "apix_intx_alloc_vector: dip=0x%p name=%s "
2538             "irqno=0x%x cpuid=%d vector=0x%x\n",
2539             (void *)dip, ddi_driver_name(dip), irqno,
2540             vecp->v_cpuid, vecp->v_vector));
2541 
2542         return (1);
2543 }
2544 
2545 /*
2546  * Return the vector number if the translated IRQ for this device
2547  * has a vector mapping setup. If no IRQ setup exists or no vector is
2548  * allocated to it then return 0.
2549  */
2550 static apix_vector_t *
2551 apix_intx_xlate_vector(dev_info_t *dip, int inum, struct intrspec *ispec)
2552 {
2553         int irqno;
2554         apix_vector_t *vecp;
2555 
2556         /* get the IRQ number */
2557         if ((irqno = apix_intx_xlate_irq(dip, inum, ispec)) == -1)
2558                 return (NULL);
2559 
2560         /* get the vector number if a vector is allocated to this irqno */
2561         vecp = apix_intx_get_vector(irqno);
2562 
2563         return (vecp);
2564 }
2565 
2566 /*
2567  * Switch between safe and x2APIC IPI sending method.
2568  * The CPU may power on in xapic mode or x2apic mode. If the CPU needs to send
2569  * an IPI to other CPUs before entering x2APIC mode, it still needs to use the
2570  * xAPIC method. Before sending a StartIPI to the target CPU, psm_send_ipi will
2571  * be changed to apic_common_send_ipi, which detects current local APIC mode and
2572  * use the right method to send an IPI. If some CPUs fail to start up,
2573  * apic_poweron_cnt won't return to zero, so apic_common_send_ipi will always be
2574  * used. psm_send_ipi can't be simply changed back to x2apic_send_ipi if some
2575  * CPUs failed to start up because those failed CPUs may recover itself later at
2576  * unpredictable time.
2577  */
2578 void
2579 apic_switch_ipi_callback(boolean_t enter)
2580 {
2581         ulong_t iflag;
2582         struct psm_ops *pops = psmops;
2583 
2584         iflag = intr_clear();
2585         lock_set(&apic_mode_switch_lock);
2586         if (enter) {
2587                 ASSERT(apic_poweron_cnt >= 0);
2588                 if (apic_poweron_cnt == 0) {
2589                         pops->psm_send_ipi = apic_common_send_ipi;
2590                         send_dirintf = pops->psm_send_ipi;
2591                 }
2592                 apic_poweron_cnt++;
2593         } else {
2594                 ASSERT(apic_poweron_cnt > 0);
2595                 apic_poweron_cnt--;
2596                 if (apic_poweron_cnt == 0) {
2597                         pops->psm_send_ipi = x2apic_send_ipi;
2598                         send_dirintf = pops->psm_send_ipi;
2599                 }
2600         }
2601         lock_clear(&apic_mode_switch_lock);
2602         intr_restore(iflag);
2603 }
2604 
2605 /* stub function */
2606 int
2607 apix_loaded(void)
2608 {
2609         return (apix_is_enabled);
2610 }