1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright (c) 2011 by Delphix. All rights reserved.
  24  */
  25 /*
  26  * Copyright (c) 2010, Intel Corporation.
  27  * All rights reserved.
  28  */
  29 /*
  30  * Portions Copyright 2009 Advanced Micro Devices, Inc.
  31  */
  32 /*
  33  * Copyright (c) 2011, Joyent, Inc. All rights reserved.
  34  */
  35 /*
  36  * Various routines to handle identification
  37  * and classification of x86 processors.
  38  */
  39 
  40 #include <sys/types.h>
  41 #include <sys/archsystm.h>
  42 #include <sys/x86_archext.h>
  43 #include <sys/kmem.h>
  44 #include <sys/systm.h>
  45 #include <sys/cmn_err.h>
  46 #include <sys/sunddi.h>
  47 #include <sys/sunndi.h>
  48 #include <sys/cpuvar.h>
  49 #include <sys/processor.h>
  50 #include <sys/sysmacros.h>
  51 #include <sys/pg.h>
  52 #include <sys/fp.h>
  53 #include <sys/controlregs.h>
  54 #include <sys/bitmap.h>
  55 #include <sys/auxv_386.h>
  56 #include <sys/memnode.h>
  57 #include <sys/pci_cfgspace.h>
  58 
  59 #ifdef __xpv
  60 #include <sys/hypervisor.h>
  61 #else
  62 #include <sys/ontrap.h>
  63 #endif
  64 
  65 /*
  66  * Pass 0 of cpuid feature analysis happens in locore. It contains special code
  67  * to recognize Cyrix processors that are not cpuid-compliant, and to deal with
  68  * them accordingly. For most modern processors, feature detection occurs here
  69  * in pass 1.
  70  *
  71  * Pass 1 of cpuid feature analysis happens just at the beginning of mlsetup()
  72  * for the boot CPU and does the basic analysis that the early kernel needs.
  73  * x86_featureset is set based on the return value of cpuid_pass1() of the boot
  74  * CPU.
  75  *
  76  * Pass 1 includes:
  77  *
  78  *      o Determining vendor/model/family/stepping and setting x86_type and
  79  *        x86_vendor accordingly.
  80  *      o Processing the feature flags returned by the cpuid instruction while
  81  *        applying any workarounds or tricks for the specific processor.
  82  *      o Mapping the feature flags into Solaris feature bits (X86_*).
  83  *      o Processing extended feature flags if supported by the processor,
  84  *        again while applying specific processor knowledge.
  85  *      o Determining the CMT characteristics of the system.
  86  *
  87  * Pass 1 is done on non-boot CPUs during their initialization and the results
  88  * are used only as a meager attempt at ensuring that all processors within the
  89  * system support the same features.
  90  *
  91  * Pass 2 of cpuid feature analysis happens just at the beginning
  92  * of startup().  It just copies in and corrects the remainder
  93  * of the cpuid data we depend on: standard cpuid functions that we didn't
  94  * need for pass1 feature analysis, and extended cpuid functions beyond the
  95  * simple feature processing done in pass1.
  96  *
  97  * Pass 3 of cpuid analysis is invoked after basic kernel services; in
  98  * particular kernel memory allocation has been made available. It creates a
  99  * readable brand string based on the data collected in the first two passes.
 100  *
 101  * Pass 4 of cpuid analysis is invoked after post_startup() when all
 102  * the support infrastructure for various hardware features has been
 103  * initialized. It determines which processor features will be reported
 104  * to userland via the aux vector.
 105  *
 106  * All passes are executed on all CPUs, but only the boot CPU determines what
 107  * features the kernel will use.
 108  *
 109  * Much of the worst junk in this file is for the support of processors
 110  * that didn't really implement the cpuid instruction properly.
 111  *
 112  * NOTE: The accessor functions (cpuid_get*) are aware of, and ASSERT upon,
 113  * the pass numbers.  Accordingly, changes to the pass code may require changes
 114  * to the accessor code.
 115  */
 116 
 117 uint_t x86_vendor = X86_VENDOR_IntelClone;
 118 uint_t x86_type = X86_TYPE_OTHER;
 119 uint_t x86_clflush_size = 0;
 120 
 121 uint_t pentiumpro_bug4046376;
 122 uint_t pentiumpro_bug4064495;
 123 
 124 uchar_t x86_featureset[BT_SIZEOFMAP(NUM_X86_FEATURES)];
 125 
 126 static char *x86_feature_names[NUM_X86_FEATURES] = {
 127         "lgpg",
 128         "tsc",
 129         "msr",
 130         "mtrr",
 131         "pge",
 132         "de",
 133         "cmov",
 134         "mmx",
 135         "mca",
 136         "pae",
 137         "cv8",
 138         "pat",
 139         "sep",
 140         "sse",
 141         "sse2",
 142         "htt",
 143         "asysc",
 144         "nx",
 145         "sse3",
 146         "cx16",
 147         "cmp",
 148         "tscp",
 149         "mwait",
 150         "sse4a",
 151         "cpuid",
 152         "ssse3",
 153         "sse4_1",
 154         "sse4_2",
 155         "1gpg",
 156         "clfsh",
 157         "64",
 158         "aes",
 159         "pclmulqdq",
 160         "xsave",
 161         "avx",
 162         "vmx",
 163         "svm"
 164 };
 165 
 166 boolean_t
 167 is_x86_feature(void *featureset, uint_t feature)
 168 {
 169         ASSERT(feature < NUM_X86_FEATURES);
 170         return (BT_TEST((ulong_t *)featureset, feature));
 171 }
 172 
 173 void
 174 add_x86_feature(void *featureset, uint_t feature)
 175 {
 176         ASSERT(feature < NUM_X86_FEATURES);
 177         BT_SET((ulong_t *)featureset, feature);
 178 }
 179 
 180 void
 181 remove_x86_feature(void *featureset, uint_t feature)
 182 {
 183         ASSERT(feature < NUM_X86_FEATURES);
 184         BT_CLEAR((ulong_t *)featureset, feature);
 185 }
 186 
 187 boolean_t
 188 compare_x86_featureset(void *setA, void *setB)
 189 {
 190         /*
 191          * We assume that the unused bits of the bitmap are always zero.
 192          */
 193         if (memcmp(setA, setB, BT_SIZEOFMAP(NUM_X86_FEATURES)) == 0) {
 194                 return (B_TRUE);
 195         } else {
 196                 return (B_FALSE);
 197         }
 198 }
 199 
 200 void
 201 print_x86_featureset(void *featureset)
 202 {
 203         uint_t i;
 204 
 205         for (i = 0; i < NUM_X86_FEATURES; i++) {
 206                 if (is_x86_feature(featureset, i)) {
 207                         cmn_err(CE_CONT, "?x86_feature: %s\n",
 208                             x86_feature_names[i]);
 209                 }
 210         }
 211 }
 212 
 213 uint_t enable486;
 214 
 215 static size_t xsave_state_size = 0;
 216 uint64_t xsave_bv_all = (XFEATURE_LEGACY_FP | XFEATURE_SSE);
 217 boolean_t xsave_force_disable = B_FALSE;
 218 
 219 /*
 220  * This is set to platform type Solaris is running on.
 221  */
 222 static int platform_type = -1;
 223 
 224 #if !defined(__xpv)
 225 /*
 226  * Variable to patch if hypervisor platform detection needs to be
 227  * disabled (e.g. platform_type will always be HW_NATIVE if this is 0).
 228  */
 229 int enable_platform_detection = 1;
 230 #endif
 231 
 232 /*
 233  * monitor/mwait info.
 234  *
 235  * size_actual and buf_actual are the real address and size allocated to get
 236  * proper mwait_buf alignement.  buf_actual and size_actual should be passed
 237  * to kmem_free().  Currently kmem_alloc() and mwait happen to both use
 238  * processor cache-line alignment, but this is not guarantied in the furture.
 239  */
 240 struct mwait_info {
 241         size_t          mon_min;        /* min size to avoid missed wakeups */
 242         size_t          mon_max;        /* size to avoid false wakeups */
 243         size_t          size_actual;    /* size actually allocated */
 244         void            *buf_actual;    /* memory actually allocated */
 245         uint32_t        support;        /* processor support of monitor/mwait */
 246 };
 247 
 248 /*
 249  * xsave/xrestor info.
 250  *
 251  * This structure contains HW feature bits and size of the xsave save area.
 252  * Note: the kernel will use the maximum size required for all hardware
 253  * features. It is not optimize for potential memory savings if features at
 254  * the end of the save area are not enabled.
 255  */
 256 struct xsave_info {
 257         uint32_t        xsav_hw_features_low;   /* Supported HW features */
 258         uint32_t        xsav_hw_features_high;  /* Supported HW features */
 259         size_t          xsav_max_size;  /* max size save area for HW features */
 260         size_t          ymm_size;       /* AVX: size of ymm save area */
 261         size_t          ymm_offset;     /* AVX: offset for ymm save area */
 262 };
 263 
 264 
 265 /*
 266  * These constants determine how many of the elements of the
 267  * cpuid we cache in the cpuid_info data structure; the
 268  * remaining elements are accessible via the cpuid instruction.
 269  */
 270 
 271 #define NMAX_CPI_STD    6               /* eax = 0 .. 5 */
 272 #define NMAX_CPI_EXTD   0x1c            /* eax = 0x80000000 .. 0x8000001b */
 273 
 274 /*
 275  * Some terminology needs to be explained:
 276  *  - Socket: Something that can be plugged into a motherboard.
 277  *  - Package: Same as socket
 278  *  - Chip: Same as socket. Note that AMD's documentation uses term "chip"
 279  *    differently: there, chip is the same as processor node (below)
 280  *  - Processor node: Some AMD processors have more than one
 281  *    "subprocessor" embedded in a package. These subprocessors (nodes)
 282  *    are fully-functional processors themselves with cores, caches,
 283  *    memory controllers, PCI configuration spaces. They are connected
 284  *    inside the package with Hypertransport links. On single-node
 285  *    processors, processor node is equivalent to chip/socket/package.
 286  */
 287 
 288 struct cpuid_info {
 289         uint_t cpi_pass;                /* last pass completed */
 290         /*
 291          * standard function information
 292          */
 293         uint_t cpi_maxeax;              /* fn 0: %eax */
 294         char cpi_vendorstr[13];         /* fn 0: %ebx:%ecx:%edx */
 295         uint_t cpi_vendor;              /* enum of cpi_vendorstr */
 296 
 297         uint_t cpi_family;              /* fn 1: extended family */
 298         uint_t cpi_model;               /* fn 1: extended model */
 299         uint_t cpi_step;                /* fn 1: stepping */
 300         chipid_t cpi_chipid;            /* fn 1: %ebx:  Intel: chip # */
 301                                         /*              AMD: package/socket # */
 302         uint_t cpi_brandid;             /* fn 1: %ebx: brand ID */
 303         int cpi_clogid;                 /* fn 1: %ebx: thread # */
 304         uint_t cpi_ncpu_per_chip;       /* fn 1: %ebx: logical cpu count */
 305         uint8_t cpi_cacheinfo[16];      /* fn 2: intel-style cache desc */
 306         uint_t cpi_ncache;              /* fn 2: number of elements */
 307         uint_t cpi_ncpu_shr_last_cache; /* fn 4: %eax: ncpus sharing cache */
 308         id_t cpi_last_lvl_cacheid;      /* fn 4: %eax: derived cache id */
 309         uint_t cpi_std_4_size;          /* fn 4: number of fn 4 elements */
 310         struct cpuid_regs **cpi_std_4;  /* fn 4: %ecx == 0 .. fn4_size */
 311         struct cpuid_regs cpi_std[NMAX_CPI_STD];        /* 0 .. 5 */
 312         /*
 313          * extended function information
 314          */
 315         uint_t cpi_xmaxeax;             /* fn 0x80000000: %eax */
 316         char cpi_brandstr[49];          /* fn 0x8000000[234] */
 317         uint8_t cpi_pabits;             /* fn 0x80000006: %eax */
 318         uint8_t cpi_vabits;             /* fn 0x80000006: %eax */
 319         struct  cpuid_regs cpi_extd[NMAX_CPI_EXTD];     /* 0x800000XX */
 320 
 321         id_t cpi_coreid;                /* same coreid => strands share core */
 322         int cpi_pkgcoreid;              /* core number within single package */
 323         uint_t cpi_ncore_per_chip;      /* AMD: fn 0x80000008: %ecx[7-0] */
 324                                         /* Intel: fn 4: %eax[31-26] */
 325         /*
 326          * supported feature information
 327          */
 328         uint32_t cpi_support[5];
 329 #define STD_EDX_FEATURES        0
 330 #define AMD_EDX_FEATURES        1
 331 #define TM_EDX_FEATURES         2
 332 #define STD_ECX_FEATURES        3
 333 #define AMD_ECX_FEATURES        4
 334         /*
 335          * Synthesized information, where known.
 336          */
 337         uint32_t cpi_chiprev;           /* See X86_CHIPREV_* in x86_archext.h */
 338         const char *cpi_chiprevstr;     /* May be NULL if chiprev unknown */
 339         uint32_t cpi_socket;            /* Chip package/socket type */
 340 
 341         struct mwait_info cpi_mwait;    /* fn 5: monitor/mwait info */
 342         uint32_t cpi_apicid;
 343         uint_t cpi_procnodeid;          /* AMD: nodeID on HT, Intel: chipid */
 344         uint_t cpi_procnodes_per_pkg;   /* AMD: # of nodes in the package */
 345                                         /* Intel: 1 */
 346 
 347         struct xsave_info cpi_xsave;    /* fn D: xsave/xrestor info */
 348 };
 349 
 350 
 351 static struct cpuid_info cpuid_info0;
 352 
 353 /*
 354  * These bit fields are defined by the Intel Application Note AP-485
 355  * "Intel Processor Identification and the CPUID Instruction"
 356  */
 357 #define CPI_FAMILY_XTD(cpi)     BITX((cpi)->cpi_std[1].cp_eax, 27, 20)
 358 #define CPI_MODEL_XTD(cpi)      BITX((cpi)->cpi_std[1].cp_eax, 19, 16)
 359 #define CPI_TYPE(cpi)           BITX((cpi)->cpi_std[1].cp_eax, 13, 12)
 360 #define CPI_FAMILY(cpi)         BITX((cpi)->cpi_std[1].cp_eax, 11, 8)
 361 #define CPI_STEP(cpi)           BITX((cpi)->cpi_std[1].cp_eax, 3, 0)
 362 #define CPI_MODEL(cpi)          BITX((cpi)->cpi_std[1].cp_eax, 7, 4)
 363 
 364 #define CPI_FEATURES_EDX(cpi)           ((cpi)->cpi_std[1].cp_edx)
 365 #define CPI_FEATURES_ECX(cpi)           ((cpi)->cpi_std[1].cp_ecx)
 366 #define CPI_FEATURES_XTD_EDX(cpi)       ((cpi)->cpi_extd[1].cp_edx)
 367 #define CPI_FEATURES_XTD_ECX(cpi)       ((cpi)->cpi_extd[1].cp_ecx)
 368 
 369 #define CPI_BRANDID(cpi)        BITX((cpi)->cpi_std[1].cp_ebx, 7, 0)
 370 #define CPI_CHUNKS(cpi)         BITX((cpi)->cpi_std[1].cp_ebx, 15, 7)
 371 #define CPI_CPU_COUNT(cpi)      BITX((cpi)->cpi_std[1].cp_ebx, 23, 16)
 372 #define CPI_APIC_ID(cpi)        BITX((cpi)->cpi_std[1].cp_ebx, 31, 24)
 373 
 374 #define CPI_MAXEAX_MAX          0x100           /* sanity control */
 375 #define CPI_XMAXEAX_MAX         0x80000100
 376 #define CPI_FN4_ECX_MAX         0x20            /* sanity: max fn 4 levels */
 377 #define CPI_FNB_ECX_MAX         0x20            /* sanity: max fn B levels */
 378 
 379 /*
 380  * Function 4 (Deterministic Cache Parameters) macros
 381  * Defined by Intel Application Note AP-485
 382  */
 383 #define CPI_NUM_CORES(regs)             BITX((regs)->cp_eax, 31, 26)
 384 #define CPI_NTHR_SHR_CACHE(regs)        BITX((regs)->cp_eax, 25, 14)
 385 #define CPI_FULL_ASSOC_CACHE(regs)      BITX((regs)->cp_eax, 9, 9)
 386 #define CPI_SELF_INIT_CACHE(regs)       BITX((regs)->cp_eax, 8, 8)
 387 #define CPI_CACHE_LVL(regs)             BITX((regs)->cp_eax, 7, 5)
 388 #define CPI_CACHE_TYPE(regs)            BITX((regs)->cp_eax, 4, 0)
 389 #define CPI_CPU_LEVEL_TYPE(regs)        BITX((regs)->cp_ecx, 15, 8)
 390 
 391 #define CPI_CACHE_WAYS(regs)            BITX((regs)->cp_ebx, 31, 22)
 392 #define CPI_CACHE_PARTS(regs)           BITX((regs)->cp_ebx, 21, 12)
 393 #define CPI_CACHE_COH_LN_SZ(regs)       BITX((regs)->cp_ebx, 11, 0)
 394 
 395 #define CPI_CACHE_SETS(regs)            BITX((regs)->cp_ecx, 31, 0)
 396 
 397 #define CPI_PREFCH_STRIDE(regs)         BITX((regs)->cp_edx, 9, 0)
 398 
 399 
 400 /*
 401  * A couple of shorthand macros to identify "later" P6-family chips
 402  * like the Pentium M and Core.  First, the "older" P6-based stuff
 403  * (loosely defined as "pre-Pentium-4"):
 404  * P6, PII, Mobile PII, PII Xeon, PIII, Mobile PIII, PIII Xeon
 405  */
 406 
 407 #define IS_LEGACY_P6(cpi) (                     \
 408         cpi->cpi_family == 6 &&              \
 409                 (cpi->cpi_model == 1 ||              \
 410                 cpi->cpi_model == 3 ||               \
 411                 cpi->cpi_model == 5 ||               \
 412                 cpi->cpi_model == 6 ||               \
 413                 cpi->cpi_model == 7 ||               \
 414                 cpi->cpi_model == 8 ||               \
 415                 cpi->cpi_model == 0xA ||     \
 416                 cpi->cpi_model == 0xB)               \
 417 )
 418 
 419 /* A "new F6" is everything with family 6 that's not the above */
 420 #define IS_NEW_F6(cpi) ((cpi->cpi_family == 6) && !IS_LEGACY_P6(cpi))
 421 
 422 /* Extended family/model support */
 423 #define IS_EXTENDED_MODEL_INTEL(cpi) (cpi->cpi_family == 0x6 || \
 424         cpi->cpi_family >= 0xf)
 425 
 426 /*
 427  * Info for monitor/mwait idle loop.
 428  *
 429  * See cpuid section of "Intel 64 and IA-32 Architectures Software Developer's
 430  * Manual Volume 2A: Instruction Set Reference, A-M" #25366-022US, November
 431  * 2006.
 432  * See MONITOR/MWAIT section of "AMD64 Architecture Programmer's Manual
 433  * Documentation Updates" #33633, Rev 2.05, December 2006.
 434  */
 435 #define MWAIT_SUPPORT           (0x00000001)    /* mwait supported */
 436 #define MWAIT_EXTENSIONS        (0x00000002)    /* extenstion supported */
 437 #define MWAIT_ECX_INT_ENABLE    (0x00000004)    /* ecx 1 extension supported */
 438 #define MWAIT_SUPPORTED(cpi)    ((cpi)->cpi_std[1].cp_ecx & CPUID_INTC_ECX_MON)
 439 #define MWAIT_INT_ENABLE(cpi)   ((cpi)->cpi_std[5].cp_ecx & 0x2)
 440 #define MWAIT_EXTENSION(cpi)    ((cpi)->cpi_std[5].cp_ecx & 0x1)
 441 #define MWAIT_SIZE_MIN(cpi)     BITX((cpi)->cpi_std[5].cp_eax, 15, 0)
 442 #define MWAIT_SIZE_MAX(cpi)     BITX((cpi)->cpi_std[5].cp_ebx, 15, 0)
 443 /*
 444  * Number of sub-cstates for a given c-state.
 445  */
 446 #define MWAIT_NUM_SUBC_STATES(cpi, c_state)                     \
 447         BITX((cpi)->cpi_std[5].cp_edx, c_state + 3, c_state)
 448 
 449 /*
 450  * XSAVE leaf 0xD enumeration
 451  */
 452 #define CPUID_LEAFD_2_YMM_OFFSET        576
 453 #define CPUID_LEAFD_2_YMM_SIZE          256
 454 
 455 /*
 456  * Functions we consune from cpuid_subr.c;  don't publish these in a header
 457  * file to try and keep people using the expected cpuid_* interfaces.
 458  */
 459 extern uint32_t _cpuid_skt(uint_t, uint_t, uint_t, uint_t);
 460 extern const char *_cpuid_sktstr(uint_t, uint_t, uint_t, uint_t);
 461 extern uint32_t _cpuid_chiprev(uint_t, uint_t, uint_t, uint_t);
 462 extern const char *_cpuid_chiprevstr(uint_t, uint_t, uint_t, uint_t);
 463 extern uint_t _cpuid_vendorstr_to_vendorcode(char *);
 464 
 465 /*
 466  * Apply up various platform-dependent restrictions where the
 467  * underlying platform restrictions mean the CPU can be marked
 468  * as less capable than its cpuid instruction would imply.
 469  */
 470 #if defined(__xpv)
 471 static void
 472 platform_cpuid_mangle(uint_t vendor, uint32_t eax, struct cpuid_regs *cp)
 473 {
 474         switch (eax) {
 475         case 1: {
 476                 uint32_t mcamask = DOMAIN_IS_INITDOMAIN(xen_info) ?
 477                     0 : CPUID_INTC_EDX_MCA;
 478                 cp->cp_edx &=
 479                     ~(mcamask |
 480                     CPUID_INTC_EDX_PSE |
 481                     CPUID_INTC_EDX_VME | CPUID_INTC_EDX_DE |
 482                     CPUID_INTC_EDX_SEP | CPUID_INTC_EDX_MTRR |
 483                     CPUID_INTC_EDX_PGE | CPUID_INTC_EDX_PAT |
 484                     CPUID_AMD_EDX_SYSC | CPUID_INTC_EDX_SEP |
 485                     CPUID_INTC_EDX_PSE36 | CPUID_INTC_EDX_HTT);
 486                 break;
 487         }
 488 
 489         case 0x80000001:
 490                 cp->cp_edx &=
 491                     ~(CPUID_AMD_EDX_PSE |
 492                     CPUID_INTC_EDX_VME | CPUID_INTC_EDX_DE |
 493                     CPUID_AMD_EDX_MTRR | CPUID_AMD_EDX_PGE |
 494                     CPUID_AMD_EDX_PAT | CPUID_AMD_EDX_PSE36 |
 495                     CPUID_AMD_EDX_SYSC | CPUID_INTC_EDX_SEP |
 496                     CPUID_AMD_EDX_TSCP);
 497                 cp->cp_ecx &= ~CPUID_AMD_ECX_CMP_LGCY;
 498                 break;
 499         default:
 500                 break;
 501         }
 502 
 503         switch (vendor) {
 504         case X86_VENDOR_Intel:
 505                 switch (eax) {
 506                 case 4:
 507                         /*
 508                          * Zero out the (ncores-per-chip - 1) field
 509                          */
 510                         cp->cp_eax &= 0x03fffffff;
 511                         break;
 512                 default:
 513                         break;
 514                 }
 515                 break;
 516         case X86_VENDOR_AMD:
 517                 switch (eax) {
 518 
 519                 case 0x80000001:
 520                         cp->cp_ecx &= ~CPUID_AMD_ECX_CR8D;
 521                         break;
 522 
 523                 case 0x80000008:
 524                         /*
 525                          * Zero out the (ncores-per-chip - 1) field
 526                          */
 527                         cp->cp_ecx &= 0xffffff00;
 528                         break;
 529                 default:
 530                         break;
 531                 }
 532                 break;
 533         default:
 534                 break;
 535         }
 536 }
 537 #else
 538 #define platform_cpuid_mangle(vendor, eax, cp)  /* nothing */
 539 #endif
 540 
 541 /*
 542  *  Some undocumented ways of patching the results of the cpuid
 543  *  instruction to permit running Solaris 10 on future cpus that
 544  *  we don't currently support.  Could be set to non-zero values
 545  *  via settings in eeprom.
 546  */
 547 
 548 uint32_t cpuid_feature_ecx_include;
 549 uint32_t cpuid_feature_ecx_exclude;
 550 uint32_t cpuid_feature_edx_include;
 551 uint32_t cpuid_feature_edx_exclude;
 552 
 553 /*
 554  * Allocate space for mcpu_cpi in the machcpu structure for all non-boot CPUs.
 555  */
 556 void
 557 cpuid_alloc_space(cpu_t *cpu)
 558 {
 559         /*
 560          * By convention, cpu0 is the boot cpu, which is set up
 561          * before memory allocation is available.  All other cpus get
 562          * their cpuid_info struct allocated here.
 563          */
 564         ASSERT(cpu->cpu_id != 0);
 565         ASSERT(cpu->cpu_m.mcpu_cpi == NULL);
 566         cpu->cpu_m.mcpu_cpi =
 567             kmem_zalloc(sizeof (*cpu->cpu_m.mcpu_cpi), KM_SLEEP);
 568 }
 569 
 570 void
 571 cpuid_free_space(cpu_t *cpu)
 572 {
 573         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
 574         int i;
 575 
 576         ASSERT(cpi != NULL);
 577         ASSERT(cpi != &cpuid_info0);
 578 
 579         /*
 580          * Free up any function 4 related dynamic storage
 581          */
 582         for (i = 1; i < cpi->cpi_std_4_size; i++)
 583                 kmem_free(cpi->cpi_std_4[i], sizeof (struct cpuid_regs));
 584         if (cpi->cpi_std_4_size > 0)
 585                 kmem_free(cpi->cpi_std_4,
 586                     cpi->cpi_std_4_size * sizeof (struct cpuid_regs *));
 587 
 588         kmem_free(cpi, sizeof (*cpi));
 589         cpu->cpu_m.mcpu_cpi = NULL;
 590 }
 591 
 592 #if !defined(__xpv)
 593 
 594 /*
 595  * Determine the type of the underlying platform. This is used to customize
 596  * initialization of various subsystems (e.g. TSC). determine_platform() must
 597  * only ever be called once to prevent two processors from seeing different
 598  * values of platform_type, it must be called before cpuid_pass1(), the
 599  * earliest consumer to execute.
 600  */
 601 void
 602 determine_platform(void)
 603 {
 604         struct cpuid_regs cp;
 605         char *xen_str;
 606         uint32_t xen_signature[4], base;
 607 
 608         ASSERT(platform_type == -1);
 609 
 610         platform_type = HW_NATIVE;
 611 
 612         if (!enable_platform_detection)
 613                 return;
 614 
 615         /*
 616          * In a fully virtualized domain, Xen's pseudo-cpuid function
 617          * returns a string representing the Xen signature in %ebx, %ecx,
 618          * and %edx. %eax contains the maximum supported cpuid function.
 619          * We need at least a (base + 2) leaf value to do what we want
 620          * to do. Try different base values, since the hypervisor might
 621          * use a different one depending on whether hyper-v emulation
 622          * is switched on by default or not.
 623          */
 624         for (base = 0x40000000; base < 0x40010000; base += 0x100) {
 625                 cp.cp_eax = base;
 626                 (void) __cpuid_insn(&cp);
 627                 xen_signature[0] = cp.cp_ebx;
 628                 xen_signature[1] = cp.cp_ecx;
 629                 xen_signature[2] = cp.cp_edx;
 630                 xen_signature[3] = 0;
 631                 xen_str = (char *)xen_signature;
 632                 if (strcmp("XenVMMXenVMM", xen_str) == 0 &&
 633                     cp.cp_eax >= (base + 2)) {
 634                         platform_type = HW_XEN_HVM;
 635                         return;
 636                 }
 637         }
 638 
 639         if (vmware_platform()) /* running under vmware hypervisor? */
 640                 platform_type = HW_VMWARE;
 641 }
 642 
 643 int
 644 get_hwenv(void)
 645 {
 646         ASSERT(platform_type != -1);
 647         return (platform_type);
 648 }
 649 
 650 int
 651 is_controldom(void)
 652 {
 653         return (0);
 654 }
 655 
 656 #else
 657 
 658 int
 659 get_hwenv(void)
 660 {
 661         return (HW_XEN_PV);
 662 }
 663 
 664 int
 665 is_controldom(void)
 666 {
 667         return (DOMAIN_IS_INITDOMAIN(xen_info));
 668 }
 669 
 670 #endif  /* __xpv */
 671 
 672 static void
 673 cpuid_intel_getids(cpu_t *cpu, void *feature)
 674 {
 675         uint_t i;
 676         uint_t chipid_shift = 0;
 677         uint_t coreid_shift = 0;
 678         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
 679 
 680         for (i = 1; i < cpi->cpi_ncpu_per_chip; i <<= 1)
 681                 chipid_shift++;
 682 
 683         cpi->cpi_chipid = cpi->cpi_apicid >> chipid_shift;
 684         cpi->cpi_clogid = cpi->cpi_apicid & ((1 << chipid_shift) - 1);
 685 
 686         if (is_x86_feature(feature, X86FSET_CMP)) {
 687                 /*
 688                  * Multi-core (and possibly multi-threaded)
 689                  * processors.
 690                  */
 691                 uint_t ncpu_per_core;
 692                 if (cpi->cpi_ncore_per_chip == 1)
 693                         ncpu_per_core = cpi->cpi_ncpu_per_chip;
 694                 else if (cpi->cpi_ncore_per_chip > 1)
 695                         ncpu_per_core = cpi->cpi_ncpu_per_chip /
 696                             cpi->cpi_ncore_per_chip;
 697                 /*
 698                  * 8bit APIC IDs on dual core Pentiums
 699                  * look like this:
 700                  *
 701                  * +-----------------------+------+------+
 702                  * | Physical Package ID   |  MC  |  HT  |
 703                  * +-----------------------+------+------+
 704                  * <------- chipid -------->
 705                  * <------- coreid --------------->
 706                  *                         <--- clogid -->
 707                  *                         <------>
 708                  *                         pkgcoreid
 709                  *
 710                  * Where the number of bits necessary to
 711                  * represent MC and HT fields together equals
 712                  * to the minimum number of bits necessary to
 713                  * store the value of cpi->cpi_ncpu_per_chip.
 714                  * Of those bits, the MC part uses the number
 715                  * of bits necessary to store the value of
 716                  * cpi->cpi_ncore_per_chip.
 717                  */
 718                 for (i = 1; i < ncpu_per_core; i <<= 1)
 719                         coreid_shift++;
 720                 cpi->cpi_coreid = cpi->cpi_apicid >> coreid_shift;
 721                 cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
 722         } else if (is_x86_feature(feature, X86FSET_HTT)) {
 723                 /*
 724                  * Single-core multi-threaded processors.
 725                  */
 726                 cpi->cpi_coreid = cpi->cpi_chipid;
 727                 cpi->cpi_pkgcoreid = 0;
 728         }
 729         cpi->cpi_procnodeid = cpi->cpi_chipid;
 730 }
 731 
 732 static void
 733 cpuid_amd_getids(cpu_t *cpu)
 734 {
 735         int i, first_half, coreidsz;
 736         uint32_t nb_caps_reg;
 737         uint_t node2_1;
 738         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
 739 
 740         /*
 741          * AMD CMP chips currently have a single thread per core.
 742          *
 743          * Since no two cpus share a core we must assign a distinct coreid
 744          * per cpu, and we do this by using the cpu_id.  This scheme does not,
 745          * however, guarantee that sibling cores of a chip will have sequential
 746          * coreids starting at a multiple of the number of cores per chip -
 747          * that is usually the case, but if the ACPI MADT table is presented
 748          * in a different order then we need to perform a few more gymnastics
 749          * for the pkgcoreid.
 750          *
 751          * All processors in the system have the same number of enabled
 752          * cores. Cores within a processor are always numbered sequentially
 753          * from 0 regardless of how many or which are disabled, and there
 754          * is no way for operating system to discover the real core id when some
 755          * are disabled.
 756          */
 757 
 758         cpi->cpi_coreid = cpu->cpu_id;
 759 
 760         if (cpi->cpi_xmaxeax >= 0x80000008) {
 761 
 762                 coreidsz = BITX((cpi)->cpi_extd[8].cp_ecx, 15, 12);
 763 
 764                 /*
 765                  * In AMD parlance chip is really a node while Solaris
 766                  * sees chip as equivalent to socket/package.
 767                  */
 768                 cpi->cpi_ncore_per_chip =
 769                     BITX((cpi)->cpi_extd[8].cp_ecx, 7, 0) + 1;
 770                 if (coreidsz == 0) {
 771                         /* Use legacy method */
 772                         for (i = 1; i < cpi->cpi_ncore_per_chip; i <<= 1)
 773                                 coreidsz++;
 774                         if (coreidsz == 0)
 775                                 coreidsz = 1;
 776                 }
 777         } else {
 778                 /* Assume single-core part */
 779                 cpi->cpi_ncore_per_chip = 1;
 780                 coreidsz = 1;
 781         }
 782 
 783         cpi->cpi_clogid = cpi->cpi_pkgcoreid =
 784             cpi->cpi_apicid & ((1<<coreidsz) - 1);
 785         cpi->cpi_ncpu_per_chip = cpi->cpi_ncore_per_chip;
 786 
 787         /* Get nodeID */
 788         if (cpi->cpi_family == 0xf) {
 789                 cpi->cpi_procnodeid = (cpi->cpi_apicid >> coreidsz) & 7;
 790                 cpi->cpi_chipid = cpi->cpi_procnodeid;
 791         } else if (cpi->cpi_family == 0x10) {
 792                 /*
 793                  * See if we are a multi-node processor.
 794                  * All processors in the system have the same number of nodes
 795                  */
 796                 nb_caps_reg =  pci_getl_func(0, 24, 3, 0xe8);
 797                 if ((cpi->cpi_model < 8) || BITX(nb_caps_reg, 29, 29) == 0) {
 798                         /* Single-node */
 799                         cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 5,
 800                             coreidsz);
 801                         cpi->cpi_chipid = cpi->cpi_procnodeid;
 802                 } else {
 803 
 804                         /*
 805                          * Multi-node revision D (2 nodes per package
 806                          * are supported)
 807                          */
 808                         cpi->cpi_procnodes_per_pkg = 2;
 809 
 810                         first_half = (cpi->cpi_pkgcoreid <=
 811                             (cpi->cpi_ncore_per_chip/2 - 1));
 812 
 813                         if (cpi->cpi_apicid == cpi->cpi_pkgcoreid) {
 814                                 /* We are BSP */
 815                                 cpi->cpi_procnodeid = (first_half ? 0 : 1);
 816                                 cpi->cpi_chipid = cpi->cpi_procnodeid >> 1;
 817                         } else {
 818 
 819                                 /* We are AP */
 820                                 /* NodeId[2:1] bits to use for reading F3xe8 */
 821                                 node2_1 = BITX(cpi->cpi_apicid, 5, 4) << 1;
 822 
 823                                 nb_caps_reg =
 824                                     pci_getl_func(0, 24 + node2_1, 3, 0xe8);
 825 
 826                                 /*
 827                                  * Check IntNodeNum bit (31:30, but bit 31 is
 828                                  * always 0 on dual-node processors)
 829                                  */
 830                                 if (BITX(nb_caps_reg, 30, 30) == 0)
 831                                         cpi->cpi_procnodeid = node2_1 +
 832                                             !first_half;
 833                                 else
 834                                         cpi->cpi_procnodeid = node2_1 +
 835                                             first_half;
 836 
 837                                 cpi->cpi_chipid = cpi->cpi_procnodeid >> 1;
 838                         }
 839                 }
 840         } else if (cpi->cpi_family >= 0x11) {
 841                 cpi->cpi_procnodeid = (cpi->cpi_apicid >> coreidsz) & 7;
 842                 cpi->cpi_chipid = cpi->cpi_procnodeid;
 843         } else {
 844                 cpi->cpi_procnodeid = 0;
 845                 cpi->cpi_chipid = cpi->cpi_procnodeid;
 846         }
 847 }
 848 
 849 /*
 850  * Setup XFeature_Enabled_Mask register. Required by xsave feature.
 851  */
 852 void
 853 setup_xfem(void)
 854 {
 855         uint64_t flags = XFEATURE_LEGACY_FP;
 856 
 857         ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
 858 
 859         if (is_x86_feature(x86_featureset, X86FSET_SSE))
 860                 flags |= XFEATURE_SSE;
 861 
 862         if (is_x86_feature(x86_featureset, X86FSET_AVX))
 863                 flags |= XFEATURE_AVX;
 864 
 865         set_xcr(XFEATURE_ENABLED_MASK, flags);
 866 
 867         xsave_bv_all = flags;
 868 }
 869 
 870 void
 871 cpuid_pass1(cpu_t *cpu, uchar_t *featureset)
 872 {
 873         uint32_t mask_ecx, mask_edx;
 874         struct cpuid_info *cpi;
 875         struct cpuid_regs *cp;
 876         int xcpuid;
 877 #if !defined(__xpv)
 878         extern int idle_cpu_prefer_mwait;
 879 #endif
 880 
 881         /*
 882          * Space statically allocated for BSP, ensure pointer is set
 883          */
 884         if (cpu->cpu_id == 0) {
 885                 if (cpu->cpu_m.mcpu_cpi == NULL)
 886                         cpu->cpu_m.mcpu_cpi = &cpuid_info0;
 887         }
 888 
 889         add_x86_feature(featureset, X86FSET_CPUID);
 890 
 891         cpi = cpu->cpu_m.mcpu_cpi;
 892         ASSERT(cpi != NULL);
 893         cp = &cpi->cpi_std[0];
 894         cp->cp_eax = 0;
 895         cpi->cpi_maxeax = __cpuid_insn(cp);
 896         {
 897                 uint32_t *iptr = (uint32_t *)cpi->cpi_vendorstr;
 898                 *iptr++ = cp->cp_ebx;
 899                 *iptr++ = cp->cp_edx;
 900                 *iptr++ = cp->cp_ecx;
 901                 *(char *)&cpi->cpi_vendorstr[12] = '\0';
 902         }
 903 
 904         cpi->cpi_vendor = _cpuid_vendorstr_to_vendorcode(cpi->cpi_vendorstr);
 905         x86_vendor = cpi->cpi_vendor; /* for compatibility */
 906 
 907         /*
 908          * Limit the range in case of weird hardware
 909          */
 910         if (cpi->cpi_maxeax > CPI_MAXEAX_MAX)
 911                 cpi->cpi_maxeax = CPI_MAXEAX_MAX;
 912         if (cpi->cpi_maxeax < 1)
 913                 goto pass1_done;
 914 
 915         cp = &cpi->cpi_std[1];
 916         cp->cp_eax = 1;
 917         (void) __cpuid_insn(cp);
 918 
 919         /*
 920          * Extract identifying constants for easy access.
 921          */
 922         cpi->cpi_model = CPI_MODEL(cpi);
 923         cpi->cpi_family = CPI_FAMILY(cpi);
 924 
 925         if (cpi->cpi_family == 0xf)
 926                 cpi->cpi_family += CPI_FAMILY_XTD(cpi);
 927 
 928         /*
 929          * Beware: AMD uses "extended model" iff base *FAMILY* == 0xf.
 930          * Intel, and presumably everyone else, uses model == 0xf, as
 931          * one would expect (max value means possible overflow).  Sigh.
 932          */
 933 
 934         switch (cpi->cpi_vendor) {
 935         case X86_VENDOR_Intel:
 936                 if (IS_EXTENDED_MODEL_INTEL(cpi))
 937                         cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
 938                 break;
 939         case X86_VENDOR_AMD:
 940                 if (CPI_FAMILY(cpi) == 0xf)
 941                         cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
 942                 break;
 943         default:
 944                 if (cpi->cpi_model == 0xf)
 945                         cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
 946                 break;
 947         }
 948 
 949         cpi->cpi_step = CPI_STEP(cpi);
 950         cpi->cpi_brandid = CPI_BRANDID(cpi);
 951 
 952         /*
 953          * *default* assumptions:
 954          * - believe %edx feature word
 955          * - ignore %ecx feature word
 956          * - 32-bit virtual and physical addressing
 957          */
 958         mask_edx = 0xffffffff;
 959         mask_ecx = 0;
 960 
 961         cpi->cpi_pabits = cpi->cpi_vabits = 32;
 962 
 963         switch (cpi->cpi_vendor) {
 964         case X86_VENDOR_Intel:
 965                 if (cpi->cpi_family == 5)
 966                         x86_type = X86_TYPE_P5;
 967                 else if (IS_LEGACY_P6(cpi)) {
 968                         x86_type = X86_TYPE_P6;
 969                         pentiumpro_bug4046376 = 1;
 970                         pentiumpro_bug4064495 = 1;
 971                         /*
 972                          * Clear the SEP bit when it was set erroneously
 973                          */
 974                         if (cpi->cpi_model < 3 && cpi->cpi_step < 3)
 975                                 cp->cp_edx &= ~CPUID_INTC_EDX_SEP;
 976                 } else if (IS_NEW_F6(cpi) || cpi->cpi_family == 0xf) {
 977                         x86_type = X86_TYPE_P4;
 978                         /*
 979                          * We don't currently depend on any of the %ecx
 980                          * features until Prescott, so we'll only check
 981                          * this from P4 onwards.  We might want to revisit
 982                          * that idea later.
 983                          */
 984                         mask_ecx = 0xffffffff;
 985                 } else if (cpi->cpi_family > 0xf)
 986                         mask_ecx = 0xffffffff;
 987                 /*
 988                  * We don't support MONITOR/MWAIT if leaf 5 is not available
 989                  * to obtain the monitor linesize.
 990                  */
 991                 if (cpi->cpi_maxeax < 5)
 992                         mask_ecx &= ~CPUID_INTC_ECX_MON;
 993                 break;
 994         case X86_VENDOR_IntelClone:
 995         default:
 996                 break;
 997         case X86_VENDOR_AMD:
 998 #if defined(OPTERON_ERRATUM_108)
 999                 if (cpi->cpi_family == 0xf && cpi->cpi_model == 0xe) {
1000                         cp->cp_eax = (0xf0f & cp->cp_eax) | 0xc0;
1001                         cpi->cpi_model = 0xc;
1002                 } else
1003 #endif
1004                 if (cpi->cpi_family == 5) {
1005                         /*
1006                          * AMD K5 and K6
1007                          *
1008                          * These CPUs have an incomplete implementation
1009                          * of MCA/MCE which we mask away.
1010                          */
1011                         mask_edx &= ~(CPUID_INTC_EDX_MCE | CPUID_INTC_EDX_MCA);
1012 
1013                         /*
1014                          * Model 0 uses the wrong (APIC) bit
1015                          * to indicate PGE.  Fix it here.
1016                          */
1017                         if (cpi->cpi_model == 0) {
1018                                 if (cp->cp_edx & 0x200) {
1019                                         cp->cp_edx &= ~0x200;
1020                                         cp->cp_edx |= CPUID_INTC_EDX_PGE;
1021                                 }
1022                         }
1023 
1024                         /*
1025                          * Early models had problems w/ MMX; disable.
1026                          */
1027                         if (cpi->cpi_model < 6)
1028                                 mask_edx &= ~CPUID_INTC_EDX_MMX;
1029                 }
1030 
1031                 /*
1032                  * For newer families, SSE3 and CX16, at least, are valid;
1033                  * enable all
1034                  */
1035                 if (cpi->cpi_family >= 0xf)
1036                         mask_ecx = 0xffffffff;
1037                 /*
1038                  * We don't support MONITOR/MWAIT if leaf 5 is not available
1039                  * to obtain the monitor linesize.
1040                  */
1041                 if (cpi->cpi_maxeax < 5)
1042                         mask_ecx &= ~CPUID_INTC_ECX_MON;
1043 
1044 #if !defined(__xpv)
1045                 /*
1046                  * Do not use MONITOR/MWAIT to halt in the idle loop on any AMD
1047                  * processors.  AMD does not intend MWAIT to be used in the cpu
1048                  * idle loop on current and future processors.  10h and future
1049                  * AMD processors use more power in MWAIT than HLT.
1050                  * Pre-family-10h Opterons do not have the MWAIT instruction.
1051                  */
1052                 idle_cpu_prefer_mwait = 0;
1053 #endif
1054 
1055                 break;
1056         case X86_VENDOR_TM:
1057                 /*
1058                  * workaround the NT workaround in CMS 4.1
1059                  */
1060                 if (cpi->cpi_family == 5 && cpi->cpi_model == 4 &&
1061                     (cpi->cpi_step == 2 || cpi->cpi_step == 3))
1062                         cp->cp_edx |= CPUID_INTC_EDX_CX8;
1063                 break;
1064         case X86_VENDOR_Centaur:
1065                 /*
1066                  * workaround the NT workarounds again
1067                  */
1068                 if (cpi->cpi_family == 6)
1069                         cp->cp_edx |= CPUID_INTC_EDX_CX8;
1070                 break;
1071         case X86_VENDOR_Cyrix:
1072                 /*
1073                  * We rely heavily on the probing in locore
1074                  * to actually figure out what parts, if any,
1075                  * of the Cyrix cpuid instruction to believe.
1076                  */
1077                 switch (x86_type) {
1078                 case X86_TYPE_CYRIX_486:
1079                         mask_edx = 0;
1080                         break;
1081                 case X86_TYPE_CYRIX_6x86:
1082                         mask_edx = 0;
1083                         break;
1084                 case X86_TYPE_CYRIX_6x86L:
1085                         mask_edx =
1086                             CPUID_INTC_EDX_DE |
1087                             CPUID_INTC_EDX_CX8;
1088                         break;
1089                 case X86_TYPE_CYRIX_6x86MX:
1090                         mask_edx =
1091                             CPUID_INTC_EDX_DE |
1092                             CPUID_INTC_EDX_MSR |
1093                             CPUID_INTC_EDX_CX8 |
1094                             CPUID_INTC_EDX_PGE |
1095                             CPUID_INTC_EDX_CMOV |
1096                             CPUID_INTC_EDX_MMX;
1097                         break;
1098                 case X86_TYPE_CYRIX_GXm:
1099                         mask_edx =
1100                             CPUID_INTC_EDX_MSR |
1101                             CPUID_INTC_EDX_CX8 |
1102                             CPUID_INTC_EDX_CMOV |
1103                             CPUID_INTC_EDX_MMX;
1104                         break;
1105                 case X86_TYPE_CYRIX_MediaGX:
1106                         break;
1107                 case X86_TYPE_CYRIX_MII:
1108                 case X86_TYPE_VIA_CYRIX_III:
1109                         mask_edx =
1110                             CPUID_INTC_EDX_DE |
1111                             CPUID_INTC_EDX_TSC |
1112                             CPUID_INTC_EDX_MSR |
1113                             CPUID_INTC_EDX_CX8 |
1114                             CPUID_INTC_EDX_PGE |
1115                             CPUID_INTC_EDX_CMOV |
1116                             CPUID_INTC_EDX_MMX;
1117                         break;
1118                 default:
1119                         break;
1120                 }
1121                 break;
1122         }
1123 
1124 #if defined(__xpv)
1125         /*
1126          * Do not support MONITOR/MWAIT under a hypervisor
1127          */
1128         mask_ecx &= ~CPUID_INTC_ECX_MON;
1129         /*
1130          * Do not support XSAVE under a hypervisor for now
1131          */
1132         xsave_force_disable = B_TRUE;
1133 
1134 #endif  /* __xpv */
1135 
1136         if (xsave_force_disable) {
1137                 mask_ecx &= ~CPUID_INTC_ECX_XSAVE;
1138                 mask_ecx &= ~CPUID_INTC_ECX_AVX;
1139         }
1140 
1141         /*
1142          * Now we've figured out the masks that determine
1143          * which bits we choose to believe, apply the masks
1144          * to the feature words, then map the kernel's view
1145          * of these feature words into its feature word.
1146          */
1147         cp->cp_edx &= mask_edx;
1148         cp->cp_ecx &= mask_ecx;
1149 
1150         /*
1151          * apply any platform restrictions (we don't call this
1152          * immediately after __cpuid_insn here, because we need the
1153          * workarounds applied above first)
1154          */
1155         platform_cpuid_mangle(cpi->cpi_vendor, 1, cp);
1156 
1157         /*
1158          * fold in overrides from the "eeprom" mechanism
1159          */
1160         cp->cp_edx |= cpuid_feature_edx_include;
1161         cp->cp_edx &= ~cpuid_feature_edx_exclude;
1162 
1163         cp->cp_ecx |= cpuid_feature_ecx_include;
1164         cp->cp_ecx &= ~cpuid_feature_ecx_exclude;
1165 
1166         if (cp->cp_edx & CPUID_INTC_EDX_PSE) {
1167                 add_x86_feature(featureset, X86FSET_LARGEPAGE);
1168         }
1169         if (cp->cp_edx & CPUID_INTC_EDX_TSC) {
1170                 add_x86_feature(featureset, X86FSET_TSC);
1171         }
1172         if (cp->cp_edx & CPUID_INTC_EDX_MSR) {
1173                 add_x86_feature(featureset, X86FSET_MSR);
1174         }
1175         if (cp->cp_edx & CPUID_INTC_EDX_MTRR) {
1176                 add_x86_feature(featureset, X86FSET_MTRR);
1177         }
1178         if (cp->cp_edx & CPUID_INTC_EDX_PGE) {
1179                 add_x86_feature(featureset, X86FSET_PGE);
1180         }
1181         if (cp->cp_edx & CPUID_INTC_EDX_CMOV) {
1182                 add_x86_feature(featureset, X86FSET_CMOV);
1183         }
1184         if (cp->cp_edx & CPUID_INTC_EDX_MMX) {
1185                 add_x86_feature(featureset, X86FSET_MMX);
1186         }
1187         if ((cp->cp_edx & CPUID_INTC_EDX_MCE) != 0 &&
1188             (cp->cp_edx & CPUID_INTC_EDX_MCA) != 0) {
1189                 add_x86_feature(featureset, X86FSET_MCA);
1190         }
1191         if (cp->cp_edx & CPUID_INTC_EDX_PAE) {
1192                 add_x86_feature(featureset, X86FSET_PAE);
1193         }
1194         if (cp->cp_edx & CPUID_INTC_EDX_CX8) {
1195                 add_x86_feature(featureset, X86FSET_CX8);
1196         }
1197         if (cp->cp_ecx & CPUID_INTC_ECX_CX16) {
1198                 add_x86_feature(featureset, X86FSET_CX16);
1199         }
1200         if (cp->cp_edx & CPUID_INTC_EDX_PAT) {
1201                 add_x86_feature(featureset, X86FSET_PAT);
1202         }
1203         if (cp->cp_edx & CPUID_INTC_EDX_SEP) {
1204                 add_x86_feature(featureset, X86FSET_SEP);
1205         }
1206         if (cp->cp_edx & CPUID_INTC_EDX_FXSR) {
1207                 /*
1208                  * In our implementation, fxsave/fxrstor
1209                  * are prerequisites before we'll even
1210                  * try and do SSE things.
1211                  */
1212                 if (cp->cp_edx & CPUID_INTC_EDX_SSE) {
1213                         add_x86_feature(featureset, X86FSET_SSE);
1214                 }
1215                 if (cp->cp_edx & CPUID_INTC_EDX_SSE2) {
1216                         add_x86_feature(featureset, X86FSET_SSE2);
1217                 }
1218                 if (cp->cp_ecx & CPUID_INTC_ECX_SSE3) {
1219                         add_x86_feature(featureset, X86FSET_SSE3);
1220                 }
1221                 if (cp->cp_ecx & CPUID_INTC_ECX_SSSE3) {
1222                         add_x86_feature(featureset, X86FSET_SSSE3);
1223                 }
1224                 if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_1) {
1225                         add_x86_feature(featureset, X86FSET_SSE4_1);
1226                 }
1227                 if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_2) {
1228                         add_x86_feature(featureset, X86FSET_SSE4_2);
1229                 }
1230                 if (cp->cp_ecx & CPUID_INTC_ECX_AES) {
1231                         add_x86_feature(featureset, X86FSET_AES);
1232                 }
1233                 if (cp->cp_ecx & CPUID_INTC_ECX_PCLMULQDQ) {
1234                         add_x86_feature(featureset, X86FSET_PCLMULQDQ);
1235                 }
1236 
1237                 if (cp->cp_ecx & CPUID_INTC_ECX_XSAVE) {
1238                         add_x86_feature(featureset, X86FSET_XSAVE);
1239                         /* We only test AVX when there is XSAVE */
1240                         if (cp->cp_ecx & CPUID_INTC_ECX_AVX) {
1241                                 add_x86_feature(featureset,
1242                                     X86FSET_AVX);
1243                         }
1244                 }
1245         }
1246         if (cp->cp_edx & CPUID_INTC_EDX_DE) {
1247                 add_x86_feature(featureset, X86FSET_DE);
1248         }
1249 #if !defined(__xpv)
1250         if (cp->cp_ecx & CPUID_INTC_ECX_MON) {
1251 
1252                 /*
1253                  * We require the CLFLUSH instruction for erratum workaround
1254                  * to use MONITOR/MWAIT.
1255                  */
1256                 if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
1257                         cpi->cpi_mwait.support |= MWAIT_SUPPORT;
1258                         add_x86_feature(featureset, X86FSET_MWAIT);
1259                 } else {
1260                         extern int idle_cpu_assert_cflush_monitor;
1261 
1262                         /*
1263                          * All processors we are aware of which have
1264                          * MONITOR/MWAIT also have CLFLUSH.
1265                          */
1266                         if (idle_cpu_assert_cflush_monitor) {
1267                                 ASSERT((cp->cp_ecx & CPUID_INTC_ECX_MON) &&
1268                                     (cp->cp_edx & CPUID_INTC_EDX_CLFSH));
1269                         }
1270                 }
1271         }
1272 #endif  /* __xpv */
1273 
1274         if (cp->cp_ecx & CPUID_INTC_ECX_VMX) {
1275                 add_x86_feature(featureset, X86FSET_VMX);
1276         }
1277 
1278         /*
1279          * Only need it first time, rest of the cpus would follow suit.
1280          * we only capture this for the bootcpu.
1281          */
1282         if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
1283                 add_x86_feature(featureset, X86FSET_CLFSH);
1284                 x86_clflush_size = (BITX(cp->cp_ebx, 15, 8) * 8);
1285         }
1286         if (is_x86_feature(featureset, X86FSET_PAE))
1287                 cpi->cpi_pabits = 36;
1288 
1289         /*
1290          * Hyperthreading configuration is slightly tricky on Intel
1291          * and pure clones, and even trickier on AMD.
1292          *
1293          * (AMD chose to set the HTT bit on their CMP processors,
1294          * even though they're not actually hyperthreaded.  Thus it
1295          * takes a bit more work to figure out what's really going
1296          * on ... see the handling of the CMP_LGCY bit below)
1297          */
1298         if (cp->cp_edx & CPUID_INTC_EDX_HTT) {
1299                 cpi->cpi_ncpu_per_chip = CPI_CPU_COUNT(cpi);
1300                 if (cpi->cpi_ncpu_per_chip > 1)
1301                         add_x86_feature(featureset, X86FSET_HTT);
1302         } else {
1303                 cpi->cpi_ncpu_per_chip = 1;
1304         }
1305 
1306         /*
1307          * Work on the "extended" feature information, doing
1308          * some basic initialization for cpuid_pass2()
1309          */
1310         xcpuid = 0;
1311         switch (cpi->cpi_vendor) {
1312         case X86_VENDOR_Intel:
1313                 if (IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf)
1314                         xcpuid++;
1315                 break;
1316         case X86_VENDOR_AMD:
1317                 if (cpi->cpi_family > 5 ||
1318                     (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
1319                         xcpuid++;
1320                 break;
1321         case X86_VENDOR_Cyrix:
1322                 /*
1323                  * Only these Cyrix CPUs are -known- to support
1324                  * extended cpuid operations.
1325                  */
1326                 if (x86_type == X86_TYPE_VIA_CYRIX_III ||
1327                     x86_type == X86_TYPE_CYRIX_GXm)
1328                         xcpuid++;
1329                 break;
1330         case X86_VENDOR_Centaur:
1331         case X86_VENDOR_TM:
1332         default:
1333                 xcpuid++;
1334                 break;
1335         }
1336 
1337         if (xcpuid) {
1338                 cp = &cpi->cpi_extd[0];
1339                 cp->cp_eax = 0x80000000;
1340                 cpi->cpi_xmaxeax = __cpuid_insn(cp);
1341         }
1342 
1343         if (cpi->cpi_xmaxeax & 0x80000000) {
1344 
1345                 if (cpi->cpi_xmaxeax > CPI_XMAXEAX_MAX)
1346                         cpi->cpi_xmaxeax = CPI_XMAXEAX_MAX;
1347 
1348                 switch (cpi->cpi_vendor) {
1349                 case X86_VENDOR_Intel:
1350                 case X86_VENDOR_AMD:
1351                         if (cpi->cpi_xmaxeax < 0x80000001)
1352                                 break;
1353                         cp = &cpi->cpi_extd[1];
1354                         cp->cp_eax = 0x80000001;
1355                         (void) __cpuid_insn(cp);
1356 
1357                         if (cpi->cpi_vendor == X86_VENDOR_AMD &&
1358                             cpi->cpi_family == 5 &&
1359                             cpi->cpi_model == 6 &&
1360                             cpi->cpi_step == 6) {
1361                                 /*
1362                                  * K6 model 6 uses bit 10 to indicate SYSC
1363                                  * Later models use bit 11. Fix it here.
1364                                  */
1365                                 if (cp->cp_edx & 0x400) {
1366                                         cp->cp_edx &= ~0x400;
1367                                         cp->cp_edx |= CPUID_AMD_EDX_SYSC;
1368                                 }
1369                         }
1370 
1371                         platform_cpuid_mangle(cpi->cpi_vendor, 0x80000001, cp);
1372 
1373                         /*
1374                          * Compute the additions to the kernel's feature word.
1375                          */
1376                         if (cp->cp_edx & CPUID_AMD_EDX_NX) {
1377                                 add_x86_feature(featureset, X86FSET_NX);
1378                         }
1379 
1380                         /*
1381                          * Regardless whether or not we boot 64-bit,
1382                          * we should have a way to identify whether
1383                          * the CPU is capable of running 64-bit.
1384                          */
1385                         if (cp->cp_edx & CPUID_AMD_EDX_LM) {
1386                                 add_x86_feature(featureset, X86FSET_64);
1387                         }
1388 
1389 #if defined(__amd64)
1390                         /* 1 GB large page - enable only for 64 bit kernel */
1391                         if (cp->cp_edx & CPUID_AMD_EDX_1GPG) {
1392                                 add_x86_feature(featureset, X86FSET_1GPG);
1393                         }
1394 #endif
1395 
1396                         if ((cpi->cpi_vendor == X86_VENDOR_AMD) &&
1397                             (cpi->cpi_std[1].cp_edx & CPUID_INTC_EDX_FXSR) &&
1398                             (cp->cp_ecx & CPUID_AMD_ECX_SSE4A)) {
1399                                 add_x86_feature(featureset, X86FSET_SSE4A);
1400                         }
1401 
1402                         /*
1403                          * If both the HTT and CMP_LGCY bits are set,
1404                          * then we're not actually HyperThreaded.  Read
1405                          * "AMD CPUID Specification" for more details.
1406                          */
1407                         if (cpi->cpi_vendor == X86_VENDOR_AMD &&
1408                             is_x86_feature(featureset, X86FSET_HTT) &&
1409                             (cp->cp_ecx & CPUID_AMD_ECX_CMP_LGCY)) {
1410                                 remove_x86_feature(featureset, X86FSET_HTT);
1411                                 add_x86_feature(featureset, X86FSET_CMP);
1412                         }
1413 #if defined(__amd64)
1414                         /*
1415                          * It's really tricky to support syscall/sysret in
1416                          * the i386 kernel; we rely on sysenter/sysexit
1417                          * instead.  In the amd64 kernel, things are -way-
1418                          * better.
1419                          */
1420                         if (cp->cp_edx & CPUID_AMD_EDX_SYSC) {
1421                                 add_x86_feature(featureset, X86FSET_ASYSC);
1422                         }
1423 
1424                         /*
1425                          * While we're thinking about system calls, note
1426                          * that AMD processors don't support sysenter
1427                          * in long mode at all, so don't try to program them.
1428                          */
1429                         if (x86_vendor == X86_VENDOR_AMD) {
1430                                 remove_x86_feature(featureset, X86FSET_SEP);
1431                         }
1432 #endif
1433                         if (cp->cp_edx & CPUID_AMD_EDX_TSCP) {
1434                                 add_x86_feature(featureset, X86FSET_TSCP);
1435                         }
1436 
1437                         if (cp->cp_ecx & CPUID_AMD_ECX_SVM) {
1438                                 add_x86_feature(featureset, X86FSET_SVM);
1439                         }
1440                         break;
1441                 default:
1442                         break;
1443                 }
1444 
1445                 /*
1446                  * Get CPUID data about processor cores and hyperthreads.
1447                  */
1448                 switch (cpi->cpi_vendor) {
1449                 case X86_VENDOR_Intel:
1450                         if (cpi->cpi_maxeax >= 4) {
1451                                 cp = &cpi->cpi_std[4];
1452                                 cp->cp_eax = 4;
1453                                 cp->cp_ecx = 0;
1454                                 (void) __cpuid_insn(cp);
1455                                 platform_cpuid_mangle(cpi->cpi_vendor, 4, cp);
1456                         }
1457                         /*FALLTHROUGH*/
1458                 case X86_VENDOR_AMD:
1459                         if (cpi->cpi_xmaxeax < 0x80000008)
1460                                 break;
1461                         cp = &cpi->cpi_extd[8];
1462                         cp->cp_eax = 0x80000008;
1463                         (void) __cpuid_insn(cp);
1464                         platform_cpuid_mangle(cpi->cpi_vendor, 0x80000008, cp);
1465 
1466                         /*
1467                          * Virtual and physical address limits from
1468                          * cpuid override previously guessed values.
1469                          */
1470                         cpi->cpi_pabits = BITX(cp->cp_eax, 7, 0);
1471                         cpi->cpi_vabits = BITX(cp->cp_eax, 15, 8);
1472                         break;
1473                 default:
1474                         break;
1475                 }
1476 
1477                 /*
1478                  * Derive the number of cores per chip
1479                  */
1480                 switch (cpi->cpi_vendor) {
1481                 case X86_VENDOR_Intel:
1482                         if (cpi->cpi_maxeax < 4) {
1483                                 cpi->cpi_ncore_per_chip = 1;
1484                                 break;
1485                         } else {
1486                                 cpi->cpi_ncore_per_chip =
1487                                     BITX((cpi)->cpi_std[4].cp_eax, 31, 26) + 1;
1488                         }
1489                         break;
1490                 case X86_VENDOR_AMD:
1491                         if (cpi->cpi_xmaxeax < 0x80000008) {
1492                                 cpi->cpi_ncore_per_chip = 1;
1493                                 break;
1494                         } else {
1495                                 /*
1496                                  * On family 0xf cpuid fn 2 ECX[7:0] "NC" is
1497                                  * 1 less than the number of physical cores on
1498                                  * the chip.  In family 0x10 this value can
1499                                  * be affected by "downcoring" - it reflects
1500                                  * 1 less than the number of cores actually
1501                                  * enabled on this node.
1502                                  */
1503                                 cpi->cpi_ncore_per_chip =
1504                                     BITX((cpi)->cpi_extd[8].cp_ecx, 7, 0) + 1;
1505                         }
1506                         break;
1507                 default:
1508                         cpi->cpi_ncore_per_chip = 1;
1509                         break;
1510                 }
1511 
1512                 /*
1513                  * Get CPUID data about TSC Invariance in Deep C-State.
1514                  */
1515                 switch (cpi->cpi_vendor) {
1516                 case X86_VENDOR_Intel:
1517                         if (cpi->cpi_maxeax >= 7) {
1518                                 cp = &cpi->cpi_extd[7];
1519                                 cp->cp_eax = 0x80000007;
1520                                 cp->cp_ecx = 0;
1521                                 (void) __cpuid_insn(cp);
1522                         }
1523                         break;
1524                 default:
1525                         break;
1526                 }
1527         } else {
1528                 cpi->cpi_ncore_per_chip = 1;
1529         }
1530 
1531         /*
1532          * If more than one core, then this processor is CMP.
1533          */
1534         if (cpi->cpi_ncore_per_chip > 1) {
1535                 add_x86_feature(featureset, X86FSET_CMP);
1536         }
1537 
1538         /*
1539          * If the number of cores is the same as the number
1540          * of CPUs, then we cannot have HyperThreading.
1541          */
1542         if (cpi->cpi_ncpu_per_chip == cpi->cpi_ncore_per_chip) {
1543                 remove_x86_feature(featureset, X86FSET_HTT);
1544         }
1545 
1546         cpi->cpi_apicid = CPI_APIC_ID(cpi);
1547         cpi->cpi_procnodes_per_pkg = 1;
1548         if (is_x86_feature(featureset, X86FSET_HTT) == B_FALSE &&
1549             is_x86_feature(featureset, X86FSET_CMP) == B_FALSE) {
1550                 /*
1551                  * Single-core single-threaded processors.
1552                  */
1553                 cpi->cpi_chipid = -1;
1554                 cpi->cpi_clogid = 0;
1555                 cpi->cpi_coreid = cpu->cpu_id;
1556                 cpi->cpi_pkgcoreid = 0;
1557                 if (cpi->cpi_vendor == X86_VENDOR_AMD)
1558                         cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 3, 0);
1559                 else
1560                         cpi->cpi_procnodeid = cpi->cpi_chipid;
1561         } else if (cpi->cpi_ncpu_per_chip > 1) {
1562                 if (cpi->cpi_vendor == X86_VENDOR_Intel)
1563                         cpuid_intel_getids(cpu, featureset);
1564                 else if (cpi->cpi_vendor == X86_VENDOR_AMD)
1565                         cpuid_amd_getids(cpu);
1566                 else {
1567                         /*
1568                          * All other processors are currently
1569                          * assumed to have single cores.
1570                          */
1571                         cpi->cpi_coreid = cpi->cpi_chipid;
1572                         cpi->cpi_pkgcoreid = 0;
1573                         cpi->cpi_procnodeid = cpi->cpi_chipid;
1574                 }
1575         }
1576 
1577         /*
1578          * Synthesize chip "revision" and socket type
1579          */
1580         cpi->cpi_chiprev = _cpuid_chiprev(cpi->cpi_vendor, cpi->cpi_family,
1581             cpi->cpi_model, cpi->cpi_step);
1582         cpi->cpi_chiprevstr = _cpuid_chiprevstr(cpi->cpi_vendor,
1583             cpi->cpi_family, cpi->cpi_model, cpi->cpi_step);
1584         cpi->cpi_socket = _cpuid_skt(cpi->cpi_vendor, cpi->cpi_family,
1585             cpi->cpi_model, cpi->cpi_step);
1586 
1587 pass1_done:
1588         cpi->cpi_pass = 1;
1589 }
1590 
1591 /*
1592  * Make copies of the cpuid table entries we depend on, in
1593  * part for ease of parsing now, in part so that we have only
1594  * one place to correct any of it, in part for ease of
1595  * later export to userland, and in part so we can look at
1596  * this stuff in a crash dump.
1597  */
1598 
1599 /*ARGSUSED*/
1600 void
1601 cpuid_pass2(cpu_t *cpu)
1602 {
1603         uint_t n, nmax;
1604         int i;
1605         struct cpuid_regs *cp;
1606         uint8_t *dp;
1607         uint32_t *iptr;
1608         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
1609 
1610         ASSERT(cpi->cpi_pass == 1);
1611 
1612         if (cpi->cpi_maxeax < 1)
1613                 goto pass2_done;
1614 
1615         if ((nmax = cpi->cpi_maxeax + 1) > NMAX_CPI_STD)
1616                 nmax = NMAX_CPI_STD;
1617         /*
1618          * (We already handled n == 0 and n == 1 in pass 1)
1619          */
1620         for (n = 2, cp = &cpi->cpi_std[2]; n < nmax; n++, cp++) {
1621                 cp->cp_eax = n;
1622 
1623                 /*
1624                  * CPUID function 4 expects %ecx to be initialized
1625                  * with an index which indicates which cache to return
1626                  * information about. The OS is expected to call function 4
1627                  * with %ecx set to 0, 1, 2, ... until it returns with
1628                  * EAX[4:0] set to 0, which indicates there are no more
1629                  * caches.
1630                  *
1631                  * Here, populate cpi_std[4] with the information returned by
1632                  * function 4 when %ecx == 0, and do the rest in cpuid_pass3()
1633                  * when dynamic memory allocation becomes available.
1634                  *
1635                  * Note: we need to explicitly initialize %ecx here, since
1636                  * function 4 may have been previously invoked.
1637                  */
1638                 if (n == 4)
1639                         cp->cp_ecx = 0;
1640 
1641                 (void) __cpuid_insn(cp);
1642                 platform_cpuid_mangle(cpi->cpi_vendor, n, cp);
1643                 switch (n) {
1644                 case 2:
1645                         /*
1646                          * "the lower 8 bits of the %eax register
1647                          * contain a value that identifies the number
1648                          * of times the cpuid [instruction] has to be
1649                          * executed to obtain a complete image of the
1650                          * processor's caching systems."
1651                          *
1652                          * How *do* they make this stuff up?
1653                          */
1654                         cpi->cpi_ncache = sizeof (*cp) *
1655                             BITX(cp->cp_eax, 7, 0);
1656                         if (cpi->cpi_ncache == 0)
1657                                 break;
1658                         cpi->cpi_ncache--;   /* skip count byte */
1659 
1660                         /*
1661                          * Well, for now, rather than attempt to implement
1662                          * this slightly dubious algorithm, we just look
1663                          * at the first 15 ..
1664                          */
1665                         if (cpi->cpi_ncache > (sizeof (*cp) - 1))
1666                                 cpi->cpi_ncache = sizeof (*cp) - 1;
1667 
1668                         dp = cpi->cpi_cacheinfo;
1669                         if (BITX(cp->cp_eax, 31, 31) == 0) {
1670                                 uint8_t *p = (void *)&cp->cp_eax;
1671                                 for (i = 1; i < 4; i++)
1672                                         if (p[i] != 0)
1673                                                 *dp++ = p[i];
1674                         }
1675                         if (BITX(cp->cp_ebx, 31, 31) == 0) {
1676                                 uint8_t *p = (void *)&cp->cp_ebx;
1677                                 for (i = 0; i < 4; i++)
1678                                         if (p[i] != 0)
1679                                                 *dp++ = p[i];
1680                         }
1681                         if (BITX(cp->cp_ecx, 31, 31) == 0) {
1682                                 uint8_t *p = (void *)&cp->cp_ecx;
1683                                 for (i = 0; i < 4; i++)
1684                                         if (p[i] != 0)
1685                                                 *dp++ = p[i];
1686                         }
1687                         if (BITX(cp->cp_edx, 31, 31) == 0) {
1688                                 uint8_t *p = (void *)&cp->cp_edx;
1689                                 for (i = 0; i < 4; i++)
1690                                         if (p[i] != 0)
1691                                                 *dp++ = p[i];
1692                         }
1693                         break;
1694 
1695                 case 3: /* Processor serial number, if PSN supported */
1696                         break;
1697 
1698                 case 4: /* Deterministic cache parameters */
1699                         break;
1700 
1701                 case 5: /* Monitor/Mwait parameters */
1702                 {
1703                         size_t mwait_size;
1704 
1705                         /*
1706                          * check cpi_mwait.support which was set in cpuid_pass1
1707                          */
1708                         if (!(cpi->cpi_mwait.support & MWAIT_SUPPORT))
1709                                 break;
1710 
1711                         /*
1712                          * Protect ourself from insane mwait line size.
1713                          * Workaround for incomplete hardware emulator(s).
1714                          */
1715                         mwait_size = (size_t)MWAIT_SIZE_MAX(cpi);
1716                         if (mwait_size < sizeof (uint32_t) ||
1717                             !ISP2(mwait_size)) {
1718 #if DEBUG
1719                                 cmn_err(CE_NOTE, "Cannot handle cpu %d mwait "
1720                                     "size %ld", cpu->cpu_id, (long)mwait_size);
1721 #endif
1722                                 break;
1723                         }
1724 
1725                         cpi->cpi_mwait.mon_min = (size_t)MWAIT_SIZE_MIN(cpi);
1726                         cpi->cpi_mwait.mon_max = mwait_size;
1727                         if (MWAIT_EXTENSION(cpi)) {
1728                                 cpi->cpi_mwait.support |= MWAIT_EXTENSIONS;
1729                                 if (MWAIT_INT_ENABLE(cpi))
1730                                         cpi->cpi_mwait.support |=
1731                                             MWAIT_ECX_INT_ENABLE;
1732                         }
1733                         break;
1734                 }
1735                 default:
1736                         break;
1737                 }
1738         }
1739 
1740         if (cpi->cpi_maxeax >= 0xB && cpi->cpi_vendor == X86_VENDOR_Intel) {
1741                 struct cpuid_regs regs;
1742 
1743                 cp = &regs;
1744                 cp->cp_eax = 0xB;
1745                 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
1746 
1747                 (void) __cpuid_insn(cp);
1748 
1749                 /*
1750                  * Check CPUID.EAX=0BH, ECX=0H:EBX is non-zero, which
1751                  * indicates that the extended topology enumeration leaf is
1752                  * available.
1753                  */
1754                 if (cp->cp_ebx) {
1755                         uint32_t x2apic_id;
1756                         uint_t coreid_shift = 0;
1757                         uint_t ncpu_per_core = 1;
1758                         uint_t chipid_shift = 0;
1759                         uint_t ncpu_per_chip = 1;
1760                         uint_t i;
1761                         uint_t level;
1762 
1763                         for (i = 0; i < CPI_FNB_ECX_MAX; i++) {
1764                                 cp->cp_eax = 0xB;
1765                                 cp->cp_ecx = i;
1766 
1767                                 (void) __cpuid_insn(cp);
1768                                 level = CPI_CPU_LEVEL_TYPE(cp);
1769 
1770                                 if (level == 1) {
1771                                         x2apic_id = cp->cp_edx;
1772                                         coreid_shift = BITX(cp->cp_eax, 4, 0);
1773                                         ncpu_per_core = BITX(cp->cp_ebx, 15, 0);
1774                                 } else if (level == 2) {
1775                                         x2apic_id = cp->cp_edx;
1776                                         chipid_shift = BITX(cp->cp_eax, 4, 0);
1777                                         ncpu_per_chip = BITX(cp->cp_ebx, 15, 0);
1778                                 }
1779                         }
1780 
1781                         cpi->cpi_apicid = x2apic_id;
1782                         cpi->cpi_ncpu_per_chip = ncpu_per_chip;
1783                         cpi->cpi_ncore_per_chip = ncpu_per_chip /
1784                             ncpu_per_core;
1785                         cpi->cpi_chipid = x2apic_id >> chipid_shift;
1786                         cpi->cpi_clogid = x2apic_id & ((1 << chipid_shift) - 1);
1787                         cpi->cpi_coreid = x2apic_id >> coreid_shift;
1788                         cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
1789                 }
1790 
1791                 /* Make cp NULL so that we don't stumble on others */
1792                 cp = NULL;
1793         }
1794 
1795         /*
1796          * XSAVE enumeration
1797          */
1798         if (cpi->cpi_maxeax >= 0xD) {
1799                 struct cpuid_regs regs;
1800                 boolean_t cpuid_d_valid = B_TRUE;
1801 
1802                 cp = &regs;
1803                 cp->cp_eax = 0xD;
1804                 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
1805 
1806                 (void) __cpuid_insn(cp);
1807 
1808                 /*
1809                  * Sanity checks for debug
1810                  */
1811                 if ((cp->cp_eax & XFEATURE_LEGACY_FP) == 0 ||
1812                     (cp->cp_eax & XFEATURE_SSE) == 0) {
1813                         cpuid_d_valid = B_FALSE;
1814                 }
1815 
1816                 cpi->cpi_xsave.xsav_hw_features_low = cp->cp_eax;
1817                 cpi->cpi_xsave.xsav_hw_features_high = cp->cp_edx;
1818                 cpi->cpi_xsave.xsav_max_size = cp->cp_ecx;
1819 
1820                 /*
1821                  * If the hw supports AVX, get the size and offset in the save
1822                  * area for the ymm state.
1823                  */
1824                 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_AVX) {
1825                         cp->cp_eax = 0xD;
1826                         cp->cp_ecx = 2;
1827                         cp->cp_edx = cp->cp_ebx = 0;
1828 
1829                         (void) __cpuid_insn(cp);
1830 
1831                         if (cp->cp_ebx != CPUID_LEAFD_2_YMM_OFFSET ||
1832                             cp->cp_eax != CPUID_LEAFD_2_YMM_SIZE) {
1833                                 cpuid_d_valid = B_FALSE;
1834                         }
1835 
1836                         cpi->cpi_xsave.ymm_size = cp->cp_eax;
1837                         cpi->cpi_xsave.ymm_offset = cp->cp_ebx;
1838                 }
1839 
1840                 if (is_x86_feature(x86_featureset, X86FSET_XSAVE)) {
1841                         xsave_state_size = 0;
1842                 } else if (cpuid_d_valid) {
1843                         xsave_state_size = cpi->cpi_xsave.xsav_max_size;
1844                 } else {
1845                         /* Broken CPUID 0xD, probably in HVM */
1846                         cmn_err(CE_WARN, "cpu%d: CPUID.0xD returns invalid "
1847                             "value: hw_low = %d, hw_high = %d, xsave_size = %d"
1848                             ", ymm_size = %d, ymm_offset = %d\n",
1849                             cpu->cpu_id, cpi->cpi_xsave.xsav_hw_features_low,
1850                             cpi->cpi_xsave.xsav_hw_features_high,
1851                             (int)cpi->cpi_xsave.xsav_max_size,
1852                             (int)cpi->cpi_xsave.ymm_size,
1853                             (int)cpi->cpi_xsave.ymm_offset);
1854 
1855                         if (xsave_state_size != 0) {
1856                                 /*
1857                                  * This must be a non-boot CPU. We cannot
1858                                  * continue, because boot cpu has already
1859                                  * enabled XSAVE.
1860                                  */
1861                                 ASSERT(cpu->cpu_id != 0);
1862                                 cmn_err(CE_PANIC, "cpu%d: we have already "
1863                                     "enabled XSAVE on boot cpu, cannot "
1864                                     "continue.", cpu->cpu_id);
1865                         } else {
1866                                 /*
1867                                  * Must be from boot CPU, OK to disable XSAVE.
1868                                  */
1869                                 ASSERT(cpu->cpu_id == 0);
1870                                 remove_x86_feature(x86_featureset,
1871                                     X86FSET_XSAVE);
1872                                 remove_x86_feature(x86_featureset, X86FSET_AVX);
1873                                 CPI_FEATURES_ECX(cpi) &= ~CPUID_INTC_ECX_XSAVE;
1874                                 CPI_FEATURES_ECX(cpi) &= ~CPUID_INTC_ECX_AVX;
1875                                 xsave_force_disable = B_TRUE;
1876                         }
1877                 }
1878         }
1879 
1880 
1881         if ((cpi->cpi_xmaxeax & 0x80000000) == 0)
1882                 goto pass2_done;
1883 
1884         if ((nmax = cpi->cpi_xmaxeax - 0x80000000 + 1) > NMAX_CPI_EXTD)
1885                 nmax = NMAX_CPI_EXTD;
1886         /*
1887          * Copy the extended properties, fixing them as we go.
1888          * (We already handled n == 0 and n == 1 in pass 1)
1889          */
1890         iptr = (void *)cpi->cpi_brandstr;
1891         for (n = 2, cp = &cpi->cpi_extd[2]; n < nmax; cp++, n++) {
1892                 cp->cp_eax = 0x80000000 + n;
1893                 (void) __cpuid_insn(cp);
1894                 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000000 + n, cp);
1895                 switch (n) {
1896                 case 2:
1897                 case 3:
1898                 case 4:
1899                         /*
1900                          * Extract the brand string
1901                          */
1902                         *iptr++ = cp->cp_eax;
1903                         *iptr++ = cp->cp_ebx;
1904                         *iptr++ = cp->cp_ecx;
1905                         *iptr++ = cp->cp_edx;
1906                         break;
1907                 case 5:
1908                         switch (cpi->cpi_vendor) {
1909                         case X86_VENDOR_AMD:
1910                                 /*
1911                                  * The Athlon and Duron were the first
1912                                  * parts to report the sizes of the
1913                                  * TLB for large pages. Before then,
1914                                  * we don't trust the data.
1915                                  */
1916                                 if (cpi->cpi_family < 6 ||
1917                                     (cpi->cpi_family == 6 &&
1918                                     cpi->cpi_model < 1))
1919                                         cp->cp_eax = 0;
1920                                 break;
1921                         default:
1922                                 break;
1923                         }
1924                         break;
1925                 case 6:
1926                         switch (cpi->cpi_vendor) {
1927                         case X86_VENDOR_AMD:
1928                                 /*
1929                                  * The Athlon and Duron were the first
1930                                  * AMD parts with L2 TLB's.
1931                                  * Before then, don't trust the data.
1932                                  */
1933                                 if (cpi->cpi_family < 6 ||
1934                                     cpi->cpi_family == 6 &&
1935                                     cpi->cpi_model < 1)
1936                                         cp->cp_eax = cp->cp_ebx = 0;
1937                                 /*
1938                                  * AMD Duron rev A0 reports L2
1939                                  * cache size incorrectly as 1K
1940                                  * when it is really 64K
1941                                  */
1942                                 if (cpi->cpi_family == 6 &&
1943                                     cpi->cpi_model == 3 &&
1944                                     cpi->cpi_step == 0) {
1945                                         cp->cp_ecx &= 0xffff;
1946                                         cp->cp_ecx |= 0x400000;
1947                                 }
1948                                 break;
1949                         case X86_VENDOR_Cyrix:  /* VIA C3 */
1950                                 /*
1951                                  * VIA C3 processors are a bit messed
1952                                  * up w.r.t. encoding cache sizes in %ecx
1953                                  */
1954                                 if (cpi->cpi_family != 6)
1955                                         break;
1956                                 /*
1957                                  * model 7 and 8 were incorrectly encoded
1958                                  *
1959                                  * xxx is model 8 really broken?
1960                                  */
1961                                 if (cpi->cpi_model == 7 ||
1962                                     cpi->cpi_model == 8)
1963                                         cp->cp_ecx =
1964                                             BITX(cp->cp_ecx, 31, 24) << 16 |
1965                                             BITX(cp->cp_ecx, 23, 16) << 12 |
1966                                             BITX(cp->cp_ecx, 15, 8) << 8 |
1967                                             BITX(cp->cp_ecx, 7, 0);
1968                                 /*
1969                                  * model 9 stepping 1 has wrong associativity
1970                                  */
1971                                 if (cpi->cpi_model == 9 && cpi->cpi_step == 1)
1972                                         cp->cp_ecx |= 8 << 12;
1973                                 break;
1974                         case X86_VENDOR_Intel:
1975                                 /*
1976                                  * Extended L2 Cache features function.
1977                                  * First appeared on Prescott.
1978                                  */
1979                         default:
1980                                 break;
1981                         }
1982                         break;
1983                 default:
1984                         break;
1985                 }
1986         }
1987 
1988 pass2_done:
1989         cpi->cpi_pass = 2;
1990 }
1991 
1992 static const char *
1993 intel_cpubrand(const struct cpuid_info *cpi)
1994 {
1995         int i;
1996 
1997         if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
1998             cpi->cpi_maxeax < 1 || cpi->cpi_family < 5)
1999                 return ("i486");
2000 
2001         switch (cpi->cpi_family) {
2002         case 5:
2003                 return ("Intel Pentium(r)");
2004         case 6:
2005                 switch (cpi->cpi_model) {
2006                         uint_t celeron, xeon;
2007                         const struct cpuid_regs *cp;
2008                 case 0:
2009                 case 1:
2010                 case 2:
2011                         return ("Intel Pentium(r) Pro");
2012                 case 3:
2013                 case 4:
2014                         return ("Intel Pentium(r) II");
2015                 case 6:
2016                         return ("Intel Celeron(r)");
2017                 case 5:
2018                 case 7:
2019                         celeron = xeon = 0;
2020                         cp = &cpi->cpi_std[2];   /* cache info */
2021 
2022                         for (i = 1; i < 4; i++) {
2023                                 uint_t tmp;
2024 
2025                                 tmp = (cp->cp_eax >> (8 * i)) & 0xff;
2026                                 if (tmp == 0x40)
2027                                         celeron++;
2028                                 if (tmp >= 0x44 && tmp <= 0x45)
2029                                         xeon++;
2030                         }
2031 
2032                         for (i = 0; i < 2; i++) {
2033                                 uint_t tmp;
2034 
2035                                 tmp = (cp->cp_ebx >> (8 * i)) & 0xff;
2036                                 if (tmp == 0x40)
2037                                         celeron++;
2038                                 else if (tmp >= 0x44 && tmp <= 0x45)
2039                                         xeon++;
2040                         }
2041 
2042                         for (i = 0; i < 4; i++) {
2043                                 uint_t tmp;
2044 
2045                                 tmp = (cp->cp_ecx >> (8 * i)) & 0xff;
2046                                 if (tmp == 0x40)
2047                                         celeron++;
2048                                 else if (tmp >= 0x44 && tmp <= 0x45)
2049                                         xeon++;
2050                         }
2051 
2052                         for (i = 0; i < 4; i++) {
2053                                 uint_t tmp;
2054 
2055                                 tmp = (cp->cp_edx >> (8 * i)) & 0xff;
2056                                 if (tmp == 0x40)
2057                                         celeron++;
2058                                 else if (tmp >= 0x44 && tmp <= 0x45)
2059                                         xeon++;
2060                         }
2061 
2062                         if (celeron)
2063                                 return ("Intel Celeron(r)");
2064                         if (xeon)
2065                                 return (cpi->cpi_model == 5 ?
2066                                     "Intel Pentium(r) II Xeon(tm)" :
2067                                     "Intel Pentium(r) III Xeon(tm)");
2068                         return (cpi->cpi_model == 5 ?
2069                             "Intel Pentium(r) II or Pentium(r) II Xeon(tm)" :
2070                             "Intel Pentium(r) III or Pentium(r) III Xeon(tm)");
2071                 default:
2072                         break;
2073                 }
2074         default:
2075                 break;
2076         }
2077 
2078         /* BrandID is present if the field is nonzero */
2079         if (cpi->cpi_brandid != 0) {
2080                 static const struct {
2081                         uint_t bt_bid;
2082                         const char *bt_str;
2083                 } brand_tbl[] = {
2084                         { 0x1,  "Intel(r) Celeron(r)" },
2085                         { 0x2,  "Intel(r) Pentium(r) III" },
2086                         { 0x3,  "Intel(r) Pentium(r) III Xeon(tm)" },
2087                         { 0x4,  "Intel(r) Pentium(r) III" },
2088                         { 0x6,  "Mobile Intel(r) Pentium(r) III" },
2089                         { 0x7,  "Mobile Intel(r) Celeron(r)" },
2090                         { 0x8,  "Intel(r) Pentium(r) 4" },
2091                         { 0x9,  "Intel(r) Pentium(r) 4" },
2092                         { 0xa,  "Intel(r) Celeron(r)" },
2093                         { 0xb,  "Intel(r) Xeon(tm)" },
2094                         { 0xc,  "Intel(r) Xeon(tm) MP" },
2095                         { 0xe,  "Mobile Intel(r) Pentium(r) 4" },
2096                         { 0xf,  "Mobile Intel(r) Celeron(r)" },
2097                         { 0x11, "Mobile Genuine Intel(r)" },
2098                         { 0x12, "Intel(r) Celeron(r) M" },
2099                         { 0x13, "Mobile Intel(r) Celeron(r)" },
2100                         { 0x14, "Intel(r) Celeron(r)" },
2101                         { 0x15, "Mobile Genuine Intel(r)" },
2102                         { 0x16, "Intel(r) Pentium(r) M" },
2103                         { 0x17, "Mobile Intel(r) Celeron(r)" }
2104                 };
2105                 uint_t btblmax = sizeof (brand_tbl) / sizeof (brand_tbl[0]);
2106                 uint_t sgn;
2107 
2108                 sgn = (cpi->cpi_family << 8) |
2109                     (cpi->cpi_model << 4) | cpi->cpi_step;
2110 
2111                 for (i = 0; i < btblmax; i++)
2112                         if (brand_tbl[i].bt_bid == cpi->cpi_brandid)
2113                                 break;
2114                 if (i < btblmax) {
2115                         if (sgn == 0x6b1 && cpi->cpi_brandid == 3)
2116                                 return ("Intel(r) Celeron(r)");
2117                         if (sgn < 0xf13 && cpi->cpi_brandid == 0xb)
2118                                 return ("Intel(r) Xeon(tm) MP");
2119                         if (sgn < 0xf13 && cpi->cpi_brandid == 0xe)
2120                                 return ("Intel(r) Xeon(tm)");
2121                         return (brand_tbl[i].bt_str);
2122                 }
2123         }
2124 
2125         return (NULL);
2126 }
2127 
2128 static const char *
2129 amd_cpubrand(const struct cpuid_info *cpi)
2130 {
2131         if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2132             cpi->cpi_maxeax < 1 || cpi->cpi_family < 5)
2133                 return ("i486 compatible");
2134 
2135         switch (cpi->cpi_family) {
2136         case 5:
2137                 switch (cpi->cpi_model) {
2138                 case 0:
2139                 case 1:
2140                 case 2:
2141                 case 3:
2142                 case 4:
2143                 case 5:
2144                         return ("AMD-K5(r)");
2145                 case 6:
2146                 case 7:
2147                         return ("AMD-K6(r)");
2148                 case 8:
2149                         return ("AMD-K6(r)-2");
2150                 case 9:
2151                         return ("AMD-K6(r)-III");
2152                 default:
2153                         return ("AMD (family 5)");
2154                 }
2155         case 6:
2156                 switch (cpi->cpi_model) {
2157                 case 1:
2158                         return ("AMD-K7(tm)");
2159                 case 0:
2160                 case 2:
2161                 case 4:
2162                         return ("AMD Athlon(tm)");
2163                 case 3:
2164                 case 7:
2165                         return ("AMD Duron(tm)");
2166                 case 6:
2167                 case 8:
2168                 case 10:
2169                         /*
2170                          * Use the L2 cache size to distinguish
2171                          */
2172                         return ((cpi->cpi_extd[6].cp_ecx >> 16) >= 256 ?
2173                             "AMD Athlon(tm)" : "AMD Duron(tm)");
2174                 default:
2175                         return ("AMD (family 6)");
2176                 }
2177         default:
2178                 break;
2179         }
2180 
2181         if (cpi->cpi_family == 0xf && cpi->cpi_model == 5 &&
2182             cpi->cpi_brandid != 0) {
2183                 switch (BITX(cpi->cpi_brandid, 7, 5)) {
2184                 case 3:
2185                         return ("AMD Opteron(tm) UP 1xx");
2186                 case 4:
2187                         return ("AMD Opteron(tm) DP 2xx");
2188                 case 5:
2189                         return ("AMD Opteron(tm) MP 8xx");
2190                 default:
2191                         return ("AMD Opteron(tm)");
2192                 }
2193         }
2194 
2195         return (NULL);
2196 }
2197 
2198 static const char *
2199 cyrix_cpubrand(struct cpuid_info *cpi, uint_t type)
2200 {
2201         if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2202             cpi->cpi_maxeax < 1 || cpi->cpi_family < 5 ||
2203             type == X86_TYPE_CYRIX_486)
2204                 return ("i486 compatible");
2205 
2206         switch (type) {
2207         case X86_TYPE_CYRIX_6x86:
2208                 return ("Cyrix 6x86");
2209         case X86_TYPE_CYRIX_6x86L:
2210                 return ("Cyrix 6x86L");
2211         case X86_TYPE_CYRIX_6x86MX:
2212                 return ("Cyrix 6x86MX");
2213         case X86_TYPE_CYRIX_GXm:
2214                 return ("Cyrix GXm");
2215         case X86_TYPE_CYRIX_MediaGX:
2216                 return ("Cyrix MediaGX");
2217         case X86_TYPE_CYRIX_MII:
2218                 return ("Cyrix M2");
2219         case X86_TYPE_VIA_CYRIX_III:
2220                 return ("VIA Cyrix M3");
2221         default:
2222                 /*
2223                  * Have another wild guess ..
2224                  */
2225                 if (cpi->cpi_family == 4 && cpi->cpi_model == 9)
2226                         return ("Cyrix 5x86");
2227                 else if (cpi->cpi_family == 5) {
2228                         switch (cpi->cpi_model) {
2229                         case 2:
2230                                 return ("Cyrix 6x86");  /* Cyrix M1 */
2231                         case 4:
2232                                 return ("Cyrix MediaGX");
2233                         default:
2234                                 break;
2235                         }
2236                 } else if (cpi->cpi_family == 6) {
2237                         switch (cpi->cpi_model) {
2238                         case 0:
2239                                 return ("Cyrix 6x86MX"); /* Cyrix M2? */
2240                         case 5:
2241                         case 6:
2242                         case 7:
2243                         case 8:
2244                         case 9:
2245                                 return ("VIA C3");
2246                         default:
2247                                 break;
2248                         }
2249                 }
2250                 break;
2251         }
2252         return (NULL);
2253 }
2254 
2255 /*
2256  * This only gets called in the case that the CPU extended
2257  * feature brand string (0x80000002, 0x80000003, 0x80000004)
2258  * aren't available, or contain null bytes for some reason.
2259  */
2260 static void
2261 fabricate_brandstr(struct cpuid_info *cpi)
2262 {
2263         const char *brand = NULL;
2264 
2265         switch (cpi->cpi_vendor) {
2266         case X86_VENDOR_Intel:
2267                 brand = intel_cpubrand(cpi);
2268                 break;
2269         case X86_VENDOR_AMD:
2270                 brand = amd_cpubrand(cpi);
2271                 break;
2272         case X86_VENDOR_Cyrix:
2273                 brand = cyrix_cpubrand(cpi, x86_type);
2274                 break;
2275         case X86_VENDOR_NexGen:
2276                 if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
2277                         brand = "NexGen Nx586";
2278                 break;
2279         case X86_VENDOR_Centaur:
2280                 if (cpi->cpi_family == 5)
2281                         switch (cpi->cpi_model) {
2282                         case 4:
2283                                 brand = "Centaur C6";
2284                                 break;
2285                         case 8:
2286                                 brand = "Centaur C2";
2287                                 break;
2288                         case 9:
2289                                 brand = "Centaur C3";
2290                                 break;
2291                         default:
2292                                 break;
2293                         }
2294                 break;
2295         case X86_VENDOR_Rise:
2296                 if (cpi->cpi_family == 5 &&
2297                     (cpi->cpi_model == 0 || cpi->cpi_model == 2))
2298                         brand = "Rise mP6";
2299                 break;
2300         case X86_VENDOR_SiS:
2301                 if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
2302                         brand = "SiS 55x";
2303                 break;
2304         case X86_VENDOR_TM:
2305                 if (cpi->cpi_family == 5 && cpi->cpi_model == 4)
2306                         brand = "Transmeta Crusoe TM3x00 or TM5x00";
2307                 break;
2308         case X86_VENDOR_NSC:
2309         case X86_VENDOR_UMC:
2310         default:
2311                 break;
2312         }
2313         if (brand) {
2314                 (void) strcpy((char *)cpi->cpi_brandstr, brand);
2315                 return;
2316         }
2317 
2318         /*
2319          * If all else fails ...
2320          */
2321         (void) snprintf(cpi->cpi_brandstr, sizeof (cpi->cpi_brandstr),
2322             "%s %d.%d.%d", cpi->cpi_vendorstr, cpi->cpi_family,
2323             cpi->cpi_model, cpi->cpi_step);
2324 }
2325 
2326 /*
2327  * This routine is called just after kernel memory allocation
2328  * becomes available on cpu0, and as part of mp_startup() on
2329  * the other cpus.
2330  *
2331  * Fixup the brand string, and collect any information from cpuid
2332  * that requires dynamicically allocated storage to represent.
2333  */
2334 /*ARGSUSED*/
2335 void
2336 cpuid_pass3(cpu_t *cpu)
2337 {
2338         int     i, max, shft, level, size;
2339         struct cpuid_regs regs;
2340         struct cpuid_regs *cp;
2341         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2342 
2343         ASSERT(cpi->cpi_pass == 2);
2344 
2345         /*
2346          * Function 4: Deterministic cache parameters
2347          *
2348          * Take this opportunity to detect the number of threads
2349          * sharing the last level cache, and construct a corresponding
2350          * cache id. The respective cpuid_info members are initialized
2351          * to the default case of "no last level cache sharing".
2352          */
2353         cpi->cpi_ncpu_shr_last_cache = 1;
2354         cpi->cpi_last_lvl_cacheid = cpu->cpu_id;
2355 
2356         if (cpi->cpi_maxeax >= 4 && cpi->cpi_vendor == X86_VENDOR_Intel) {
2357 
2358                 /*
2359                  * Find the # of elements (size) returned by fn 4, and along
2360                  * the way detect last level cache sharing details.
2361                  */
2362                 bzero(&regs, sizeof (regs));
2363                 cp = &regs;
2364                 for (i = 0, max = 0; i < CPI_FN4_ECX_MAX; i++) {
2365                         cp->cp_eax = 4;
2366                         cp->cp_ecx = i;
2367 
2368                         (void) __cpuid_insn(cp);
2369 
2370                         if (CPI_CACHE_TYPE(cp) == 0)
2371                                 break;
2372                         level = CPI_CACHE_LVL(cp);
2373                         if (level > max) {
2374                                 max = level;
2375                                 cpi->cpi_ncpu_shr_last_cache =
2376                                     CPI_NTHR_SHR_CACHE(cp) + 1;
2377                         }
2378                 }
2379                 cpi->cpi_std_4_size = size = i;
2380 
2381                 /*
2382                  * Allocate the cpi_std_4 array. The first element
2383                  * references the regs for fn 4, %ecx == 0, which
2384                  * cpuid_pass2() stashed in cpi->cpi_std[4].
2385                  */
2386                 if (size > 0) {
2387                         cpi->cpi_std_4 =
2388                             kmem_alloc(size * sizeof (cp), KM_SLEEP);
2389                         cpi->cpi_std_4[0] = &cpi->cpi_std[4];
2390 
2391                         /*
2392                          * Allocate storage to hold the additional regs
2393                          * for function 4, %ecx == 1 .. cpi_std_4_size.
2394                          *
2395                          * The regs for fn 4, %ecx == 0 has already
2396                          * been allocated as indicated above.
2397                          */
2398                         for (i = 1; i < size; i++) {
2399                                 cp = cpi->cpi_std_4[i] =
2400                                     kmem_zalloc(sizeof (regs), KM_SLEEP);
2401                                 cp->cp_eax = 4;
2402                                 cp->cp_ecx = i;
2403 
2404                                 (void) __cpuid_insn(cp);
2405                         }
2406                 }
2407                 /*
2408                  * Determine the number of bits needed to represent
2409                  * the number of CPUs sharing the last level cache.
2410                  *
2411                  * Shift off that number of bits from the APIC id to
2412                  * derive the cache id.
2413                  */
2414                 shft = 0;
2415                 for (i = 1; i < cpi->cpi_ncpu_shr_last_cache; i <<= 1)
2416                         shft++;
2417                 cpi->cpi_last_lvl_cacheid = cpi->cpi_apicid >> shft;
2418         }
2419 
2420         /*
2421          * Now fixup the brand string
2422          */
2423         if ((cpi->cpi_xmaxeax & 0x80000000) == 0) {
2424                 fabricate_brandstr(cpi);
2425         } else {
2426 
2427                 /*
2428                  * If we successfully extracted a brand string from the cpuid
2429                  * instruction, clean it up by removing leading spaces and
2430                  * similar junk.
2431                  */
2432                 if (cpi->cpi_brandstr[0]) {
2433                         size_t maxlen = sizeof (cpi->cpi_brandstr);
2434                         char *src, *dst;
2435 
2436                         dst = src = (char *)cpi->cpi_brandstr;
2437                         src[maxlen - 1] = '\0';
2438                         /*
2439                          * strip leading spaces
2440                          */
2441                         while (*src == ' ')
2442                                 src++;
2443                         /*
2444                          * Remove any 'Genuine' or "Authentic" prefixes
2445                          */
2446                         if (strncmp(src, "Genuine ", 8) == 0)
2447                                 src += 8;
2448                         if (strncmp(src, "Authentic ", 10) == 0)
2449                                 src += 10;
2450 
2451                         /*
2452                          * Now do an in-place copy.
2453                          * Map (R) to (r) and (TM) to (tm).
2454                          * The era of teletypes is long gone, and there's
2455                          * -really- no need to shout.
2456                          */
2457                         while (*src != '\0') {
2458                                 if (src[0] == '(') {
2459                                         if (strncmp(src + 1, "R)", 2) == 0) {
2460                                                 (void) strncpy(dst, "(r)", 3);
2461                                                 src += 3;
2462                                                 dst += 3;
2463                                                 continue;
2464                                         }
2465                                         if (strncmp(src + 1, "TM)", 3) == 0) {
2466                                                 (void) strncpy(dst, "(tm)", 4);
2467                                                 src += 4;
2468                                                 dst += 4;
2469                                                 continue;
2470                                         }
2471                                 }
2472                                 *dst++ = *src++;
2473                         }
2474                         *dst = '\0';
2475 
2476                         /*
2477                          * Finally, remove any trailing spaces
2478                          */
2479                         while (--dst > cpi->cpi_brandstr)
2480                                 if (*dst == ' ')
2481                                         *dst = '\0';
2482                                 else
2483                                         break;
2484                 } else
2485                         fabricate_brandstr(cpi);
2486         }
2487         cpi->cpi_pass = 3;
2488 }
2489 
2490 /*
2491  * This routine is called out of bind_hwcap() much later in the life
2492  * of the kernel (post_startup()).  The job of this routine is to resolve
2493  * the hardware feature support and kernel support for those features into
2494  * what we're actually going to tell applications via the aux vector.
2495  */
2496 uint_t
2497 cpuid_pass4(cpu_t *cpu)
2498 {
2499         struct cpuid_info *cpi;
2500         uint_t hwcap_flags = 0;
2501 
2502         if (cpu == NULL)
2503                 cpu = CPU;
2504         cpi = cpu->cpu_m.mcpu_cpi;
2505 
2506         ASSERT(cpi->cpi_pass == 3);
2507 
2508         if (cpi->cpi_maxeax >= 1) {
2509                 uint32_t *edx = &cpi->cpi_support[STD_EDX_FEATURES];
2510                 uint32_t *ecx = &cpi->cpi_support[STD_ECX_FEATURES];
2511 
2512                 *edx = CPI_FEATURES_EDX(cpi);
2513                 *ecx = CPI_FEATURES_ECX(cpi);
2514 
2515                 /*
2516                  * [these require explicit kernel support]
2517                  */
2518                 if (!is_x86_feature(x86_featureset, X86FSET_SEP))
2519                         *edx &= ~CPUID_INTC_EDX_SEP;
2520 
2521                 if (!is_x86_feature(x86_featureset, X86FSET_SSE))
2522                         *edx &= ~(CPUID_INTC_EDX_FXSR|CPUID_INTC_EDX_SSE);
2523                 if (!is_x86_feature(x86_featureset, X86FSET_SSE2))
2524                         *edx &= ~CPUID_INTC_EDX_SSE2;
2525 
2526                 if (!is_x86_feature(x86_featureset, X86FSET_HTT))
2527                         *edx &= ~CPUID_INTC_EDX_HTT;
2528 
2529                 if (!is_x86_feature(x86_featureset, X86FSET_SSE3))
2530                         *ecx &= ~CPUID_INTC_ECX_SSE3;
2531 
2532                 if (!is_x86_feature(x86_featureset, X86FSET_SSSE3))
2533                         *ecx &= ~CPUID_INTC_ECX_SSSE3;
2534                 if (!is_x86_feature(x86_featureset, X86FSET_SSE4_1))
2535                         *ecx &= ~CPUID_INTC_ECX_SSE4_1;
2536                 if (!is_x86_feature(x86_featureset, X86FSET_SSE4_2))
2537                         *ecx &= ~CPUID_INTC_ECX_SSE4_2;
2538                 if (!is_x86_feature(x86_featureset, X86FSET_AES))
2539                         *ecx &= ~CPUID_INTC_ECX_AES;
2540                 if (!is_x86_feature(x86_featureset, X86FSET_PCLMULQDQ))
2541                         *ecx &= ~CPUID_INTC_ECX_PCLMULQDQ;
2542                 if (!is_x86_feature(x86_featureset, X86FSET_XSAVE))
2543                         *ecx &= ~(CPUID_INTC_ECX_XSAVE |
2544                             CPUID_INTC_ECX_OSXSAVE);
2545                 if (!is_x86_feature(x86_featureset, X86FSET_AVX))
2546                         *ecx &= ~CPUID_INTC_ECX_AVX;
2547 
2548                 /*
2549                  * [no explicit support required beyond x87 fp context]
2550                  */
2551                 if (!fpu_exists)
2552                         *edx &= ~(CPUID_INTC_EDX_FPU | CPUID_INTC_EDX_MMX);
2553 
2554                 /*
2555                  * Now map the supported feature vector to things that we
2556                  * think userland will care about.
2557                  */
2558                 if (*edx & CPUID_INTC_EDX_SEP)
2559                         hwcap_flags |= AV_386_SEP;
2560                 if (*edx & CPUID_INTC_EDX_SSE)
2561                         hwcap_flags |= AV_386_FXSR | AV_386_SSE;
2562                 if (*edx & CPUID_INTC_EDX_SSE2)
2563                         hwcap_flags |= AV_386_SSE2;
2564                 if (*ecx & CPUID_INTC_ECX_SSE3)
2565                         hwcap_flags |= AV_386_SSE3;
2566                 if (*ecx & CPUID_INTC_ECX_SSSE3)
2567                         hwcap_flags |= AV_386_SSSE3;
2568                 if (*ecx & CPUID_INTC_ECX_SSE4_1)
2569                         hwcap_flags |= AV_386_SSE4_1;
2570                 if (*ecx & CPUID_INTC_ECX_SSE4_2)
2571                         hwcap_flags |= AV_386_SSE4_2;
2572                 if (*ecx & CPUID_INTC_ECX_MOVBE)
2573                         hwcap_flags |= AV_386_MOVBE;
2574                 if (*ecx & CPUID_INTC_ECX_AES)
2575                         hwcap_flags |= AV_386_AES;
2576                 if (*ecx & CPUID_INTC_ECX_PCLMULQDQ)
2577                         hwcap_flags |= AV_386_PCLMULQDQ;
2578                 if ((*ecx & CPUID_INTC_ECX_XSAVE) &&
2579                     (*ecx & CPUID_INTC_ECX_OSXSAVE))
2580                         hwcap_flags |= AV_386_XSAVE;
2581                 if (*ecx & CPUID_INTC_ECX_VMX)
2582                         hwcap_flags |= AV_386_VMX;
2583                 if (*ecx & CPUID_INTC_ECX_POPCNT)
2584                         hwcap_flags |= AV_386_POPCNT;
2585                 if (*edx & CPUID_INTC_EDX_FPU)
2586                         hwcap_flags |= AV_386_FPU;
2587                 if (*edx & CPUID_INTC_EDX_MMX)
2588                         hwcap_flags |= AV_386_MMX;
2589 
2590                 if (*edx & CPUID_INTC_EDX_TSC)
2591                         hwcap_flags |= AV_386_TSC;
2592                 if (*edx & CPUID_INTC_EDX_CX8)
2593                         hwcap_flags |= AV_386_CX8;
2594                 if (*edx & CPUID_INTC_EDX_CMOV)
2595                         hwcap_flags |= AV_386_CMOV;
2596                 if (*ecx & CPUID_INTC_ECX_CX16)
2597                         hwcap_flags |= AV_386_CX16;
2598         }
2599 
2600         if (cpi->cpi_xmaxeax < 0x80000001)
2601                 goto pass4_done;
2602 
2603         switch (cpi->cpi_vendor) {
2604                 struct cpuid_regs cp;
2605                 uint32_t *edx, *ecx;
2606 
2607         case X86_VENDOR_Intel:
2608                 /*
2609                  * Seems like Intel duplicated what we necessary
2610                  * here to make the initial crop of 64-bit OS's work.
2611                  * Hopefully, those are the only "extended" bits
2612                  * they'll add.
2613                  */
2614                 /*FALLTHROUGH*/
2615 
2616         case X86_VENDOR_AMD:
2617                 edx = &cpi->cpi_support[AMD_EDX_FEATURES];
2618                 ecx = &cpi->cpi_support[AMD_ECX_FEATURES];
2619 
2620                 *edx = CPI_FEATURES_XTD_EDX(cpi);
2621                 *ecx = CPI_FEATURES_XTD_ECX(cpi);
2622 
2623                 /*
2624                  * [these features require explicit kernel support]
2625                  */
2626                 switch (cpi->cpi_vendor) {
2627                 case X86_VENDOR_Intel:
2628                         if (!is_x86_feature(x86_featureset, X86FSET_TSCP))
2629                                 *edx &= ~CPUID_AMD_EDX_TSCP;
2630                         break;
2631 
2632                 case X86_VENDOR_AMD:
2633                         if (!is_x86_feature(x86_featureset, X86FSET_TSCP))
2634                                 *edx &= ~CPUID_AMD_EDX_TSCP;
2635                         if (!is_x86_feature(x86_featureset, X86FSET_SSE4A))
2636                                 *ecx &= ~CPUID_AMD_ECX_SSE4A;
2637                         break;
2638 
2639                 default:
2640                         break;
2641                 }
2642 
2643                 /*
2644                  * [no explicit support required beyond
2645                  * x87 fp context and exception handlers]
2646                  */
2647                 if (!fpu_exists)
2648                         *edx &= ~(CPUID_AMD_EDX_MMXamd |
2649                             CPUID_AMD_EDX_3DNow | CPUID_AMD_EDX_3DNowx);
2650 
2651                 if (!is_x86_feature(x86_featureset, X86FSET_NX))
2652                         *edx &= ~CPUID_AMD_EDX_NX;
2653 #if !defined(__amd64)
2654                 *edx &= ~CPUID_AMD_EDX_LM;
2655 #endif
2656                 /*
2657                  * Now map the supported feature vector to
2658                  * things that we think userland will care about.
2659                  */
2660 #if defined(__amd64)
2661                 if (*edx & CPUID_AMD_EDX_SYSC)
2662                         hwcap_flags |= AV_386_AMD_SYSC;
2663 #endif
2664                 if (*edx & CPUID_AMD_EDX_MMXamd)
2665                         hwcap_flags |= AV_386_AMD_MMX;
2666                 if (*edx & CPUID_AMD_EDX_3DNow)
2667                         hwcap_flags |= AV_386_AMD_3DNow;
2668                 if (*edx & CPUID_AMD_EDX_3DNowx)
2669                         hwcap_flags |= AV_386_AMD_3DNowx;
2670                 if (*ecx & CPUID_AMD_ECX_SVM)
2671                         hwcap_flags |= AV_386_AMD_SVM;
2672 
2673                 switch (cpi->cpi_vendor) {
2674                 case X86_VENDOR_AMD:
2675                         if (*edx & CPUID_AMD_EDX_TSCP)
2676                                 hwcap_flags |= AV_386_TSCP;
2677                         if (*ecx & CPUID_AMD_ECX_AHF64)
2678                                 hwcap_flags |= AV_386_AHF;
2679                         if (*ecx & CPUID_AMD_ECX_SSE4A)
2680                                 hwcap_flags |= AV_386_AMD_SSE4A;
2681                         if (*ecx & CPUID_AMD_ECX_LZCNT)
2682                                 hwcap_flags |= AV_386_AMD_LZCNT;
2683                         break;
2684 
2685                 case X86_VENDOR_Intel:
2686                         if (*edx & CPUID_AMD_EDX_TSCP)
2687                                 hwcap_flags |= AV_386_TSCP;
2688                         /*
2689                          * Aarrgh.
2690                          * Intel uses a different bit in the same word.
2691                          */
2692                         if (*ecx & CPUID_INTC_ECX_AHF64)
2693                                 hwcap_flags |= AV_386_AHF;
2694                         break;
2695 
2696                 default:
2697                         break;
2698                 }
2699                 break;
2700 
2701         case X86_VENDOR_TM:
2702                 cp.cp_eax = 0x80860001;
2703                 (void) __cpuid_insn(&cp);
2704                 cpi->cpi_support[TM_EDX_FEATURES] = cp.cp_edx;
2705                 break;
2706 
2707         default:
2708                 break;
2709         }
2710 
2711 pass4_done:
2712         cpi->cpi_pass = 4;
2713         return (hwcap_flags);
2714 }
2715 
2716 
2717 /*
2718  * Simulate the cpuid instruction using the data we previously
2719  * captured about this CPU.  We try our best to return the truth
2720  * about the hardware, independently of kernel support.
2721  */
2722 uint32_t
2723 cpuid_insn(cpu_t *cpu, struct cpuid_regs *cp)
2724 {
2725         struct cpuid_info *cpi;
2726         struct cpuid_regs *xcp;
2727 
2728         if (cpu == NULL)
2729                 cpu = CPU;
2730         cpi = cpu->cpu_m.mcpu_cpi;
2731 
2732         ASSERT(cpuid_checkpass(cpu, 3));
2733 
2734         /*
2735          * CPUID data is cached in two separate places: cpi_std for standard
2736          * CPUID functions, and cpi_extd for extended CPUID functions.
2737          */
2738         if (cp->cp_eax <= cpi->cpi_maxeax && cp->cp_eax < NMAX_CPI_STD)
2739                 xcp = &cpi->cpi_std[cp->cp_eax];
2740         else if (cp->cp_eax >= 0x80000000 && cp->cp_eax <= cpi->cpi_xmaxeax &&
2741             cp->cp_eax < 0x80000000 + NMAX_CPI_EXTD)
2742                 xcp = &cpi->cpi_extd[cp->cp_eax - 0x80000000];
2743         else
2744                 /*
2745                  * The caller is asking for data from an input parameter which
2746                  * the kernel has not cached.  In this case we go fetch from
2747                  * the hardware and return the data directly to the user.
2748                  */
2749                 return (__cpuid_insn(cp));
2750 
2751         cp->cp_eax = xcp->cp_eax;
2752         cp->cp_ebx = xcp->cp_ebx;
2753         cp->cp_ecx = xcp->cp_ecx;
2754         cp->cp_edx = xcp->cp_edx;
2755         return (cp->cp_eax);
2756 }
2757 
2758 int
2759 cpuid_checkpass(cpu_t *cpu, int pass)
2760 {
2761         return (cpu != NULL && cpu->cpu_m.mcpu_cpi != NULL &&
2762             cpu->cpu_m.mcpu_cpi->cpi_pass >= pass);
2763 }
2764 
2765 int
2766 cpuid_getbrandstr(cpu_t *cpu, char *s, size_t n)
2767 {
2768         ASSERT(cpuid_checkpass(cpu, 3));
2769 
2770         return (snprintf(s, n, "%s", cpu->cpu_m.mcpu_cpi->cpi_brandstr));
2771 }
2772 
2773 int
2774 cpuid_is_cmt(cpu_t *cpu)
2775 {
2776         if (cpu == NULL)
2777                 cpu = CPU;
2778 
2779         ASSERT(cpuid_checkpass(cpu, 1));
2780 
2781         return (cpu->cpu_m.mcpu_cpi->cpi_chipid >= 0);
2782 }
2783 
2784 /*
2785  * AMD and Intel both implement the 64-bit variant of the syscall
2786  * instruction (syscallq), so if there's -any- support for syscall,
2787  * cpuid currently says "yes, we support this".
2788  *
2789  * However, Intel decided to -not- implement the 32-bit variant of the
2790  * syscall instruction, so we provide a predicate to allow our caller
2791  * to test that subtlety here.
2792  *
2793  * XXPV Currently, 32-bit syscall instructions don't work via the hypervisor,
2794  *      even in the case where the hardware would in fact support it.
2795  */
2796 /*ARGSUSED*/
2797 int
2798 cpuid_syscall32_insn(cpu_t *cpu)
2799 {
2800         ASSERT(cpuid_checkpass((cpu == NULL ? CPU : cpu), 1));
2801 
2802 #if !defined(__xpv)
2803         if (cpu == NULL)
2804                 cpu = CPU;
2805 
2806         /*CSTYLED*/
2807         {
2808                 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2809 
2810                 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
2811                     cpi->cpi_xmaxeax >= 0x80000001 &&
2812                     (CPI_FEATURES_XTD_EDX(cpi) & CPUID_AMD_EDX_SYSC))
2813                         return (1);
2814         }
2815 #endif
2816         return (0);
2817 }
2818 
2819 int
2820 cpuid_getidstr(cpu_t *cpu, char *s, size_t n)
2821 {
2822         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2823 
2824         static const char fmt[] =
2825             "x86 (%s %X family %d model %d step %d clock %d MHz)";
2826         static const char fmt_ht[] =
2827             "x86 (chipid 0x%x %s %X family %d model %d step %d clock %d MHz)";
2828 
2829         ASSERT(cpuid_checkpass(cpu, 1));
2830 
2831         if (cpuid_is_cmt(cpu))
2832                 return (snprintf(s, n, fmt_ht, cpi->cpi_chipid,
2833                     cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
2834                     cpi->cpi_family, cpi->cpi_model,
2835                     cpi->cpi_step, cpu->cpu_type_info.pi_clock));
2836         return (snprintf(s, n, fmt,
2837             cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
2838             cpi->cpi_family, cpi->cpi_model,
2839             cpi->cpi_step, cpu->cpu_type_info.pi_clock));
2840 }
2841 
2842 const char *
2843 cpuid_getvendorstr(cpu_t *cpu)
2844 {
2845         ASSERT(cpuid_checkpass(cpu, 1));
2846         return ((const char *)cpu->cpu_m.mcpu_cpi->cpi_vendorstr);
2847 }
2848 
2849 uint_t
2850 cpuid_getvendor(cpu_t *cpu)
2851 {
2852         ASSERT(cpuid_checkpass(cpu, 1));
2853         return (cpu->cpu_m.mcpu_cpi->cpi_vendor);
2854 }
2855 
2856 uint_t
2857 cpuid_getfamily(cpu_t *cpu)
2858 {
2859         ASSERT(cpuid_checkpass(cpu, 1));
2860         return (cpu->cpu_m.mcpu_cpi->cpi_family);
2861 }
2862 
2863 uint_t
2864 cpuid_getmodel(cpu_t *cpu)
2865 {
2866         ASSERT(cpuid_checkpass(cpu, 1));
2867         return (cpu->cpu_m.mcpu_cpi->cpi_model);
2868 }
2869 
2870 uint_t
2871 cpuid_get_ncpu_per_chip(cpu_t *cpu)
2872 {
2873         ASSERT(cpuid_checkpass(cpu, 1));
2874         return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_per_chip);
2875 }
2876 
2877 uint_t
2878 cpuid_get_ncore_per_chip(cpu_t *cpu)
2879 {
2880         ASSERT(cpuid_checkpass(cpu, 1));
2881         return (cpu->cpu_m.mcpu_cpi->cpi_ncore_per_chip);
2882 }
2883 
2884 uint_t
2885 cpuid_get_ncpu_sharing_last_cache(cpu_t *cpu)
2886 {
2887         ASSERT(cpuid_checkpass(cpu, 2));
2888         return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_shr_last_cache);
2889 }
2890 
2891 id_t
2892 cpuid_get_last_lvl_cacheid(cpu_t *cpu)
2893 {
2894         ASSERT(cpuid_checkpass(cpu, 2));
2895         return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
2896 }
2897 
2898 uint_t
2899 cpuid_getstep(cpu_t *cpu)
2900 {
2901         ASSERT(cpuid_checkpass(cpu, 1));
2902         return (cpu->cpu_m.mcpu_cpi->cpi_step);
2903 }
2904 
2905 uint_t
2906 cpuid_getsig(struct cpu *cpu)
2907 {
2908         ASSERT(cpuid_checkpass(cpu, 1));
2909         return (cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_eax);
2910 }
2911 
2912 uint32_t
2913 cpuid_getchiprev(struct cpu *cpu)
2914 {
2915         ASSERT(cpuid_checkpass(cpu, 1));
2916         return (cpu->cpu_m.mcpu_cpi->cpi_chiprev);
2917 }
2918 
2919 const char *
2920 cpuid_getchiprevstr(struct cpu *cpu)
2921 {
2922         ASSERT(cpuid_checkpass(cpu, 1));
2923         return (cpu->cpu_m.mcpu_cpi->cpi_chiprevstr);
2924 }
2925 
2926 uint32_t
2927 cpuid_getsockettype(struct cpu *cpu)
2928 {
2929         ASSERT(cpuid_checkpass(cpu, 1));
2930         return (cpu->cpu_m.mcpu_cpi->cpi_socket);
2931 }
2932 
2933 const char *
2934 cpuid_getsocketstr(cpu_t *cpu)
2935 {
2936         static const char *socketstr = NULL;
2937         struct cpuid_info *cpi;
2938 
2939         ASSERT(cpuid_checkpass(cpu, 1));
2940         cpi = cpu->cpu_m.mcpu_cpi;
2941 
2942         /* Assume that socket types are the same across the system */
2943         if (socketstr == NULL)
2944                 socketstr = _cpuid_sktstr(cpi->cpi_vendor, cpi->cpi_family,
2945                     cpi->cpi_model, cpi->cpi_step);
2946 
2947 
2948         return (socketstr);
2949 }
2950 
2951 int
2952 cpuid_get_chipid(cpu_t *cpu)
2953 {
2954         ASSERT(cpuid_checkpass(cpu, 1));
2955 
2956         if (cpuid_is_cmt(cpu))
2957                 return (cpu->cpu_m.mcpu_cpi->cpi_chipid);
2958         return (cpu->cpu_id);
2959 }
2960 
2961 id_t
2962 cpuid_get_coreid(cpu_t *cpu)
2963 {
2964         ASSERT(cpuid_checkpass(cpu, 1));
2965         return (cpu->cpu_m.mcpu_cpi->cpi_coreid);
2966 }
2967 
2968 int
2969 cpuid_get_pkgcoreid(cpu_t *cpu)
2970 {
2971         ASSERT(cpuid_checkpass(cpu, 1));
2972         return (cpu->cpu_m.mcpu_cpi->cpi_pkgcoreid);
2973 }
2974 
2975 int
2976 cpuid_get_clogid(cpu_t *cpu)
2977 {
2978         ASSERT(cpuid_checkpass(cpu, 1));
2979         return (cpu->cpu_m.mcpu_cpi->cpi_clogid);
2980 }
2981 
2982 int
2983 cpuid_get_cacheid(cpu_t *cpu)
2984 {
2985         ASSERT(cpuid_checkpass(cpu, 1));
2986         return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
2987 }
2988 
2989 uint_t
2990 cpuid_get_procnodeid(cpu_t *cpu)
2991 {
2992         ASSERT(cpuid_checkpass(cpu, 1));
2993         return (cpu->cpu_m.mcpu_cpi->cpi_procnodeid);
2994 }
2995 
2996 uint_t
2997 cpuid_get_procnodes_per_pkg(cpu_t *cpu)
2998 {
2999         ASSERT(cpuid_checkpass(cpu, 1));
3000         return (cpu->cpu_m.mcpu_cpi->cpi_procnodes_per_pkg);
3001 }
3002 
3003 /*ARGSUSED*/
3004 int
3005 cpuid_have_cr8access(cpu_t *cpu)
3006 {
3007 #if defined(__amd64)
3008         return (1);
3009 #else
3010         struct cpuid_info *cpi;
3011 
3012         ASSERT(cpu != NULL);
3013         cpi = cpu->cpu_m.mcpu_cpi;
3014         if (cpi->cpi_vendor == X86_VENDOR_AMD && cpi->cpi_maxeax >= 1 &&
3015             (CPI_FEATURES_XTD_ECX(cpi) & CPUID_AMD_ECX_CR8D) != 0)
3016                 return (1);
3017         return (0);
3018 #endif
3019 }
3020 
3021 uint32_t
3022 cpuid_get_apicid(cpu_t *cpu)
3023 {
3024         ASSERT(cpuid_checkpass(cpu, 1));
3025         if (cpu->cpu_m.mcpu_cpi->cpi_maxeax < 1) {
3026                 return (UINT32_MAX);
3027         } else {
3028                 return (cpu->cpu_m.mcpu_cpi->cpi_apicid);
3029         }
3030 }
3031 
3032 void
3033 cpuid_get_addrsize(cpu_t *cpu, uint_t *pabits, uint_t *vabits)
3034 {
3035         struct cpuid_info *cpi;
3036 
3037         if (cpu == NULL)
3038                 cpu = CPU;
3039         cpi = cpu->cpu_m.mcpu_cpi;
3040 
3041         ASSERT(cpuid_checkpass(cpu, 1));
3042 
3043         if (pabits)
3044                 *pabits = cpi->cpi_pabits;
3045         if (vabits)
3046                 *vabits = cpi->cpi_vabits;
3047 }
3048 
3049 /*
3050  * Returns the number of data TLB entries for a corresponding
3051  * pagesize.  If it can't be computed, or isn't known, the
3052  * routine returns zero.  If you ask about an architecturally
3053  * impossible pagesize, the routine will panic (so that the
3054  * hat implementor knows that things are inconsistent.)
3055  */
3056 uint_t
3057 cpuid_get_dtlb_nent(cpu_t *cpu, size_t pagesize)
3058 {
3059         struct cpuid_info *cpi;
3060         uint_t dtlb_nent = 0;
3061 
3062         if (cpu == NULL)
3063                 cpu = CPU;
3064         cpi = cpu->cpu_m.mcpu_cpi;
3065 
3066         ASSERT(cpuid_checkpass(cpu, 1));
3067 
3068         /*
3069          * Check the L2 TLB info
3070          */
3071         if (cpi->cpi_xmaxeax >= 0x80000006) {
3072                 struct cpuid_regs *cp = &cpi->cpi_extd[6];
3073 
3074                 switch (pagesize) {
3075 
3076                 case 4 * 1024:
3077                         /*
3078                          * All zero in the top 16 bits of the register
3079                          * indicates a unified TLB. Size is in low 16 bits.
3080                          */
3081                         if ((cp->cp_ebx & 0xffff0000) == 0)
3082                                 dtlb_nent = cp->cp_ebx & 0x0000ffff;
3083                         else
3084                                 dtlb_nent = BITX(cp->cp_ebx, 27, 16);
3085                         break;
3086 
3087                 case 2 * 1024 * 1024:
3088                         if ((cp->cp_eax & 0xffff0000) == 0)
3089                                 dtlb_nent = cp->cp_eax & 0x0000ffff;
3090                         else
3091                                 dtlb_nent = BITX(cp->cp_eax, 27, 16);
3092                         break;
3093 
3094                 default:
3095                         panic("unknown L2 pagesize");
3096                         /*NOTREACHED*/
3097                 }
3098         }
3099 
3100         if (dtlb_nent != 0)
3101                 return (dtlb_nent);
3102 
3103         /*
3104          * No L2 TLB support for this size, try L1.
3105          */
3106         if (cpi->cpi_xmaxeax >= 0x80000005) {
3107                 struct cpuid_regs *cp = &cpi->cpi_extd[5];
3108 
3109                 switch (pagesize) {
3110                 case 4 * 1024:
3111                         dtlb_nent = BITX(cp->cp_ebx, 23, 16);
3112                         break;
3113                 case 2 * 1024 * 1024:
3114                         dtlb_nent = BITX(cp->cp_eax, 23, 16);
3115                         break;
3116                 default:
3117                         panic("unknown L1 d-TLB pagesize");
3118                         /*NOTREACHED*/
3119                 }
3120         }
3121 
3122         return (dtlb_nent);
3123 }
3124 
3125 /*
3126  * Return 0 if the erratum is not present or not applicable, positive
3127  * if it is, and negative if the status of the erratum is unknown.
3128  *
3129  * See "Revision Guide for AMD Athlon(tm) 64 and AMD Opteron(tm)
3130  * Processors" #25759, Rev 3.57, August 2005
3131  */
3132 int
3133 cpuid_opteron_erratum(cpu_t *cpu, uint_t erratum)
3134 {
3135         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3136         uint_t eax;
3137 
3138         /*
3139          * Bail out if this CPU isn't an AMD CPU, or if it's
3140          * a legacy (32-bit) AMD CPU.
3141          */
3142         if (cpi->cpi_vendor != X86_VENDOR_AMD ||
3143             cpi->cpi_family == 4 || cpi->cpi_family == 5 ||
3144             cpi->cpi_family == 6)
3145 
3146                 return (0);
3147 
3148         eax = cpi->cpi_std[1].cp_eax;
3149 
3150 #define SH_B0(eax)      (eax == 0xf40 || eax == 0xf50)
3151 #define SH_B3(eax)      (eax == 0xf51)
3152 #define B(eax)          (SH_B0(eax) || SH_B3(eax))
3153 
3154 #define SH_C0(eax)      (eax == 0xf48 || eax == 0xf58)
3155 
3156 #define SH_CG(eax)      (eax == 0xf4a || eax == 0xf5a || eax == 0xf7a)
3157 #define DH_CG(eax)      (eax == 0xfc0 || eax == 0xfe0 || eax == 0xff0)
3158 #define CH_CG(eax)      (eax == 0xf82 || eax == 0xfb2)
3159 #define CG(eax)         (SH_CG(eax) || DH_CG(eax) || CH_CG(eax))
3160 
3161 #define SH_D0(eax)      (eax == 0x10f40 || eax == 0x10f50 || eax == 0x10f70)
3162 #define DH_D0(eax)      (eax == 0x10fc0 || eax == 0x10ff0)
3163 #define CH_D0(eax)      (eax == 0x10f80 || eax == 0x10fb0)
3164 #define D0(eax)         (SH_D0(eax) || DH_D0(eax) || CH_D0(eax))
3165 
3166 #define SH_E0(eax)      (eax == 0x20f50 || eax == 0x20f40 || eax == 0x20f70)
3167 #define JH_E1(eax)      (eax == 0x20f10)        /* JH8_E0 had 0x20f30 */
3168 #define DH_E3(eax)      (eax == 0x20fc0 || eax == 0x20ff0)
3169 #define SH_E4(eax)      (eax == 0x20f51 || eax == 0x20f71)
3170 #define BH_E4(eax)      (eax == 0x20fb1)
3171 #define SH_E5(eax)      (eax == 0x20f42)
3172 #define DH_E6(eax)      (eax == 0x20ff2 || eax == 0x20fc2)
3173 #define JH_E6(eax)      (eax == 0x20f12 || eax == 0x20f32)
3174 #define EX(eax)         (SH_E0(eax) || JH_E1(eax) || DH_E3(eax) || \
3175                             SH_E4(eax) || BH_E4(eax) || SH_E5(eax) || \
3176                             DH_E6(eax) || JH_E6(eax))
3177 
3178 #define DR_AX(eax)      (eax == 0x100f00 || eax == 0x100f01 || eax == 0x100f02)
3179 #define DR_B0(eax)      (eax == 0x100f20)
3180 #define DR_B1(eax)      (eax == 0x100f21)
3181 #define DR_BA(eax)      (eax == 0x100f2a)
3182 #define DR_B2(eax)      (eax == 0x100f22)
3183 #define DR_B3(eax)      (eax == 0x100f23)
3184 #define RB_C0(eax)      (eax == 0x100f40)
3185 
3186         switch (erratum) {
3187         case 1:
3188                 return (cpi->cpi_family < 0x10);
3189         case 51:        /* what does the asterisk mean? */
3190                 return (B(eax) || SH_C0(eax) || CG(eax));
3191         case 52:
3192                 return (B(eax));
3193         case 57:
3194                 return (cpi->cpi_family <= 0x11);
3195         case 58:
3196                 return (B(eax));
3197         case 60:
3198                 return (cpi->cpi_family <= 0x11);
3199         case 61:
3200         case 62:
3201         case 63:
3202         case 64:
3203         case 65:
3204         case 66:
3205         case 68:
3206         case 69:
3207         case 70:
3208         case 71:
3209                 return (B(eax));
3210         case 72:
3211                 return (SH_B0(eax));
3212         case 74:
3213                 return (B(eax));
3214         case 75:
3215                 return (cpi->cpi_family < 0x10);
3216         case 76:
3217                 return (B(eax));
3218         case 77:
3219                 return (cpi->cpi_family <= 0x11);
3220         case 78:
3221                 return (B(eax) || SH_C0(eax));
3222         case 79:
3223                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3224         case 80:
3225         case 81:
3226         case 82:
3227                 return (B(eax));
3228         case 83:
3229                 return (B(eax) || SH_C0(eax) || CG(eax));
3230         case 85:
3231                 return (cpi->cpi_family < 0x10);
3232         case 86:
3233                 return (SH_C0(eax) || CG(eax));
3234         case 88:
3235 #if !defined(__amd64)
3236                 return (0);
3237 #else
3238                 return (B(eax) || SH_C0(eax));
3239 #endif
3240         case 89:
3241                 return (cpi->cpi_family < 0x10);
3242         case 90:
3243                 return (B(eax) || SH_C0(eax) || CG(eax));
3244         case 91:
3245         case 92:
3246                 return (B(eax) || SH_C0(eax));
3247         case 93:
3248                 return (SH_C0(eax));
3249         case 94:
3250                 return (B(eax) || SH_C0(eax) || CG(eax));
3251         case 95:
3252 #if !defined(__amd64)
3253                 return (0);
3254 #else
3255                 return (B(eax) || SH_C0(eax));
3256 #endif
3257         case 96:
3258                 return (B(eax) || SH_C0(eax) || CG(eax));
3259         case 97:
3260         case 98:
3261                 return (SH_C0(eax) || CG(eax));
3262         case 99:
3263                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3264         case 100:
3265                 return (B(eax) || SH_C0(eax));
3266         case 101:
3267         case 103:
3268                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3269         case 104:
3270                 return (SH_C0(eax) || CG(eax) || D0(eax));
3271         case 105:
3272         case 106:
3273         case 107:
3274                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3275         case 108:
3276                 return (DH_CG(eax));
3277         case 109:
3278                 return (SH_C0(eax) || CG(eax) || D0(eax));
3279         case 110:
3280                 return (D0(eax) || EX(eax));
3281         case 111:
3282                 return (CG(eax));
3283         case 112:
3284                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3285         case 113:
3286                 return (eax == 0x20fc0);
3287         case 114:
3288                 return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
3289         case 115:
3290                 return (SH_E0(eax) || JH_E1(eax));
3291         case 116:
3292                 return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
3293         case 117:
3294                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3295         case 118:
3296                 return (SH_E0(eax) || JH_E1(eax) || SH_E4(eax) || BH_E4(eax) ||
3297                     JH_E6(eax));
3298         case 121:
3299                 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3300         case 122:
3301                 return (cpi->cpi_family < 0x10 || cpi->cpi_family == 0x11);
3302         case 123:
3303                 return (JH_E1(eax) || BH_E4(eax) || JH_E6(eax));
3304         case 131:
3305                 return (cpi->cpi_family < 0x10);
3306         case 6336786:
3307                 /*
3308                  * Test for AdvPowerMgmtInfo.TscPStateInvariant
3309                  * if this is a K8 family or newer processor
3310                  */
3311                 if (CPI_FAMILY(cpi) == 0xf) {
3312                         struct cpuid_regs regs;
3313                         regs.cp_eax = 0x80000007;
3314                         (void) __cpuid_insn(&regs);
3315                         return (!(regs.cp_edx & 0x100));
3316                 }
3317                 return (0);
3318         case 6323525:
3319                 return (((((eax >> 12) & 0xff00) + (eax & 0xf00)) |
3320                     (((eax >> 4) & 0xf) | ((eax >> 12) & 0xf0))) < 0xf40);
3321 
3322         case 6671130:
3323                 /*
3324                  * check for processors (pre-Shanghai) that do not provide
3325                  * optimal management of 1gb ptes in its tlb.
3326                  */
3327                 return (cpi->cpi_family == 0x10 && cpi->cpi_model < 4);
3328 
3329         case 298:
3330                 return (DR_AX(eax) || DR_B0(eax) || DR_B1(eax) || DR_BA(eax) ||
3331                     DR_B2(eax) || RB_C0(eax));
3332 
3333         case 721:
3334 #if defined(__amd64)
3335                 return (cpi->cpi_family == 0x10 || cpi->cpi_family == 0x12);
3336 #else
3337                 return (0);
3338 #endif
3339 
3340         default:
3341                 return (-1);
3342 
3343         }
3344 }
3345 
3346 /*
3347  * Determine if specified erratum is present via OSVW (OS Visible Workaround).
3348  * Return 1 if erratum is present, 0 if not present and -1 if indeterminate.
3349  */
3350 int
3351 osvw_opteron_erratum(cpu_t *cpu, uint_t erratum)
3352 {
3353         struct cpuid_info       *cpi;
3354         uint_t                  osvwid;
3355         static int              osvwfeature = -1;
3356         uint64_t                osvwlength;
3357 
3358 
3359         cpi = cpu->cpu_m.mcpu_cpi;
3360 
3361         /* confirm OSVW supported */
3362         if (osvwfeature == -1) {
3363                 osvwfeature = cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW;
3364         } else {
3365                 /* assert that osvw feature setting is consistent on all cpus */
3366                 ASSERT(osvwfeature ==
3367                     (cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW));
3368         }
3369         if (!osvwfeature)
3370                 return (-1);
3371 
3372         osvwlength = rdmsr(MSR_AMD_OSVW_ID_LEN) & OSVW_ID_LEN_MASK;
3373 
3374         switch (erratum) {
3375         case 298:       /* osvwid is 0 */
3376                 osvwid = 0;
3377                 if (osvwlength <= (uint64_t)osvwid) {
3378                         /* osvwid 0 is unknown */
3379                         return (-1);
3380                 }
3381 
3382                 /*
3383                  * Check the OSVW STATUS MSR to determine the state
3384                  * of the erratum where:
3385                  *   0 - fixed by HW
3386                  *   1 - BIOS has applied the workaround when BIOS
3387                  *   workaround is available. (Or for other errata,
3388                  *   OS workaround is required.)
3389                  * For a value of 1, caller will confirm that the
3390                  * erratum 298 workaround has indeed been applied by BIOS.
3391                  *
3392                  * A 1 may be set in cpus that have a HW fix
3393                  * in a mixed cpu system. Regarding erratum 298:
3394                  *   In a multiprocessor platform, the workaround above
3395                  *   should be applied to all processors regardless of
3396                  *   silicon revision when an affected processor is
3397                  *   present.
3398                  */
3399 
3400                 return (rdmsr(MSR_AMD_OSVW_STATUS +
3401                     (osvwid / OSVW_ID_CNT_PER_MSR)) &
3402                     (1ULL << (osvwid % OSVW_ID_CNT_PER_MSR)));
3403 
3404         default:
3405                 return (-1);
3406         }
3407 }
3408 
3409 static const char assoc_str[] = "associativity";
3410 static const char line_str[] = "line-size";
3411 static const char size_str[] = "size";
3412 
3413 static void
3414 add_cache_prop(dev_info_t *devi, const char *label, const char *type,
3415     uint32_t val)
3416 {
3417         char buf[128];
3418 
3419         /*
3420          * ndi_prop_update_int() is used because it is desirable for
3421          * DDI_PROP_HW_DEF and DDI_PROP_DONTSLEEP to be set.
3422          */
3423         if (snprintf(buf, sizeof (buf), "%s-%s", label, type) < sizeof (buf))
3424                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, devi, buf, val);
3425 }
3426 
3427 /*
3428  * Intel-style cache/tlb description
3429  *
3430  * Standard cpuid level 2 gives a randomly ordered
3431  * selection of tags that index into a table that describes
3432  * cache and tlb properties.
3433  */
3434 
3435 static const char l1_icache_str[] = "l1-icache";
3436 static const char l1_dcache_str[] = "l1-dcache";
3437 static const char l2_cache_str[] = "l2-cache";
3438 static const char l3_cache_str[] = "l3-cache";
3439 static const char itlb4k_str[] = "itlb-4K";
3440 static const char dtlb4k_str[] = "dtlb-4K";
3441 static const char itlb2M_str[] = "itlb-2M";
3442 static const char itlb4M_str[] = "itlb-4M";
3443 static const char dtlb4M_str[] = "dtlb-4M";
3444 static const char dtlb24_str[] = "dtlb0-2M-4M";
3445 static const char itlb424_str[] = "itlb-4K-2M-4M";
3446 static const char itlb24_str[] = "itlb-2M-4M";
3447 static const char dtlb44_str[] = "dtlb-4K-4M";
3448 static const char sl1_dcache_str[] = "sectored-l1-dcache";
3449 static const char sl2_cache_str[] = "sectored-l2-cache";
3450 static const char itrace_str[] = "itrace-cache";
3451 static const char sl3_cache_str[] = "sectored-l3-cache";
3452 static const char sh_l2_tlb4k_str[] = "shared-l2-tlb-4k";
3453 
3454 static const struct cachetab {
3455         uint8_t         ct_code;
3456         uint8_t         ct_assoc;
3457         uint16_t        ct_line_size;
3458         size_t          ct_size;
3459         const char      *ct_label;
3460 } intel_ctab[] = {
3461         /*
3462          * maintain descending order!
3463          *
3464          * Codes ignored - Reason
3465          * ----------------------
3466          * 40H - intel_cpuid_4_cache_info() disambiguates l2/l3 cache
3467          * f0H/f1H - Currently we do not interpret prefetch size by design
3468          */
3469         { 0xe4, 16, 64, 8*1024*1024, l3_cache_str},
3470         { 0xe3, 16, 64, 4*1024*1024, l3_cache_str},
3471         { 0xe2, 16, 64, 2*1024*1024, l3_cache_str},
3472         { 0xde, 12, 64, 6*1024*1024, l3_cache_str},
3473         { 0xdd, 12, 64, 3*1024*1024, l3_cache_str},
3474         { 0xdc, 12, 64, ((1*1024*1024)+(512*1024)), l3_cache_str},
3475         { 0xd8, 8, 64, 4*1024*1024, l3_cache_str},
3476         { 0xd7, 8, 64, 2*1024*1024, l3_cache_str},
3477         { 0xd6, 8, 64, 1*1024*1024, l3_cache_str},
3478         { 0xd2, 4, 64, 2*1024*1024, l3_cache_str},
3479         { 0xd1, 4, 64, 1*1024*1024, l3_cache_str},
3480         { 0xd0, 4, 64, 512*1024, l3_cache_str},
3481         { 0xca, 4, 0, 512, sh_l2_tlb4k_str},
3482         { 0xc0, 4, 0, 8, dtlb44_str },
3483         { 0xba, 4, 0, 64, dtlb4k_str },
3484         { 0xb4, 4, 0, 256, dtlb4k_str },
3485         { 0xb3, 4, 0, 128, dtlb4k_str },
3486         { 0xb2, 4, 0, 64, itlb4k_str },
3487         { 0xb0, 4, 0, 128, itlb4k_str },
3488         { 0x87, 8, 64, 1024*1024, l2_cache_str},
3489         { 0x86, 4, 64, 512*1024, l2_cache_str},
3490         { 0x85, 8, 32, 2*1024*1024, l2_cache_str},
3491         { 0x84, 8, 32, 1024*1024, l2_cache_str},
3492         { 0x83, 8, 32, 512*1024, l2_cache_str},
3493         { 0x82, 8, 32, 256*1024, l2_cache_str},
3494         { 0x80, 8, 64, 512*1024, l2_cache_str},
3495         { 0x7f, 2, 64, 512*1024, l2_cache_str},
3496         { 0x7d, 8, 64, 2*1024*1024, sl2_cache_str},
3497         { 0x7c, 8, 64, 1024*1024, sl2_cache_str},
3498         { 0x7b, 8, 64, 512*1024, sl2_cache_str},
3499         { 0x7a, 8, 64, 256*1024, sl2_cache_str},
3500         { 0x79, 8, 64, 128*1024, sl2_cache_str},
3501         { 0x78, 8, 64, 1024*1024, l2_cache_str},
3502         { 0x73, 8, 0, 64*1024, itrace_str},
3503         { 0x72, 8, 0, 32*1024, itrace_str},
3504         { 0x71, 8, 0, 16*1024, itrace_str},
3505         { 0x70, 8, 0, 12*1024, itrace_str},
3506         { 0x68, 4, 64, 32*1024, sl1_dcache_str},
3507         { 0x67, 4, 64, 16*1024, sl1_dcache_str},
3508         { 0x66, 4, 64, 8*1024, sl1_dcache_str},
3509         { 0x60, 8, 64, 16*1024, sl1_dcache_str},
3510         { 0x5d, 0, 0, 256, dtlb44_str},
3511         { 0x5c, 0, 0, 128, dtlb44_str},
3512         { 0x5b, 0, 0, 64, dtlb44_str},
3513         { 0x5a, 4, 0, 32, dtlb24_str},
3514         { 0x59, 0, 0, 16, dtlb4k_str},
3515         { 0x57, 4, 0, 16, dtlb4k_str},
3516         { 0x56, 4, 0, 16, dtlb4M_str},
3517         { 0x55, 0, 0, 7, itlb24_str},
3518         { 0x52, 0, 0, 256, itlb424_str},
3519         { 0x51, 0, 0, 128, itlb424_str},
3520         { 0x50, 0, 0, 64, itlb424_str},
3521         { 0x4f, 0, 0, 32, itlb4k_str},
3522         { 0x4e, 24, 64, 6*1024*1024, l2_cache_str},
3523         { 0x4d, 16, 64, 16*1024*1024, l3_cache_str},
3524         { 0x4c, 12, 64, 12*1024*1024, l3_cache_str},
3525         { 0x4b, 16, 64, 8*1024*1024, l3_cache_str},
3526         { 0x4a, 12, 64, 6*1024*1024, l3_cache_str},
3527         { 0x49, 16, 64, 4*1024*1024, l3_cache_str},
3528         { 0x48, 12, 64, 3*1024*1024, l2_cache_str},
3529         { 0x47, 8, 64, 8*1024*1024, l3_cache_str},
3530         { 0x46, 4, 64, 4*1024*1024, l3_cache_str},
3531         { 0x45, 4, 32, 2*1024*1024, l2_cache_str},
3532         { 0x44, 4, 32, 1024*1024, l2_cache_str},
3533         { 0x43, 4, 32, 512*1024, l2_cache_str},
3534         { 0x42, 4, 32, 256*1024, l2_cache_str},
3535         { 0x41, 4, 32, 128*1024, l2_cache_str},
3536         { 0x3e, 4, 64, 512*1024, sl2_cache_str},
3537         { 0x3d, 6, 64, 384*1024, sl2_cache_str},
3538         { 0x3c, 4, 64, 256*1024, sl2_cache_str},
3539         { 0x3b, 2, 64, 128*1024, sl2_cache_str},
3540         { 0x3a, 6, 64, 192*1024, sl2_cache_str},
3541         { 0x39, 4, 64, 128*1024, sl2_cache_str},
3542         { 0x30, 8, 64, 32*1024, l1_icache_str},
3543         { 0x2c, 8, 64, 32*1024, l1_dcache_str},
3544         { 0x29, 8, 64, 4096*1024, sl3_cache_str},
3545         { 0x25, 8, 64, 2048*1024, sl3_cache_str},
3546         { 0x23, 8, 64, 1024*1024, sl3_cache_str},
3547         { 0x22, 4, 64, 512*1024, sl3_cache_str},
3548         { 0x0e, 6, 64, 24*1024, l1_dcache_str},
3549         { 0x0d, 4, 32, 16*1024, l1_dcache_str},
3550         { 0x0c, 4, 32, 16*1024, l1_dcache_str},
3551         { 0x0b, 4, 0, 4, itlb4M_str},
3552         { 0x0a, 2, 32, 8*1024, l1_dcache_str},
3553         { 0x08, 4, 32, 16*1024, l1_icache_str},
3554         { 0x06, 4, 32, 8*1024, l1_icache_str},
3555         { 0x05, 4, 0, 32, dtlb4M_str},
3556         { 0x04, 4, 0, 8, dtlb4M_str},
3557         { 0x03, 4, 0, 64, dtlb4k_str},
3558         { 0x02, 4, 0, 2, itlb4M_str},
3559         { 0x01, 4, 0, 32, itlb4k_str},
3560         { 0 }
3561 };
3562 
3563 static const struct cachetab cyrix_ctab[] = {
3564         { 0x70, 4, 0, 32, "tlb-4K" },
3565         { 0x80, 4, 16, 16*1024, "l1-cache" },
3566         { 0 }
3567 };
3568 
3569 /*
3570  * Search a cache table for a matching entry
3571  */
3572 static const struct cachetab *
3573 find_cacheent(const struct cachetab *ct, uint_t code)
3574 {
3575         if (code != 0) {
3576                 for (; ct->ct_code != 0; ct++)
3577                         if (ct->ct_code <= code)
3578                                 break;
3579                 if (ct->ct_code == code)
3580                         return (ct);
3581         }
3582         return (NULL);
3583 }
3584 
3585 /*
3586  * Populate cachetab entry with L2 or L3 cache-information using
3587  * cpuid function 4. This function is called from intel_walk_cacheinfo()
3588  * when descriptor 0x49 is encountered. It returns 0 if no such cache
3589  * information is found.
3590  */
3591 static int
3592 intel_cpuid_4_cache_info(struct cachetab *ct, struct cpuid_info *cpi)
3593 {
3594         uint32_t level, i;
3595         int ret = 0;
3596 
3597         for (i = 0; i < cpi->cpi_std_4_size; i++) {
3598                 level = CPI_CACHE_LVL(cpi->cpi_std_4[i]);
3599 
3600                 if (level == 2 || level == 3) {
3601                         ct->ct_assoc = CPI_CACHE_WAYS(cpi->cpi_std_4[i]) + 1;
3602                         ct->ct_line_size =
3603                             CPI_CACHE_COH_LN_SZ(cpi->cpi_std_4[i]) + 1;
3604                         ct->ct_size = ct->ct_assoc *
3605                             (CPI_CACHE_PARTS(cpi->cpi_std_4[i]) + 1) *
3606                             ct->ct_line_size *
3607                             (cpi->cpi_std_4[i]->cp_ecx + 1);
3608 
3609                         if (level == 2) {
3610                                 ct->ct_label = l2_cache_str;
3611                         } else if (level == 3) {
3612                                 ct->ct_label = l3_cache_str;
3613                         }
3614                         ret = 1;
3615                 }
3616         }
3617 
3618         return (ret);
3619 }
3620 
3621 /*
3622  * Walk the cacheinfo descriptor, applying 'func' to every valid element
3623  * The walk is terminated if the walker returns non-zero.
3624  */
3625 static void
3626 intel_walk_cacheinfo(struct cpuid_info *cpi,
3627     void *arg, int (*func)(void *, const struct cachetab *))
3628 {
3629         const struct cachetab *ct;
3630         struct cachetab des_49_ct, des_b1_ct;
3631         uint8_t *dp;
3632         int i;
3633 
3634         if ((dp = cpi->cpi_cacheinfo) == NULL)
3635                 return;
3636         for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
3637                 /*
3638                  * For overloaded descriptor 0x49 we use cpuid function 4
3639                  * if supported by the current processor, to create
3640                  * cache information.
3641                  * For overloaded descriptor 0xb1 we use X86_PAE flag
3642                  * to disambiguate the cache information.
3643                  */
3644                 if (*dp == 0x49 && cpi->cpi_maxeax >= 0x4 &&
3645                     intel_cpuid_4_cache_info(&des_49_ct, cpi) == 1) {
3646                                 ct = &des_49_ct;
3647                 } else if (*dp == 0xb1) {
3648                         des_b1_ct.ct_code = 0xb1;
3649                         des_b1_ct.ct_assoc = 4;
3650                         des_b1_ct.ct_line_size = 0;
3651                         if (is_x86_feature(x86_featureset, X86FSET_PAE)) {
3652                                 des_b1_ct.ct_size = 8;
3653                                 des_b1_ct.ct_label = itlb2M_str;
3654                         } else {
3655                                 des_b1_ct.ct_size = 4;
3656                                 des_b1_ct.ct_label = itlb4M_str;
3657                         }
3658                         ct = &des_b1_ct;
3659                 } else {
3660                         if ((ct = find_cacheent(intel_ctab, *dp)) == NULL) {
3661                                 continue;
3662                         }
3663                 }
3664 
3665                 if (func(arg, ct) != 0) {
3666                         break;
3667                 }
3668         }
3669 }
3670 
3671 /*
3672  * (Like the Intel one, except for Cyrix CPUs)
3673  */
3674 static void
3675 cyrix_walk_cacheinfo(struct cpuid_info *cpi,
3676     void *arg, int (*func)(void *, const struct cachetab *))
3677 {
3678         const struct cachetab *ct;
3679         uint8_t *dp;
3680         int i;
3681 
3682         if ((dp = cpi->cpi_cacheinfo) == NULL)
3683                 return;
3684         for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
3685                 /*
3686                  * Search Cyrix-specific descriptor table first ..
3687                  */
3688                 if ((ct = find_cacheent(cyrix_ctab, *dp)) != NULL) {
3689                         if (func(arg, ct) != 0)
3690                                 break;
3691                         continue;
3692                 }
3693                 /*
3694                  * .. else fall back to the Intel one
3695                  */
3696                 if ((ct = find_cacheent(intel_ctab, *dp)) != NULL) {
3697                         if (func(arg, ct) != 0)
3698                                 break;
3699                         continue;
3700                 }
3701         }
3702 }
3703 
3704 /*
3705  * A cacheinfo walker that adds associativity, line-size, and size properties
3706  * to the devinfo node it is passed as an argument.
3707  */
3708 static int
3709 add_cacheent_props(void *arg, const struct cachetab *ct)
3710 {
3711         dev_info_t *devi = arg;
3712 
3713         add_cache_prop(devi, ct->ct_label, assoc_str, ct->ct_assoc);
3714         if (ct->ct_line_size != 0)
3715                 add_cache_prop(devi, ct->ct_label, line_str,
3716                     ct->ct_line_size);
3717         add_cache_prop(devi, ct->ct_label, size_str, ct->ct_size);
3718         return (0);
3719 }
3720 
3721 
3722 static const char fully_assoc[] = "fully-associative?";
3723 
3724 /*
3725  * AMD style cache/tlb description
3726  *
3727  * Extended functions 5 and 6 directly describe properties of
3728  * tlbs and various cache levels.
3729  */
3730 static void
3731 add_amd_assoc(dev_info_t *devi, const char *label, uint_t assoc)
3732 {
3733         switch (assoc) {
3734         case 0: /* reserved; ignore */
3735                 break;
3736         default:
3737                 add_cache_prop(devi, label, assoc_str, assoc);
3738                 break;
3739         case 0xff:
3740                 add_cache_prop(devi, label, fully_assoc, 1);
3741                 break;
3742         }
3743 }
3744 
3745 static void
3746 add_amd_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
3747 {
3748         if (size == 0)
3749                 return;
3750         add_cache_prop(devi, label, size_str, size);
3751         add_amd_assoc(devi, label, assoc);
3752 }
3753 
3754 static void
3755 add_amd_cache(dev_info_t *devi, const char *label,
3756     uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
3757 {
3758         if (size == 0 || line_size == 0)
3759                 return;
3760         add_amd_assoc(devi, label, assoc);
3761         /*
3762          * Most AMD parts have a sectored cache. Multiple cache lines are
3763          * associated with each tag. A sector consists of all cache lines
3764          * associated with a tag. For example, the AMD K6-III has a sector
3765          * size of 2 cache lines per tag.
3766          */
3767         if (lines_per_tag != 0)
3768                 add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
3769         add_cache_prop(devi, label, line_str, line_size);
3770         add_cache_prop(devi, label, size_str, size * 1024);
3771 }
3772 
3773 static void
3774 add_amd_l2_assoc(dev_info_t *devi, const char *label, uint_t assoc)
3775 {
3776         switch (assoc) {
3777         case 0: /* off */
3778                 break;
3779         case 1:
3780         case 2:
3781         case 4:
3782                 add_cache_prop(devi, label, assoc_str, assoc);
3783                 break;
3784         case 6:
3785                 add_cache_prop(devi, label, assoc_str, 8);
3786                 break;
3787         case 8:
3788                 add_cache_prop(devi, label, assoc_str, 16);
3789                 break;
3790         case 0xf:
3791                 add_cache_prop(devi, label, fully_assoc, 1);
3792                 break;
3793         default: /* reserved; ignore */
3794                 break;
3795         }
3796 }
3797 
3798 static void
3799 add_amd_l2_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
3800 {
3801         if (size == 0 || assoc == 0)
3802                 return;
3803         add_amd_l2_assoc(devi, label, assoc);
3804         add_cache_prop(devi, label, size_str, size);
3805 }
3806 
3807 static void
3808 add_amd_l2_cache(dev_info_t *devi, const char *label,
3809     uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
3810 {
3811         if (size == 0 || assoc == 0 || line_size == 0)
3812                 return;
3813         add_amd_l2_assoc(devi, label, assoc);
3814         if (lines_per_tag != 0)
3815                 add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
3816         add_cache_prop(devi, label, line_str, line_size);
3817         add_cache_prop(devi, label, size_str, size * 1024);
3818 }
3819 
3820 static void
3821 amd_cache_info(struct cpuid_info *cpi, dev_info_t *devi)
3822 {
3823         struct cpuid_regs *cp;
3824 
3825         if (cpi->cpi_xmaxeax < 0x80000005)
3826                 return;
3827         cp = &cpi->cpi_extd[5];
3828 
3829         /*
3830          * 4M/2M L1 TLB configuration
3831          *
3832          * We report the size for 2M pages because AMD uses two
3833          * TLB entries for one 4M page.
3834          */
3835         add_amd_tlb(devi, "dtlb-2M",
3836             BITX(cp->cp_eax, 31, 24), BITX(cp->cp_eax, 23, 16));
3837         add_amd_tlb(devi, "itlb-2M",
3838             BITX(cp->cp_eax, 15, 8), BITX(cp->cp_eax, 7, 0));
3839 
3840         /*
3841          * 4K L1 TLB configuration
3842          */
3843 
3844         switch (cpi->cpi_vendor) {
3845                 uint_t nentries;
3846         case X86_VENDOR_TM:
3847                 if (cpi->cpi_family >= 5) {
3848                         /*
3849                          * Crusoe processors have 256 TLB entries, but
3850                          * cpuid data format constrains them to only
3851                          * reporting 255 of them.
3852                          */
3853                         if ((nentries = BITX(cp->cp_ebx, 23, 16)) == 255)
3854                                 nentries = 256;
3855                         /*
3856                          * Crusoe processors also have a unified TLB
3857                          */
3858                         add_amd_tlb(devi, "tlb-4K", BITX(cp->cp_ebx, 31, 24),
3859                             nentries);
3860                         break;
3861                 }
3862                 /*FALLTHROUGH*/
3863         default:
3864                 add_amd_tlb(devi, itlb4k_str,
3865                     BITX(cp->cp_ebx, 31, 24), BITX(cp->cp_ebx, 23, 16));
3866                 add_amd_tlb(devi, dtlb4k_str,
3867                     BITX(cp->cp_ebx, 15, 8), BITX(cp->cp_ebx, 7, 0));
3868                 break;
3869         }
3870 
3871         /*
3872          * data L1 cache configuration
3873          */
3874 
3875         add_amd_cache(devi, l1_dcache_str,
3876             BITX(cp->cp_ecx, 31, 24), BITX(cp->cp_ecx, 23, 16),
3877             BITX(cp->cp_ecx, 15, 8), BITX(cp->cp_ecx, 7, 0));
3878 
3879         /*
3880          * code L1 cache configuration
3881          */
3882 
3883         add_amd_cache(devi, l1_icache_str,
3884             BITX(cp->cp_edx, 31, 24), BITX(cp->cp_edx, 23, 16),
3885             BITX(cp->cp_edx, 15, 8), BITX(cp->cp_edx, 7, 0));
3886 
3887         if (cpi->cpi_xmaxeax < 0x80000006)
3888                 return;
3889         cp = &cpi->cpi_extd[6];
3890 
3891         /* Check for a unified L2 TLB for large pages */
3892 
3893         if (BITX(cp->cp_eax, 31, 16) == 0)
3894                 add_amd_l2_tlb(devi, "l2-tlb-2M",
3895                     BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
3896         else {
3897                 add_amd_l2_tlb(devi, "l2-dtlb-2M",
3898                     BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
3899                 add_amd_l2_tlb(devi, "l2-itlb-2M",
3900                     BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
3901         }
3902 
3903         /* Check for a unified L2 TLB for 4K pages */
3904 
3905         if (BITX(cp->cp_ebx, 31, 16) == 0) {
3906                 add_amd_l2_tlb(devi, "l2-tlb-4K",
3907                     BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
3908         } else {
3909                 add_amd_l2_tlb(devi, "l2-dtlb-4K",
3910                     BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
3911                 add_amd_l2_tlb(devi, "l2-itlb-4K",
3912                     BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
3913         }
3914 
3915         add_amd_l2_cache(devi, l2_cache_str,
3916             BITX(cp->cp_ecx, 31, 16), BITX(cp->cp_ecx, 15, 12),
3917             BITX(cp->cp_ecx, 11, 8), BITX(cp->cp_ecx, 7, 0));
3918 }
3919 
3920 /*
3921  * There are two basic ways that the x86 world describes it cache
3922  * and tlb architecture - Intel's way and AMD's way.
3923  *
3924  * Return which flavor of cache architecture we should use
3925  */
3926 static int
3927 x86_which_cacheinfo(struct cpuid_info *cpi)
3928 {
3929         switch (cpi->cpi_vendor) {
3930         case X86_VENDOR_Intel:
3931                 if (cpi->cpi_maxeax >= 2)
3932                         return (X86_VENDOR_Intel);
3933                 break;
3934         case X86_VENDOR_AMD:
3935                 /*
3936                  * The K5 model 1 was the first part from AMD that reported
3937                  * cache sizes via extended cpuid functions.
3938                  */
3939                 if (cpi->cpi_family > 5 ||
3940                     (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
3941                         return (X86_VENDOR_AMD);
3942                 break;
3943         case X86_VENDOR_TM:
3944                 if (cpi->cpi_family >= 5)
3945                         return (X86_VENDOR_AMD);
3946                 /*FALLTHROUGH*/
3947         default:
3948                 /*
3949                  * If they have extended CPU data for 0x80000005
3950                  * then we assume they have AMD-format cache
3951                  * information.
3952                  *
3953                  * If not, and the vendor happens to be Cyrix,
3954                  * then try our-Cyrix specific handler.
3955                  *
3956                  * If we're not Cyrix, then assume we're using Intel's
3957                  * table-driven format instead.
3958                  */
3959                 if (cpi->cpi_xmaxeax >= 0x80000005)
3960                         return (X86_VENDOR_AMD);
3961                 else if (cpi->cpi_vendor == X86_VENDOR_Cyrix)
3962                         return (X86_VENDOR_Cyrix);
3963                 else if (cpi->cpi_maxeax >= 2)
3964                         return (X86_VENDOR_Intel);
3965                 break;
3966         }
3967         return (-1);
3968 }
3969 
3970 void
3971 cpuid_set_cpu_properties(void *dip, processorid_t cpu_id,
3972     struct cpuid_info *cpi)
3973 {
3974         dev_info_t *cpu_devi;
3975         int create;
3976 
3977         cpu_devi = (dev_info_t *)dip;
3978 
3979         /* device_type */
3980         (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
3981             "device_type", "cpu");
3982 
3983         /* reg */
3984         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
3985             "reg", cpu_id);
3986 
3987         /* cpu-mhz, and clock-frequency */
3988         if (cpu_freq > 0) {
3989                 long long mul;
3990 
3991                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
3992                     "cpu-mhz", cpu_freq);
3993                 if ((mul = cpu_freq * 1000000LL) <= INT_MAX)
3994                         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
3995                             "clock-frequency", (int)mul);
3996         }
3997 
3998         if (!is_x86_feature(x86_featureset, X86FSET_CPUID)) {
3999                 return;
4000         }
4001 
4002         /* vendor-id */
4003         (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4004             "vendor-id", cpi->cpi_vendorstr);
4005 
4006         if (cpi->cpi_maxeax == 0) {
4007                 return;
4008         }
4009 
4010         /*
4011          * family, model, and step
4012          */
4013         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4014             "family", CPI_FAMILY(cpi));
4015         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4016             "cpu-model", CPI_MODEL(cpi));
4017         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4018             "stepping-id", CPI_STEP(cpi));
4019 
4020         /* type */
4021         switch (cpi->cpi_vendor) {
4022         case X86_VENDOR_Intel:
4023                 create = 1;
4024                 break;
4025         default:
4026                 create = 0;
4027                 break;
4028         }
4029         if (create)
4030                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4031                     "type", CPI_TYPE(cpi));
4032 
4033         /* ext-family */
4034         switch (cpi->cpi_vendor) {
4035         case X86_VENDOR_Intel:
4036         case X86_VENDOR_AMD:
4037                 create = cpi->cpi_family >= 0xf;
4038                 break;
4039         default:
4040                 create = 0;
4041                 break;
4042         }
4043         if (create)
4044                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4045                     "ext-family", CPI_FAMILY_XTD(cpi));
4046 
4047         /* ext-model */
4048         switch (cpi->cpi_vendor) {
4049         case X86_VENDOR_Intel:
4050                 create = IS_EXTENDED_MODEL_INTEL(cpi);
4051                 break;
4052         case X86_VENDOR_AMD:
4053                 create = CPI_FAMILY(cpi) == 0xf;
4054                 break;
4055         default:
4056                 create = 0;
4057                 break;
4058         }
4059         if (create)
4060                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4061                     "ext-model", CPI_MODEL_XTD(cpi));
4062 
4063         /* generation */
4064         switch (cpi->cpi_vendor) {
4065         case X86_VENDOR_AMD:
4066                 /*
4067                  * AMD K5 model 1 was the first part to support this
4068                  */
4069                 create = cpi->cpi_xmaxeax >= 0x80000001;
4070                 break;
4071         default:
4072                 create = 0;
4073                 break;
4074         }
4075         if (create)
4076                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4077                     "generation", BITX((cpi)->cpi_extd[1].cp_eax, 11, 8));
4078 
4079         /* brand-id */
4080         switch (cpi->cpi_vendor) {
4081         case X86_VENDOR_Intel:
4082                 /*
4083                  * brand id first appeared on Pentium III Xeon model 8,
4084                  * and Celeron model 8 processors and Opteron
4085                  */
4086                 create = cpi->cpi_family > 6 ||
4087                     (cpi->cpi_family == 6 && cpi->cpi_model >= 8);
4088                 break;
4089         case X86_VENDOR_AMD:
4090                 create = cpi->cpi_family >= 0xf;
4091                 break;
4092         default:
4093                 create = 0;
4094                 break;
4095         }
4096         if (create && cpi->cpi_brandid != 0) {
4097                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4098                     "brand-id", cpi->cpi_brandid);
4099         }
4100 
4101         /* chunks, and apic-id */
4102         switch (cpi->cpi_vendor) {
4103                 /*
4104                  * first available on Pentium IV and Opteron (K8)
4105                  */
4106         case X86_VENDOR_Intel:
4107                 create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
4108                 break;
4109         case X86_VENDOR_AMD:
4110                 create = cpi->cpi_family >= 0xf;
4111                 break;
4112         default:
4113                 create = 0;
4114                 break;
4115         }
4116         if (create) {
4117                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4118                     "chunks", CPI_CHUNKS(cpi));
4119                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4120                     "apic-id", cpi->cpi_apicid);
4121                 if (cpi->cpi_chipid >= 0) {
4122                         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4123                             "chip#", cpi->cpi_chipid);
4124                         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4125                             "clog#", cpi->cpi_clogid);
4126                 }
4127         }
4128 
4129         /* cpuid-features */
4130         (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4131             "cpuid-features", CPI_FEATURES_EDX(cpi));
4132 
4133 
4134         /* cpuid-features-ecx */
4135         switch (cpi->cpi_vendor) {
4136         case X86_VENDOR_Intel:
4137                 create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
4138                 break;
4139         case X86_VENDOR_AMD:
4140                 create = cpi->cpi_family >= 0xf;
4141                 break;
4142         default:
4143                 create = 0;
4144                 break;
4145         }
4146         if (create)
4147                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4148                     "cpuid-features-ecx", CPI_FEATURES_ECX(cpi));
4149 
4150         /* ext-cpuid-features */
4151         switch (cpi->cpi_vendor) {
4152         case X86_VENDOR_Intel:
4153         case X86_VENDOR_AMD:
4154         case X86_VENDOR_Cyrix:
4155         case X86_VENDOR_TM:
4156         case X86_VENDOR_Centaur:
4157                 create = cpi->cpi_xmaxeax >= 0x80000001;
4158                 break;
4159         default:
4160                 create = 0;
4161                 break;
4162         }
4163         if (create) {
4164                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4165                     "ext-cpuid-features", CPI_FEATURES_XTD_EDX(cpi));
4166                 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4167                     "ext-cpuid-features-ecx", CPI_FEATURES_XTD_ECX(cpi));
4168         }
4169 
4170         /*
4171          * Brand String first appeared in Intel Pentium IV, AMD K5
4172          * model 1, and Cyrix GXm.  On earlier models we try and
4173          * simulate something similar .. so this string should always
4174          * same -something- about the processor, however lame.
4175          */
4176         (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4177             "brand-string", cpi->cpi_brandstr);
4178 
4179         /*
4180          * Finally, cache and tlb information
4181          */
4182         switch (x86_which_cacheinfo(cpi)) {
4183         case X86_VENDOR_Intel:
4184                 intel_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
4185                 break;
4186         case X86_VENDOR_Cyrix:
4187                 cyrix_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
4188                 break;
4189         case X86_VENDOR_AMD:
4190                 amd_cache_info(cpi, cpu_devi);
4191                 break;
4192         default:
4193                 break;
4194         }
4195 }
4196 
4197 struct l2info {
4198         int *l2i_csz;
4199         int *l2i_lsz;
4200         int *l2i_assoc;
4201         int l2i_ret;
4202 };
4203 
4204 /*
4205  * A cacheinfo walker that fetches the size, line-size and associativity
4206  * of the L2 cache
4207  */
4208 static int
4209 intel_l2cinfo(void *arg, const struct cachetab *ct)
4210 {
4211         struct l2info *l2i = arg;
4212         int *ip;
4213 
4214         if (ct->ct_label != l2_cache_str &&
4215             ct->ct_label != sl2_cache_str)
4216                 return (0);     /* not an L2 -- keep walking */
4217 
4218         if ((ip = l2i->l2i_csz) != NULL)
4219                 *ip = ct->ct_size;
4220         if ((ip = l2i->l2i_lsz) != NULL)
4221                 *ip = ct->ct_line_size;
4222         if ((ip = l2i->l2i_assoc) != NULL)
4223                 *ip = ct->ct_assoc;
4224         l2i->l2i_ret = ct->ct_size;
4225         return (1);             /* was an L2 -- terminate walk */
4226 }
4227 
4228 /*
4229  * AMD L2/L3 Cache and TLB Associativity Field Definition:
4230  *
4231  *      Unlike the associativity for the L1 cache and tlb where the 8 bit
4232  *      value is the associativity, the associativity for the L2 cache and
4233  *      tlb is encoded in the following table. The 4 bit L2 value serves as
4234  *      an index into the amd_afd[] array to determine the associativity.
4235  *      -1 is undefined. 0 is fully associative.
4236  */
4237 
4238 static int amd_afd[] =
4239         {-1, 1, 2, -1, 4, -1, 8, -1, 16, -1, 32, 48, 64, 96, 128, 0};
4240 
4241 static void
4242 amd_l2cacheinfo(struct cpuid_info *cpi, struct l2info *l2i)
4243 {
4244         struct cpuid_regs *cp;
4245         uint_t size, assoc;
4246         int i;
4247         int *ip;
4248 
4249         if (cpi->cpi_xmaxeax < 0x80000006)
4250                 return;
4251         cp = &cpi->cpi_extd[6];
4252 
4253         if ((i = BITX(cp->cp_ecx, 15, 12)) != 0 &&
4254             (size = BITX(cp->cp_ecx, 31, 16)) != 0) {
4255                 uint_t cachesz = size * 1024;
4256                 assoc = amd_afd[i];
4257 
4258                 ASSERT(assoc != -1);
4259 
4260                 if ((ip = l2i->l2i_csz) != NULL)
4261                         *ip = cachesz;
4262                 if ((ip = l2i->l2i_lsz) != NULL)
4263                         *ip = BITX(cp->cp_ecx, 7, 0);
4264                 if ((ip = l2i->l2i_assoc) != NULL)
4265                         *ip = assoc;
4266                 l2i->l2i_ret = cachesz;
4267         }
4268 }
4269 
4270 int
4271 getl2cacheinfo(cpu_t *cpu, int *csz, int *lsz, int *assoc)
4272 {
4273         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
4274         struct l2info __l2info, *l2i = &__l2info;
4275 
4276         l2i->l2i_csz = csz;
4277         l2i->l2i_lsz = lsz;
4278         l2i->l2i_assoc = assoc;
4279         l2i->l2i_ret = -1;
4280 
4281         switch (x86_which_cacheinfo(cpi)) {
4282         case X86_VENDOR_Intel:
4283                 intel_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
4284                 break;
4285         case X86_VENDOR_Cyrix:
4286                 cyrix_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
4287                 break;
4288         case X86_VENDOR_AMD:
4289                 amd_l2cacheinfo(cpi, l2i);
4290                 break;
4291         default:
4292                 break;
4293         }
4294         return (l2i->l2i_ret);
4295 }
4296 
4297 #if !defined(__xpv)
4298 
4299 uint32_t *
4300 cpuid_mwait_alloc(cpu_t *cpu)
4301 {
4302         uint32_t        *ret;
4303         size_t          mwait_size;
4304 
4305         ASSERT(cpuid_checkpass(CPU, 2));
4306 
4307         mwait_size = CPU->cpu_m.mcpu_cpi->cpi_mwait.mon_max;
4308         if (mwait_size == 0)
4309                 return (NULL);
4310 
4311         /*
4312          * kmem_alloc() returns cache line size aligned data for mwait_size
4313          * allocations.  mwait_size is currently cache line sized.  Neither
4314          * of these implementation details are guarantied to be true in the
4315          * future.
4316          *
4317          * First try allocating mwait_size as kmem_alloc() currently returns
4318          * correctly aligned memory.  If kmem_alloc() does not return
4319          * mwait_size aligned memory, then use mwait_size ROUNDUP.
4320          *
4321          * Set cpi_mwait.buf_actual and cpi_mwait.size_actual in case we
4322          * decide to free this memory.
4323          */
4324         ret = kmem_zalloc(mwait_size, KM_SLEEP);
4325         if (ret == (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size)) {
4326                 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = ret;
4327                 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = mwait_size;
4328                 *ret = MWAIT_RUNNING;
4329                 return (ret);
4330         } else {
4331                 kmem_free(ret, mwait_size);
4332                 ret = kmem_zalloc(mwait_size * 2, KM_SLEEP);
4333                 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = ret;
4334                 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = mwait_size * 2;
4335                 ret = (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size);
4336                 *ret = MWAIT_RUNNING;
4337                 return (ret);
4338         }
4339 }
4340 
4341 void
4342 cpuid_mwait_free(cpu_t *cpu)
4343 {
4344         if (cpu->cpu_m.mcpu_cpi == NULL) {
4345                 return;
4346         }
4347 
4348         if (cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual != NULL &&
4349             cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual > 0) {
4350                 kmem_free(cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual,
4351                     cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual);
4352         }
4353 
4354         cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = NULL;
4355         cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = 0;
4356 }
4357 
4358 void
4359 patch_tsc_read(int flag)
4360 {
4361         size_t cnt;
4362 
4363         switch (flag) {
4364         case X86_NO_TSC:
4365                 cnt = &_no_rdtsc_end - &_no_rdtsc_start;
4366                 (void) memcpy((void *)tsc_read, (void *)&_no_rdtsc_start, cnt);
4367                 break;
4368         case X86_HAVE_TSCP:
4369                 cnt = &_tscp_end - &_tscp_start;
4370                 (void) memcpy((void *)tsc_read, (void *)&_tscp_start, cnt);
4371                 break;
4372         case X86_TSC_MFENCE:
4373                 cnt = &_tsc_mfence_end - &_tsc_mfence_start;
4374                 (void) memcpy((void *)tsc_read,
4375                     (void *)&_tsc_mfence_start, cnt);
4376                 break;
4377         case X86_TSC_LFENCE:
4378                 cnt = &_tsc_lfence_end - &_tsc_lfence_start;
4379                 (void) memcpy((void *)tsc_read,
4380                     (void *)&_tsc_lfence_start, cnt);
4381                 break;
4382         default:
4383                 break;
4384         }
4385 }
4386 
4387 int
4388 cpuid_deep_cstates_supported(void)
4389 {
4390         struct cpuid_info *cpi;
4391         struct cpuid_regs regs;
4392 
4393         ASSERT(cpuid_checkpass(CPU, 1));
4394 
4395         cpi = CPU->cpu_m.mcpu_cpi;
4396 
4397         if (!is_x86_feature(x86_featureset, X86FSET_CPUID))
4398                 return (0);
4399 
4400         switch (cpi->cpi_vendor) {
4401         case X86_VENDOR_Intel:
4402                 if (cpi->cpi_xmaxeax < 0x80000007)
4403                         return (0);
4404 
4405                 /*
4406                  * TSC run at a constant rate in all ACPI C-states?
4407                  */
4408                 regs.cp_eax = 0x80000007;
4409                 (void) __cpuid_insn(&regs);
4410                 return (regs.cp_edx & CPUID_TSC_CSTATE_INVARIANCE);
4411 
4412         default:
4413                 return (0);
4414         }
4415 }
4416 
4417 #endif  /* !__xpv */
4418 
4419 void
4420 post_startup_cpu_fixups(void)
4421 {
4422 #ifndef __xpv
4423         /*
4424          * Some AMD processors support C1E state. Entering this state will
4425          * cause the local APIC timer to stop, which we can't deal with at
4426          * this time.
4427          */
4428         if (cpuid_getvendor(CPU) == X86_VENDOR_AMD) {
4429                 on_trap_data_t otd;
4430                 uint64_t reg;
4431 
4432                 if (!on_trap(&otd, OT_DATA_ACCESS)) {
4433                         reg = rdmsr(MSR_AMD_INT_PENDING_CMP_HALT);
4434                         /* Disable C1E state if it is enabled by BIOS */
4435                         if ((reg >> AMD_ACTONCMPHALT_SHIFT) &
4436                             AMD_ACTONCMPHALT_MASK) {
4437                                 reg &= ~(AMD_ACTONCMPHALT_MASK <<
4438                                     AMD_ACTONCMPHALT_SHIFT);
4439                                 wrmsr(MSR_AMD_INT_PENDING_CMP_HALT, reg);
4440                         }
4441                 }
4442                 no_trap();
4443         }
4444 #endif  /* !__xpv */
4445 }
4446 
4447 /*
4448  * Setup necessary registers to enable XSAVE feature on this processor.
4449  * This function needs to be called early enough, so that no xsave/xrstor
4450  * ops will execute on the processor before the MSRs are properly set up.
4451  *
4452  * Current implementation has the following assumption:
4453  * - cpuid_pass1() is done, so that X86 features are known.
4454  * - fpu_probe() is done, so that fp_save_mech is chosen.
4455  */
4456 void
4457 xsave_setup_msr(cpu_t *cpu)
4458 {
4459         ASSERT(fp_save_mech == FP_XSAVE);
4460         ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
4461 
4462         /* Enable OSXSAVE in CR4. */
4463         setcr4(getcr4() | CR4_OSXSAVE);
4464         /*
4465          * Update SW copy of ECX, so that /dev/cpu/self/cpuid will report
4466          * correct value.
4467          */
4468         cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_ecx |= CPUID_INTC_ECX_OSXSAVE;
4469         setup_xfem();
4470 }
4471 
4472 /*
4473  * Starting with the Westmere processor the local
4474  * APIC timer will continue running in all C-states,
4475  * including the deepest C-states.
4476  */
4477 int
4478 cpuid_arat_supported(void)
4479 {
4480         struct cpuid_info *cpi;
4481         struct cpuid_regs regs;
4482 
4483         ASSERT(cpuid_checkpass(CPU, 1));
4484         ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
4485 
4486         cpi = CPU->cpu_m.mcpu_cpi;
4487 
4488         switch (cpi->cpi_vendor) {
4489         case X86_VENDOR_Intel:
4490                 /*
4491                  * Always-running Local APIC Timer is
4492                  * indicated by CPUID.6.EAX[2].
4493                  */
4494                 if (cpi->cpi_maxeax >= 6) {
4495                         regs.cp_eax = 6;
4496                         (void) cpuid_insn(NULL, &regs);
4497                         return (regs.cp_eax & CPUID_CSTATE_ARAT);
4498                 } else {
4499                         return (0);
4500                 }
4501         default:
4502                 return (0);
4503         }
4504 }
4505 
4506 /*
4507  * Check support for Intel ENERGY_PERF_BIAS feature
4508  */
4509 int
4510 cpuid_iepb_supported(struct cpu *cp)
4511 {
4512         struct cpuid_info *cpi = cp->cpu_m.mcpu_cpi;
4513         struct cpuid_regs regs;
4514 
4515         ASSERT(cpuid_checkpass(cp, 1));
4516 
4517         if (!(is_x86_feature(x86_featureset, X86FSET_CPUID)) ||
4518             !(is_x86_feature(x86_featureset, X86FSET_MSR))) {
4519                 return (0);
4520         }
4521 
4522         /*
4523          * Intel ENERGY_PERF_BIAS MSR is indicated by
4524          * capability bit CPUID.6.ECX.3
4525          */
4526         if ((cpi->cpi_vendor != X86_VENDOR_Intel) || (cpi->cpi_maxeax < 6))
4527                 return (0);
4528 
4529         regs.cp_eax = 0x6;
4530         (void) cpuid_insn(NULL, &regs);
4531         return (regs.cp_ecx & CPUID_EPB_SUPPORT);
4532 }
4533 
4534 /*
4535  * Check support for TSC deadline timer
4536  *
4537  * TSC deadline timer provides a superior software programming
4538  * model over local APIC timer that eliminates "time drifts".
4539  * Instead of specifying a relative time, software specifies an
4540  * absolute time as the target at which the processor should
4541  * generate a timer event.
4542  */
4543 int
4544 cpuid_deadline_tsc_supported(void)
4545 {
4546         struct cpuid_info *cpi = CPU->cpu_m.mcpu_cpi;
4547         struct cpuid_regs regs;
4548 
4549         ASSERT(cpuid_checkpass(CPU, 1));
4550         ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
4551 
4552         switch (cpi->cpi_vendor) {
4553         case X86_VENDOR_Intel:
4554                 if (cpi->cpi_maxeax >= 1) {
4555                         regs.cp_eax = 1;
4556                         (void) cpuid_insn(NULL, &regs);
4557                         return (regs.cp_ecx & CPUID_DEADLINE_TSC);
4558                 } else {
4559                         return (0);
4560                 }
4561         default:
4562                 return (0);
4563         }
4564 }
4565 
4566 #if defined(__amd64) && !defined(__xpv)
4567 /*
4568  * Patch in versions of bcopy for high performance Intel Nhm processors
4569  * and later...
4570  */
4571 void
4572 patch_memops(uint_t vendor)
4573 {
4574         size_t cnt, i;
4575         caddr_t to, from;
4576 
4577         if ((vendor == X86_VENDOR_Intel) &&
4578             is_x86_feature(x86_featureset, X86FSET_SSE4_2)) {
4579                 cnt = &bcopy_patch_end - &bcopy_patch_start;
4580                 to = &bcopy_ck_size;
4581                 from = &bcopy_patch_start;
4582                 for (i = 0; i < cnt; i++) {
4583                         *to++ = *from++;
4584                 }
4585         }
4586 }
4587 #endif  /* __amd64 && !__xpv */
4588 
4589 /*
4590  * This function finds the number of bits to represent the number of cores per
4591  * chip and the number of strands per core for the Intel platforms.
4592  * It re-uses the x2APIC cpuid code of the cpuid_pass2().
4593  */
4594 void
4595 cpuid_get_ext_topo(uint_t vendor, uint_t *core_nbits, uint_t *strand_nbits)
4596 {
4597         struct cpuid_regs regs;
4598         struct cpuid_regs *cp = &regs;
4599 
4600         if (vendor != X86_VENDOR_Intel) {
4601                 return;
4602         }
4603 
4604         /* if the cpuid level is 0xB, extended topo is available. */
4605         cp->cp_eax = 0;
4606         if (__cpuid_insn(cp) >= 0xB) {
4607 
4608                 cp->cp_eax = 0xB;
4609                 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
4610                 (void) __cpuid_insn(cp);
4611 
4612                 /*
4613                  * Check CPUID.EAX=0BH, ECX=0H:EBX is non-zero, which
4614                  * indicates that the extended topology enumeration leaf is
4615                  * available.
4616                  */
4617                 if (cp->cp_ebx) {
4618                         uint_t coreid_shift = 0;
4619                         uint_t chipid_shift = 0;
4620                         uint_t i;
4621                         uint_t level;
4622 
4623                         for (i = 0; i < CPI_FNB_ECX_MAX; i++) {
4624                                 cp->cp_eax = 0xB;
4625                                 cp->cp_ecx = i;
4626 
4627                                 (void) __cpuid_insn(cp);
4628                                 level = CPI_CPU_LEVEL_TYPE(cp);
4629 
4630                                 if (level == 1) {
4631                                         /*
4632                                          * Thread level processor topology
4633                                          * Number of bits shift right APIC ID
4634                                          * to get the coreid.
4635                                          */
4636                                         coreid_shift = BITX(cp->cp_eax, 4, 0);
4637                                 } else if (level == 2) {
4638                                         /*
4639                                          * Core level processor topology
4640                                          * Number of bits shift right APIC ID
4641                                          * to get the chipid.
4642                                          */
4643                                         chipid_shift = BITX(cp->cp_eax, 4, 0);
4644                                 }
4645                         }
4646 
4647                         if (coreid_shift > 0 && chipid_shift > coreid_shift) {
4648                                 *strand_nbits = coreid_shift;
4649                                 *core_nbits = chipid_shift - coreid_shift;
4650                         }
4651                 }
4652         }
4653 }