1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2011 by Delphix. All rights reserved.
24 */
25 /*
26 * Copyright (c) 2010, Intel Corporation.
27 * All rights reserved.
28 */
29 /*
30 * Portions Copyright 2009 Advanced Micro Devices, Inc.
31 */
32 /*
33 * Copyright (c) 2011, Joyent, Inc. All rights reserved.
34 */
35 /*
36 * Various routines to handle identification
37 * and classification of x86 processors.
38 */
39
40 #include <sys/types.h>
41 #include <sys/archsystm.h>
42 #include <sys/x86_archext.h>
43 #include <sys/kmem.h>
44 #include <sys/systm.h>
45 #include <sys/cmn_err.h>
46 #include <sys/sunddi.h>
47 #include <sys/sunndi.h>
48 #include <sys/cpuvar.h>
49 #include <sys/processor.h>
50 #include <sys/sysmacros.h>
51 #include <sys/pg.h>
52 #include <sys/fp.h>
53 #include <sys/controlregs.h>
54 #include <sys/bitmap.h>
55 #include <sys/auxv_386.h>
56 #include <sys/memnode.h>
57 #include <sys/pci_cfgspace.h>
58
59 #ifdef __xpv
60 #include <sys/hypervisor.h>
61 #else
62 #include <sys/ontrap.h>
63 #endif
64
65 /*
66 * Pass 0 of cpuid feature analysis happens in locore. It contains special code
67 * to recognize Cyrix processors that are not cpuid-compliant, and to deal with
68 * them accordingly. For most modern processors, feature detection occurs here
69 * in pass 1.
70 *
71 * Pass 1 of cpuid feature analysis happens just at the beginning of mlsetup()
72 * for the boot CPU and does the basic analysis that the early kernel needs.
73 * x86_featureset is set based on the return value of cpuid_pass1() of the boot
74 * CPU.
75 *
76 * Pass 1 includes:
77 *
78 * o Determining vendor/model/family/stepping and setting x86_type and
79 * x86_vendor accordingly.
80 * o Processing the feature flags returned by the cpuid instruction while
81 * applying any workarounds or tricks for the specific processor.
82 * o Mapping the feature flags into Solaris feature bits (X86_*).
83 * o Processing extended feature flags if supported by the processor,
84 * again while applying specific processor knowledge.
85 * o Determining the CMT characteristics of the system.
86 *
87 * Pass 1 is done on non-boot CPUs during their initialization and the results
88 * are used only as a meager attempt at ensuring that all processors within the
89 * system support the same features.
90 *
91 * Pass 2 of cpuid feature analysis happens just at the beginning
92 * of startup(). It just copies in and corrects the remainder
93 * of the cpuid data we depend on: standard cpuid functions that we didn't
94 * need for pass1 feature analysis, and extended cpuid functions beyond the
95 * simple feature processing done in pass1.
96 *
97 * Pass 3 of cpuid analysis is invoked after basic kernel services; in
98 * particular kernel memory allocation has been made available. It creates a
99 * readable brand string based on the data collected in the first two passes.
100 *
101 * Pass 4 of cpuid analysis is invoked after post_startup() when all
102 * the support infrastructure for various hardware features has been
103 * initialized. It determines which processor features will be reported
104 * to userland via the aux vector.
105 *
106 * All passes are executed on all CPUs, but only the boot CPU determines what
107 * features the kernel will use.
108 *
109 * Much of the worst junk in this file is for the support of processors
110 * that didn't really implement the cpuid instruction properly.
111 *
112 * NOTE: The accessor functions (cpuid_get*) are aware of, and ASSERT upon,
113 * the pass numbers. Accordingly, changes to the pass code may require changes
114 * to the accessor code.
115 */
116
117 uint_t x86_vendor = X86_VENDOR_IntelClone;
118 uint_t x86_type = X86_TYPE_OTHER;
119 uint_t x86_clflush_size = 0;
120
121 uint_t pentiumpro_bug4046376;
122 uint_t pentiumpro_bug4064495;
123
124 uchar_t x86_featureset[BT_SIZEOFMAP(NUM_X86_FEATURES)];
125
126 static char *x86_feature_names[NUM_X86_FEATURES] = {
127 "lgpg",
128 "tsc",
129 "msr",
130 "mtrr",
131 "pge",
132 "de",
133 "cmov",
134 "mmx",
135 "mca",
136 "pae",
137 "cv8",
138 "pat",
139 "sep",
140 "sse",
141 "sse2",
142 "htt",
143 "asysc",
144 "nx",
145 "sse3",
146 "cx16",
147 "cmp",
148 "tscp",
149 "mwait",
150 "sse4a",
151 "cpuid",
152 "ssse3",
153 "sse4_1",
154 "sse4_2",
155 "1gpg",
156 "clfsh",
157 "64",
158 "aes",
159 "pclmulqdq",
160 "xsave",
161 "avx",
162 "vmx",
163 "svm",
164 "topoext"
165 };
166
167 boolean_t
168 is_x86_feature(void *featureset, uint_t feature)
169 {
170 ASSERT(feature < NUM_X86_FEATURES);
171 return (BT_TEST((ulong_t *)featureset, feature));
172 }
173
174 void
175 add_x86_feature(void *featureset, uint_t feature)
176 {
177 ASSERT(feature < NUM_X86_FEATURES);
178 BT_SET((ulong_t *)featureset, feature);
179 }
180
181 void
182 remove_x86_feature(void *featureset, uint_t feature)
183 {
184 ASSERT(feature < NUM_X86_FEATURES);
185 BT_CLEAR((ulong_t *)featureset, feature);
186 }
187
188 boolean_t
189 compare_x86_featureset(void *setA, void *setB)
190 {
191 /*
192 * We assume that the unused bits of the bitmap are always zero.
193 */
194 if (memcmp(setA, setB, BT_SIZEOFMAP(NUM_X86_FEATURES)) == 0) {
195 return (B_TRUE);
196 } else {
197 return (B_FALSE);
198 }
199 }
200
201 void
202 print_x86_featureset(void *featureset)
203 {
204 uint_t i;
205
206 for (i = 0; i < NUM_X86_FEATURES; i++) {
207 if (is_x86_feature(featureset, i)) {
208 cmn_err(CE_CONT, "?x86_feature: %s\n",
209 x86_feature_names[i]);
210 }
211 }
212 }
213
214 uint_t enable486;
215
216 static size_t xsave_state_size = 0;
217 uint64_t xsave_bv_all = (XFEATURE_LEGACY_FP | XFEATURE_SSE);
218 boolean_t xsave_force_disable = B_FALSE;
219
220 /*
221 * This is set to platform type Solaris is running on.
222 */
223 static int platform_type = -1;
224
225 #if !defined(__xpv)
226 /*
227 * Variable to patch if hypervisor platform detection needs to be
228 * disabled (e.g. platform_type will always be HW_NATIVE if this is 0).
229 */
230 int enable_platform_detection = 1;
231 #endif
232
233 /*
234 * monitor/mwait info.
235 *
236 * size_actual and buf_actual are the real address and size allocated to get
237 * proper mwait_buf alignement. buf_actual and size_actual should be passed
238 * to kmem_free(). Currently kmem_alloc() and mwait happen to both use
239 * processor cache-line alignment, but this is not guarantied in the furture.
240 */
241 struct mwait_info {
242 size_t mon_min; /* min size to avoid missed wakeups */
243 size_t mon_max; /* size to avoid false wakeups */
244 size_t size_actual; /* size actually allocated */
245 void *buf_actual; /* memory actually allocated */
246 uint32_t support; /* processor support of monitor/mwait */
247 };
248
249 /*
250 * xsave/xrestor info.
251 *
252 * This structure contains HW feature bits and size of the xsave save area.
253 * Note: the kernel will use the maximum size required for all hardware
254 * features. It is not optimize for potential memory savings if features at
255 * the end of the save area are not enabled.
256 */
257 struct xsave_info {
258 uint32_t xsav_hw_features_low; /* Supported HW features */
259 uint32_t xsav_hw_features_high; /* Supported HW features */
260 size_t xsav_max_size; /* max size save area for HW features */
261 size_t ymm_size; /* AVX: size of ymm save area */
262 size_t ymm_offset; /* AVX: offset for ymm save area */
263 };
264
265
266 /*
267 * These constants determine how many of the elements of the
268 * cpuid we cache in the cpuid_info data structure; the
269 * remaining elements are accessible via the cpuid instruction.
270 */
271
272 #define NMAX_CPI_STD 6 /* eax = 0 .. 5 */
273 #define NMAX_CPI_EXTD 0x1f /* eax = 0x80000000 .. 0x8000001e */
274
275 /*
276 * Some terminology needs to be explained:
277 * - Socket: Something that can be plugged into a motherboard.
278 * - Package: Same as socket
279 * - Chip: Same as socket. Note that AMD's documentation uses term "chip"
280 * differently: there, chip is the same as processor node (below)
281 * - Processor node: Some AMD processors have more than one
282 * "subprocessor" embedded in a package. These subprocessors (nodes)
283 * are fully-functional processors themselves with cores, caches,
284 * memory controllers, PCI configuration spaces. They are connected
285 * inside the package with Hypertransport links. On single-node
286 * processors, processor node is equivalent to chip/socket/package.
287 * - Compute Unit: Some AMD processors pair cores in "compute units" that
288 * share the FPU and the L1I and L2 caches.
289 */
290
291 struct cpuid_info {
292 uint_t cpi_pass; /* last pass completed */
293 /*
294 * standard function information
295 */
296 uint_t cpi_maxeax; /* fn 0: %eax */
297 char cpi_vendorstr[13]; /* fn 0: %ebx:%ecx:%edx */
298 uint_t cpi_vendor; /* enum of cpi_vendorstr */
299
300 uint_t cpi_family; /* fn 1: extended family */
301 uint_t cpi_model; /* fn 1: extended model */
302 uint_t cpi_step; /* fn 1: stepping */
303 chipid_t cpi_chipid; /* fn 1: %ebx: Intel: chip # */
304 /* AMD: package/socket # */
305 uint_t cpi_brandid; /* fn 1: %ebx: brand ID */
306 int cpi_clogid; /* fn 1: %ebx: thread # */
307 uint_t cpi_ncpu_per_chip; /* fn 1: %ebx: logical cpu count */
308 uint8_t cpi_cacheinfo[16]; /* fn 2: intel-style cache desc */
309 uint_t cpi_ncache; /* fn 2: number of elements */
310 uint_t cpi_ncpu_shr_last_cache; /* fn 4: %eax: ncpus sharing cache */
311 id_t cpi_last_lvl_cacheid; /* fn 4: %eax: derived cache id */
312 uint_t cpi_std_4_size; /* fn 4: number of fn 4 elements */
313 struct cpuid_regs **cpi_std_4; /* fn 4: %ecx == 0 .. fn4_size */
314 struct cpuid_regs cpi_std[NMAX_CPI_STD]; /* 0 .. 5 */
315 /*
316 * extended function information
317 */
318 uint_t cpi_xmaxeax; /* fn 0x80000000: %eax */
319 char cpi_brandstr[49]; /* fn 0x8000000[234] */
320 uint8_t cpi_pabits; /* fn 0x80000006: %eax */
321 uint8_t cpi_vabits; /* fn 0x80000006: %eax */
322 struct cpuid_regs cpi_extd[NMAX_CPI_EXTD]; /* 0x800000XX */
323
324 id_t cpi_coreid; /* same coreid => strands share core */
325 int cpi_pkgcoreid; /* core number within single package */
326 uint_t cpi_ncore_per_chip; /* AMD: fn 0x80000008: %ecx[7-0] */
327 /* Intel: fn 4: %eax[31-26] */
328 /*
329 * supported feature information
330 */
331 uint32_t cpi_support[5];
332 #define STD_EDX_FEATURES 0
333 #define AMD_EDX_FEATURES 1
334 #define TM_EDX_FEATURES 2
335 #define STD_ECX_FEATURES 3
336 #define AMD_ECX_FEATURES 4
337 /*
338 * Synthesized information, where known.
339 */
340 uint32_t cpi_chiprev; /* See X86_CHIPREV_* in x86_archext.h */
341 const char *cpi_chiprevstr; /* May be NULL if chiprev unknown */
342 uint32_t cpi_socket; /* Chip package/socket type */
343
344 struct mwait_info cpi_mwait; /* fn 5: monitor/mwait info */
345 uint32_t cpi_apicid;
346 uint_t cpi_procnodeid; /* AMD: nodeID on HT, Intel: chipid */
347 uint_t cpi_procnodes_per_pkg; /* AMD: # of nodes in the package */
348 /* Intel: 1 */
349 uint_t cpi_compunitid; /* AMD: compute unit ID, Intel: coreid */
350 uint_t cpi_cores_per_compunit; /* AMD: # of cores in the compute unit */
351
352 struct xsave_info cpi_xsave; /* fn D: xsave/xrestor info */
353 };
354
355
356 static struct cpuid_info cpuid_info0;
357
358 /*
359 * These bit fields are defined by the Intel Application Note AP-485
360 * "Intel Processor Identification and the CPUID Instruction"
361 */
362 #define CPI_FAMILY_XTD(cpi) BITX((cpi)->cpi_std[1].cp_eax, 27, 20)
363 #define CPI_MODEL_XTD(cpi) BITX((cpi)->cpi_std[1].cp_eax, 19, 16)
364 #define CPI_TYPE(cpi) BITX((cpi)->cpi_std[1].cp_eax, 13, 12)
365 #define CPI_FAMILY(cpi) BITX((cpi)->cpi_std[1].cp_eax, 11, 8)
366 #define CPI_STEP(cpi) BITX((cpi)->cpi_std[1].cp_eax, 3, 0)
367 #define CPI_MODEL(cpi) BITX((cpi)->cpi_std[1].cp_eax, 7, 4)
368
369 #define CPI_FEATURES_EDX(cpi) ((cpi)->cpi_std[1].cp_edx)
370 #define CPI_FEATURES_ECX(cpi) ((cpi)->cpi_std[1].cp_ecx)
371 #define CPI_FEATURES_XTD_EDX(cpi) ((cpi)->cpi_extd[1].cp_edx)
372 #define CPI_FEATURES_XTD_ECX(cpi) ((cpi)->cpi_extd[1].cp_ecx)
373
374 #define CPI_BRANDID(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 7, 0)
375 #define CPI_CHUNKS(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 15, 7)
376 #define CPI_CPU_COUNT(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 23, 16)
377 #define CPI_APIC_ID(cpi) BITX((cpi)->cpi_std[1].cp_ebx, 31, 24)
378
379 #define CPI_MAXEAX_MAX 0x100 /* sanity control */
380 #define CPI_XMAXEAX_MAX 0x80000100
381 #define CPI_FN4_ECX_MAX 0x20 /* sanity: max fn 4 levels */
382 #define CPI_FNB_ECX_MAX 0x20 /* sanity: max fn B levels */
383
384 /*
385 * Function 4 (Deterministic Cache Parameters) macros
386 * Defined by Intel Application Note AP-485
387 */
388 #define CPI_NUM_CORES(regs) BITX((regs)->cp_eax, 31, 26)
389 #define CPI_NTHR_SHR_CACHE(regs) BITX((regs)->cp_eax, 25, 14)
390 #define CPI_FULL_ASSOC_CACHE(regs) BITX((regs)->cp_eax, 9, 9)
391 #define CPI_SELF_INIT_CACHE(regs) BITX((regs)->cp_eax, 8, 8)
392 #define CPI_CACHE_LVL(regs) BITX((regs)->cp_eax, 7, 5)
393 #define CPI_CACHE_TYPE(regs) BITX((regs)->cp_eax, 4, 0)
394 #define CPI_CPU_LEVEL_TYPE(regs) BITX((regs)->cp_ecx, 15, 8)
395
396 #define CPI_CACHE_WAYS(regs) BITX((regs)->cp_ebx, 31, 22)
397 #define CPI_CACHE_PARTS(regs) BITX((regs)->cp_ebx, 21, 12)
398 #define CPI_CACHE_COH_LN_SZ(regs) BITX((regs)->cp_ebx, 11, 0)
399
400 #define CPI_CACHE_SETS(regs) BITX((regs)->cp_ecx, 31, 0)
401
402 #define CPI_PREFCH_STRIDE(regs) BITX((regs)->cp_edx, 9, 0)
403
404
405 /*
406 * A couple of shorthand macros to identify "later" P6-family chips
407 * like the Pentium M and Core. First, the "older" P6-based stuff
408 * (loosely defined as "pre-Pentium-4"):
409 * P6, PII, Mobile PII, PII Xeon, PIII, Mobile PIII, PIII Xeon
410 */
411
412 #define IS_LEGACY_P6(cpi) ( \
413 cpi->cpi_family == 6 && \
414 (cpi->cpi_model == 1 || \
415 cpi->cpi_model == 3 || \
416 cpi->cpi_model == 5 || \
417 cpi->cpi_model == 6 || \
418 cpi->cpi_model == 7 || \
419 cpi->cpi_model == 8 || \
420 cpi->cpi_model == 0xA || \
421 cpi->cpi_model == 0xB) \
422 )
423
424 /* A "new F6" is everything with family 6 that's not the above */
425 #define IS_NEW_F6(cpi) ((cpi->cpi_family == 6) && !IS_LEGACY_P6(cpi))
426
427 /* Extended family/model support */
428 #define IS_EXTENDED_MODEL_INTEL(cpi) (cpi->cpi_family == 0x6 || \
429 cpi->cpi_family >= 0xf)
430
431 /*
432 * Info for monitor/mwait idle loop.
433 *
434 * See cpuid section of "Intel 64 and IA-32 Architectures Software Developer's
435 * Manual Volume 2A: Instruction Set Reference, A-M" #25366-022US, November
436 * 2006.
437 * See MONITOR/MWAIT section of "AMD64 Architecture Programmer's Manual
438 * Documentation Updates" #33633, Rev 2.05, December 2006.
439 */
440 #define MWAIT_SUPPORT (0x00000001) /* mwait supported */
441 #define MWAIT_EXTENSIONS (0x00000002) /* extenstion supported */
442 #define MWAIT_ECX_INT_ENABLE (0x00000004) /* ecx 1 extension supported */
443 #define MWAIT_SUPPORTED(cpi) ((cpi)->cpi_std[1].cp_ecx & CPUID_INTC_ECX_MON)
444 #define MWAIT_INT_ENABLE(cpi) ((cpi)->cpi_std[5].cp_ecx & 0x2)
445 #define MWAIT_EXTENSION(cpi) ((cpi)->cpi_std[5].cp_ecx & 0x1)
446 #define MWAIT_SIZE_MIN(cpi) BITX((cpi)->cpi_std[5].cp_eax, 15, 0)
447 #define MWAIT_SIZE_MAX(cpi) BITX((cpi)->cpi_std[5].cp_ebx, 15, 0)
448 /*
449 * Number of sub-cstates for a given c-state.
450 */
451 #define MWAIT_NUM_SUBC_STATES(cpi, c_state) \
452 BITX((cpi)->cpi_std[5].cp_edx, c_state + 3, c_state)
453
454 /*
455 * XSAVE leaf 0xD enumeration
456 */
457 #define CPUID_LEAFD_2_YMM_OFFSET 576
458 #define CPUID_LEAFD_2_YMM_SIZE 256
459
460 /*
461 * Functions we consune from cpuid_subr.c; don't publish these in a header
462 * file to try and keep people using the expected cpuid_* interfaces.
463 */
464 extern uint32_t _cpuid_skt(uint_t, uint_t, uint_t, uint_t);
465 extern const char *_cpuid_sktstr(uint_t, uint_t, uint_t, uint_t);
466 extern uint32_t _cpuid_chiprev(uint_t, uint_t, uint_t, uint_t);
467 extern const char *_cpuid_chiprevstr(uint_t, uint_t, uint_t, uint_t);
468 extern uint_t _cpuid_vendorstr_to_vendorcode(char *);
469
470 /*
471 * Apply up various platform-dependent restrictions where the
472 * underlying platform restrictions mean the CPU can be marked
473 * as less capable than its cpuid instruction would imply.
474 */
475 #if defined(__xpv)
476 static void
477 platform_cpuid_mangle(uint_t vendor, uint32_t eax, struct cpuid_regs *cp)
478 {
479 switch (eax) {
480 case 1: {
481 uint32_t mcamask = DOMAIN_IS_INITDOMAIN(xen_info) ?
482 0 : CPUID_INTC_EDX_MCA;
483 cp->cp_edx &=
484 ~(mcamask |
485 CPUID_INTC_EDX_PSE |
486 CPUID_INTC_EDX_VME | CPUID_INTC_EDX_DE |
487 CPUID_INTC_EDX_SEP | CPUID_INTC_EDX_MTRR |
488 CPUID_INTC_EDX_PGE | CPUID_INTC_EDX_PAT |
489 CPUID_AMD_EDX_SYSC | CPUID_INTC_EDX_SEP |
490 CPUID_INTC_EDX_PSE36 | CPUID_INTC_EDX_HTT);
491 break;
492 }
493
494 case 0x80000001:
495 cp->cp_edx &=
496 ~(CPUID_AMD_EDX_PSE |
497 CPUID_INTC_EDX_VME | CPUID_INTC_EDX_DE |
498 CPUID_AMD_EDX_MTRR | CPUID_AMD_EDX_PGE |
499 CPUID_AMD_EDX_PAT | CPUID_AMD_EDX_PSE36 |
500 CPUID_AMD_EDX_SYSC | CPUID_INTC_EDX_SEP |
501 CPUID_AMD_EDX_TSCP);
502 cp->cp_ecx &= ~CPUID_AMD_ECX_CMP_LGCY;
503 break;
504 default:
505 break;
506 }
507
508 switch (vendor) {
509 case X86_VENDOR_Intel:
510 switch (eax) {
511 case 4:
512 /*
513 * Zero out the (ncores-per-chip - 1) field
514 */
515 cp->cp_eax &= 0x03fffffff;
516 break;
517 default:
518 break;
519 }
520 break;
521 case X86_VENDOR_AMD:
522 switch (eax) {
523
524 case 0x80000001:
525 cp->cp_ecx &= ~CPUID_AMD_ECX_CR8D;
526 break;
527
528 case 0x80000008:
529 /*
530 * Zero out the (ncores-per-chip - 1) field
531 */
532 cp->cp_ecx &= 0xffffff00;
533 break;
534 default:
535 break;
536 }
537 break;
538 default:
539 break;
540 }
541 }
542 #else
543 #define platform_cpuid_mangle(vendor, eax, cp) /* nothing */
544 #endif
545
546 /*
547 * Some undocumented ways of patching the results of the cpuid
548 * instruction to permit running Solaris 10 on future cpus that
549 * we don't currently support. Could be set to non-zero values
550 * via settings in eeprom.
551 */
552
553 uint32_t cpuid_feature_ecx_include;
554 uint32_t cpuid_feature_ecx_exclude;
555 uint32_t cpuid_feature_edx_include;
556 uint32_t cpuid_feature_edx_exclude;
557
558 /*
559 * Allocate space for mcpu_cpi in the machcpu structure for all non-boot CPUs.
560 */
561 void
562 cpuid_alloc_space(cpu_t *cpu)
563 {
564 /*
565 * By convention, cpu0 is the boot cpu, which is set up
566 * before memory allocation is available. All other cpus get
567 * their cpuid_info struct allocated here.
568 */
569 ASSERT(cpu->cpu_id != 0);
570 ASSERT(cpu->cpu_m.mcpu_cpi == NULL);
571 cpu->cpu_m.mcpu_cpi =
572 kmem_zalloc(sizeof (*cpu->cpu_m.mcpu_cpi), KM_SLEEP);
573 }
574
575 void
576 cpuid_free_space(cpu_t *cpu)
577 {
578 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
579 int i;
580
581 ASSERT(cpi != NULL);
582 ASSERT(cpi != &cpuid_info0);
583
584 /*
585 * Free up any function 4 related dynamic storage
586 */
587 for (i = 1; i < cpi->cpi_std_4_size; i++)
588 kmem_free(cpi->cpi_std_4[i], sizeof (struct cpuid_regs));
589 if (cpi->cpi_std_4_size > 0)
590 kmem_free(cpi->cpi_std_4,
591 cpi->cpi_std_4_size * sizeof (struct cpuid_regs *));
592
593 kmem_free(cpi, sizeof (*cpi));
594 cpu->cpu_m.mcpu_cpi = NULL;
595 }
596
597 #if !defined(__xpv)
598
599 /*
600 * Determine the type of the underlying platform. This is used to customize
601 * initialization of various subsystems (e.g. TSC). determine_platform() must
602 * only ever be called once to prevent two processors from seeing different
603 * values of platform_type, it must be called before cpuid_pass1(), the
604 * earliest consumer to execute.
605 */
606 void
607 determine_platform(void)
608 {
609 struct cpuid_regs cp;
610 char *xen_str;
611 uint32_t xen_signature[4], base;
612
613 ASSERT(platform_type == -1);
614
615 platform_type = HW_NATIVE;
616
617 if (!enable_platform_detection)
618 return;
619
620 /*
621 * In a fully virtualized domain, Xen's pseudo-cpuid function
622 * returns a string representing the Xen signature in %ebx, %ecx,
623 * and %edx. %eax contains the maximum supported cpuid function.
624 * We need at least a (base + 2) leaf value to do what we want
625 * to do. Try different base values, since the hypervisor might
626 * use a different one depending on whether hyper-v emulation
627 * is switched on by default or not.
628 */
629 for (base = 0x40000000; base < 0x40010000; base += 0x100) {
630 cp.cp_eax = base;
631 (void) __cpuid_insn(&cp);
632 xen_signature[0] = cp.cp_ebx;
633 xen_signature[1] = cp.cp_ecx;
634 xen_signature[2] = cp.cp_edx;
635 xen_signature[3] = 0;
636 xen_str = (char *)xen_signature;
637 if (strcmp("XenVMMXenVMM", xen_str) == 0 &&
638 cp.cp_eax >= (base + 2)) {
639 platform_type = HW_XEN_HVM;
640 return;
641 }
642 }
643
644 if (vmware_platform()) /* running under vmware hypervisor? */
645 platform_type = HW_VMWARE;
646 }
647
648 int
649 get_hwenv(void)
650 {
651 ASSERT(platform_type != -1);
652 return (platform_type);
653 }
654
655 int
656 is_controldom(void)
657 {
658 return (0);
659 }
660
661 #else
662
663 int
664 get_hwenv(void)
665 {
666 return (HW_XEN_PV);
667 }
668
669 int
670 is_controldom(void)
671 {
672 return (DOMAIN_IS_INITDOMAIN(xen_info));
673 }
674
675 #endif /* __xpv */
676
677 static void
678 cpuid_intel_getids(cpu_t *cpu, void *feature)
679 {
680 uint_t i;
681 uint_t chipid_shift = 0;
682 uint_t coreid_shift = 0;
683 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
684
685 for (i = 1; i < cpi->cpi_ncpu_per_chip; i <<= 1)
686 chipid_shift++;
687
688 cpi->cpi_chipid = cpi->cpi_apicid >> chipid_shift;
689 cpi->cpi_clogid = cpi->cpi_apicid & ((1 << chipid_shift) - 1);
690
691 if (is_x86_feature(feature, X86FSET_CMP)) {
692 /*
693 * Multi-core (and possibly multi-threaded)
694 * processors.
695 */
696 uint_t ncpu_per_core;
697 if (cpi->cpi_ncore_per_chip == 1)
698 ncpu_per_core = cpi->cpi_ncpu_per_chip;
699 else if (cpi->cpi_ncore_per_chip > 1)
700 ncpu_per_core = cpi->cpi_ncpu_per_chip /
701 cpi->cpi_ncore_per_chip;
702 /*
703 * 8bit APIC IDs on dual core Pentiums
704 * look like this:
705 *
706 * +-----------------------+------+------+
707 * | Physical Package ID | MC | HT |
708 * +-----------------------+------+------+
709 * <------- chipid -------->
710 * <------- coreid --------------->
711 * <--- clogid -->
712 * <------>
713 * pkgcoreid
714 *
715 * Where the number of bits necessary to
716 * represent MC and HT fields together equals
717 * to the minimum number of bits necessary to
718 * store the value of cpi->cpi_ncpu_per_chip.
719 * Of those bits, the MC part uses the number
720 * of bits necessary to store the value of
721 * cpi->cpi_ncore_per_chip.
722 */
723 for (i = 1; i < ncpu_per_core; i <<= 1)
724 coreid_shift++;
725 cpi->cpi_coreid = cpi->cpi_apicid >> coreid_shift;
726 cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
727 } else if (is_x86_feature(feature, X86FSET_HTT)) {
728 /*
729 * Single-core multi-threaded processors.
730 */
731 cpi->cpi_coreid = cpi->cpi_chipid;
732 cpi->cpi_pkgcoreid = 0;
733 }
734 cpi->cpi_procnodeid = cpi->cpi_chipid;
735 cpi->cpi_compunitid = cpi->cpi_coreid;
736 }
737
738 static void
739 cpuid_amd_getids(cpu_t *cpu)
740 {
741 int i, first_half, coreidsz;
742 uint32_t nb_caps_reg;
743 uint_t node2_1;
744 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
745 struct cpuid_regs *cp;
746
747 /*
748 * AMD CMP chips currently have a single thread per core.
749 *
750 * Since no two cpus share a core we must assign a distinct coreid
751 * per cpu, and we do this by using the cpu_id. This scheme does not,
752 * however, guarantee that sibling cores of a chip will have sequential
753 * coreids starting at a multiple of the number of cores per chip -
754 * that is usually the case, but if the ACPI MADT table is presented
755 * in a different order then we need to perform a few more gymnastics
756 * for the pkgcoreid.
757 *
758 * All processors in the system have the same number of enabled
759 * cores. Cores within a processor are always numbered sequentially
760 * from 0 regardless of how many or which are disabled, and there
761 * is no way for operating system to discover the real core id when some
762 * are disabled.
763 *
764 * In family 0x15, the cores come in pairs called compute units. They
765 * share L1I and L2 caches and the FPU. Enumeration of this features is
766 * simplified by the new topology extensions CPUID leaf, indicated by the
767 * X86 feature X86FSET_TOPOEXT.
768 */
769
770 cpi->cpi_coreid = cpu->cpu_id;
771 cpi->cpi_compunitid = cpu->cpu_id;
772
773 if (cpi->cpi_xmaxeax >= 0x80000008) {
774
775 coreidsz = BITX((cpi)->cpi_extd[8].cp_ecx, 15, 12);
776
777 /*
778 * In AMD parlance chip is really a node while Solaris
779 * sees chip as equivalent to socket/package.
780 */
781 cpi->cpi_ncore_per_chip =
782 BITX((cpi)->cpi_extd[8].cp_ecx, 7, 0) + 1;
783 if (coreidsz == 0) {
784 /* Use legacy method */
785 for (i = 1; i < cpi->cpi_ncore_per_chip; i <<= 1)
786 coreidsz++;
787 if (coreidsz == 0)
788 coreidsz = 1;
789 }
790 } else {
791 /* Assume single-core part */
792 cpi->cpi_ncore_per_chip = 1;
793 coreidsz = 1;
794 }
795
796 cpi->cpi_clogid = cpi->cpi_pkgcoreid =
797 cpi->cpi_apicid & ((1<<coreidsz) - 1);
798 cpi->cpi_ncpu_per_chip = cpi->cpi_ncore_per_chip;
799
800 /* Get node ID, compute unit ID */
801 if (is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
802 cpi->cpi_xmaxeax >= 0x8000001e) {
803 cp = &cpi->cpi_extd[0x1e];
804 cp->cp_eax = 0x8000001e;
805 (void) __cpuid_insn(cp);
806
807 cpi->cpi_procnodes_per_pkg = BITX(cp->cp_ecx, 10, 8) + 1;
808 cpi->cpi_procnodeid = BITX(cp->cp_ecx, 7, 0);
809 cpi->cpi_cores_per_compunit = BITX(cp->cp_ebx, 15, 8) + 1;
810 cpi->cpi_compunitid = BITX(cp->cp_ebx, 7, 0);
811 } else if (cpi->cpi_family == 0xf || cpi->cpi_family >= 0x11) {
812 cpi->cpi_procnodeid = (cpi->cpi_apicid >> coreidsz) & 7;
813 cpi->cpi_chipid = cpi->cpi_procnodeid;
814 } else if (cpi->cpi_family == 0x10) {
815 /*
816 * See if we are a multi-node processor.
817 * All processors in the system have the same number of nodes
818 */
819 nb_caps_reg = pci_getl_func(0, 24, 3, 0xe8);
820 if ((cpi->cpi_model < 8) || BITX(nb_caps_reg, 29, 29) == 0) {
821 /* Single-node */
822 cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 5,
823 coreidsz);
824 cpi->cpi_chipid = cpi->cpi_procnodeid;
825 } else {
826
827 /*
828 * Multi-node revision D (2 nodes per package
829 * are supported)
830 */
831 cpi->cpi_procnodes_per_pkg = 2;
832
833 first_half = (cpi->cpi_pkgcoreid <=
834 (cpi->cpi_ncore_per_chip/2 - 1));
835
836 if (cpi->cpi_apicid == cpi->cpi_pkgcoreid) {
837 /* We are BSP */
838 cpi->cpi_procnodeid = (first_half ? 0 : 1);
839 cpi->cpi_chipid = cpi->cpi_procnodeid >> 1;
840 } else {
841
842 /* We are AP */
843 /* NodeId[2:1] bits to use for reading F3xe8 */
844 node2_1 = BITX(cpi->cpi_apicid, 5, 4) << 1;
845
846 nb_caps_reg =
847 pci_getl_func(0, 24 + node2_1, 3, 0xe8);
848
849 /*
850 * Check IntNodeNum bit (31:30, but bit 31 is
851 * always 0 on dual-node processors)
852 */
853 if (BITX(nb_caps_reg, 30, 30) == 0)
854 cpi->cpi_procnodeid = node2_1 +
855 !first_half;
856 else
857 cpi->cpi_procnodeid = node2_1 +
858 first_half;
859
860 cpi->cpi_chipid = cpi->cpi_procnodeid >> 1;
861 }
862 }
863 } else {
864 cpi->cpi_procnodeid = 0;
865 cpi->cpi_chipid = cpi->cpi_procnodeid;
866 }
867 }
868
869 /*
870 * Setup XFeature_Enabled_Mask register. Required by xsave feature.
871 */
872 void
873 setup_xfem(void)
874 {
875 uint64_t flags = XFEATURE_LEGACY_FP;
876
877 ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
878
879 if (is_x86_feature(x86_featureset, X86FSET_SSE))
880 flags |= XFEATURE_SSE;
881
882 if (is_x86_feature(x86_featureset, X86FSET_AVX))
883 flags |= XFEATURE_AVX;
884
885 set_xcr(XFEATURE_ENABLED_MASK, flags);
886
887 xsave_bv_all = flags;
888 }
889
890 void
891 cpuid_pass1(cpu_t *cpu, uchar_t *featureset)
892 {
893 uint32_t mask_ecx, mask_edx;
894 struct cpuid_info *cpi;
895 struct cpuid_regs *cp;
896 int xcpuid;
897 #if !defined(__xpv)
898 extern int idle_cpu_prefer_mwait;
899 #endif
900
901 /*
902 * Space statically allocated for BSP, ensure pointer is set
903 */
904 if (cpu->cpu_id == 0) {
905 if (cpu->cpu_m.mcpu_cpi == NULL)
906 cpu->cpu_m.mcpu_cpi = &cpuid_info0;
907 }
908
909 add_x86_feature(featureset, X86FSET_CPUID);
910
911 cpi = cpu->cpu_m.mcpu_cpi;
912 ASSERT(cpi != NULL);
913 cp = &cpi->cpi_std[0];
914 cp->cp_eax = 0;
915 cpi->cpi_maxeax = __cpuid_insn(cp);
916 {
917 uint32_t *iptr = (uint32_t *)cpi->cpi_vendorstr;
918 *iptr++ = cp->cp_ebx;
919 *iptr++ = cp->cp_edx;
920 *iptr++ = cp->cp_ecx;
921 *(char *)&cpi->cpi_vendorstr[12] = '\0';
922 }
923
924 cpi->cpi_vendor = _cpuid_vendorstr_to_vendorcode(cpi->cpi_vendorstr);
925 x86_vendor = cpi->cpi_vendor; /* for compatibility */
926
927 /*
928 * Limit the range in case of weird hardware
929 */
930 if (cpi->cpi_maxeax > CPI_MAXEAX_MAX)
931 cpi->cpi_maxeax = CPI_MAXEAX_MAX;
932 if (cpi->cpi_maxeax < 1)
933 goto pass1_done;
934
935 cp = &cpi->cpi_std[1];
936 cp->cp_eax = 1;
937 (void) __cpuid_insn(cp);
938
939 /*
940 * Extract identifying constants for easy access.
941 */
942 cpi->cpi_model = CPI_MODEL(cpi);
943 cpi->cpi_family = CPI_FAMILY(cpi);
944
945 if (cpi->cpi_family == 0xf)
946 cpi->cpi_family += CPI_FAMILY_XTD(cpi);
947
948 /*
949 * Beware: AMD uses "extended model" iff base *FAMILY* == 0xf.
950 * Intel, and presumably everyone else, uses model == 0xf, as
951 * one would expect (max value means possible overflow). Sigh.
952 */
953
954 switch (cpi->cpi_vendor) {
955 case X86_VENDOR_Intel:
956 if (IS_EXTENDED_MODEL_INTEL(cpi))
957 cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
958 break;
959 case X86_VENDOR_AMD:
960 if (CPI_FAMILY(cpi) == 0xf)
961 cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
962 break;
963 default:
964 if (cpi->cpi_model == 0xf)
965 cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
966 break;
967 }
968
969 cpi->cpi_step = CPI_STEP(cpi);
970 cpi->cpi_brandid = CPI_BRANDID(cpi);
971
972 /*
973 * *default* assumptions:
974 * - believe %edx feature word
975 * - ignore %ecx feature word
976 * - 32-bit virtual and physical addressing
977 */
978 mask_edx = 0xffffffff;
979 mask_ecx = 0;
980
981 cpi->cpi_pabits = cpi->cpi_vabits = 32;
982
983 switch (cpi->cpi_vendor) {
984 case X86_VENDOR_Intel:
985 if (cpi->cpi_family == 5)
986 x86_type = X86_TYPE_P5;
987 else if (IS_LEGACY_P6(cpi)) {
988 x86_type = X86_TYPE_P6;
989 pentiumpro_bug4046376 = 1;
990 pentiumpro_bug4064495 = 1;
991 /*
992 * Clear the SEP bit when it was set erroneously
993 */
994 if (cpi->cpi_model < 3 && cpi->cpi_step < 3)
995 cp->cp_edx &= ~CPUID_INTC_EDX_SEP;
996 } else if (IS_NEW_F6(cpi) || cpi->cpi_family == 0xf) {
997 x86_type = X86_TYPE_P4;
998 /*
999 * We don't currently depend on any of the %ecx
1000 * features until Prescott, so we'll only check
1001 * this from P4 onwards. We might want to revisit
1002 * that idea later.
1003 */
1004 mask_ecx = 0xffffffff;
1005 } else if (cpi->cpi_family > 0xf)
1006 mask_ecx = 0xffffffff;
1007 /*
1008 * We don't support MONITOR/MWAIT if leaf 5 is not available
1009 * to obtain the monitor linesize.
1010 */
1011 if (cpi->cpi_maxeax < 5)
1012 mask_ecx &= ~CPUID_INTC_ECX_MON;
1013 break;
1014 case X86_VENDOR_IntelClone:
1015 default:
1016 break;
1017 case X86_VENDOR_AMD:
1018 #if defined(OPTERON_ERRATUM_108)
1019 if (cpi->cpi_family == 0xf && cpi->cpi_model == 0xe) {
1020 cp->cp_eax = (0xf0f & cp->cp_eax) | 0xc0;
1021 cpi->cpi_model = 0xc;
1022 } else
1023 #endif
1024 if (cpi->cpi_family == 5) {
1025 /*
1026 * AMD K5 and K6
1027 *
1028 * These CPUs have an incomplete implementation
1029 * of MCA/MCE which we mask away.
1030 */
1031 mask_edx &= ~(CPUID_INTC_EDX_MCE | CPUID_INTC_EDX_MCA);
1032
1033 /*
1034 * Model 0 uses the wrong (APIC) bit
1035 * to indicate PGE. Fix it here.
1036 */
1037 if (cpi->cpi_model == 0) {
1038 if (cp->cp_edx & 0x200) {
1039 cp->cp_edx &= ~0x200;
1040 cp->cp_edx |= CPUID_INTC_EDX_PGE;
1041 }
1042 }
1043
1044 /*
1045 * Early models had problems w/ MMX; disable.
1046 */
1047 if (cpi->cpi_model < 6)
1048 mask_edx &= ~CPUID_INTC_EDX_MMX;
1049 }
1050
1051 /*
1052 * For newer families, SSE3 and CX16, at least, are valid;
1053 * enable all
1054 */
1055 if (cpi->cpi_family >= 0xf)
1056 mask_ecx = 0xffffffff;
1057 /*
1058 * We don't support MONITOR/MWAIT if leaf 5 is not available
1059 * to obtain the monitor linesize.
1060 */
1061 if (cpi->cpi_maxeax < 5)
1062 mask_ecx &= ~CPUID_INTC_ECX_MON;
1063
1064 #if !defined(__xpv)
1065 /*
1066 * Do not use MONITOR/MWAIT to halt in the idle loop on any AMD
1067 * processors. AMD does not intend MWAIT to be used in the cpu
1068 * idle loop on current and future processors. 10h and future
1069 * AMD processors use more power in MWAIT than HLT.
1070 * Pre-family-10h Opterons do not have the MWAIT instruction.
1071 */
1072 idle_cpu_prefer_mwait = 0;
1073 #endif
1074
1075 break;
1076 case X86_VENDOR_TM:
1077 /*
1078 * workaround the NT workaround in CMS 4.1
1079 */
1080 if (cpi->cpi_family == 5 && cpi->cpi_model == 4 &&
1081 (cpi->cpi_step == 2 || cpi->cpi_step == 3))
1082 cp->cp_edx |= CPUID_INTC_EDX_CX8;
1083 break;
1084 case X86_VENDOR_Centaur:
1085 /*
1086 * workaround the NT workarounds again
1087 */
1088 if (cpi->cpi_family == 6)
1089 cp->cp_edx |= CPUID_INTC_EDX_CX8;
1090 break;
1091 case X86_VENDOR_Cyrix:
1092 /*
1093 * We rely heavily on the probing in locore
1094 * to actually figure out what parts, if any,
1095 * of the Cyrix cpuid instruction to believe.
1096 */
1097 switch (x86_type) {
1098 case X86_TYPE_CYRIX_486:
1099 mask_edx = 0;
1100 break;
1101 case X86_TYPE_CYRIX_6x86:
1102 mask_edx = 0;
1103 break;
1104 case X86_TYPE_CYRIX_6x86L:
1105 mask_edx =
1106 CPUID_INTC_EDX_DE |
1107 CPUID_INTC_EDX_CX8;
1108 break;
1109 case X86_TYPE_CYRIX_6x86MX:
1110 mask_edx =
1111 CPUID_INTC_EDX_DE |
1112 CPUID_INTC_EDX_MSR |
1113 CPUID_INTC_EDX_CX8 |
1114 CPUID_INTC_EDX_PGE |
1115 CPUID_INTC_EDX_CMOV |
1116 CPUID_INTC_EDX_MMX;
1117 break;
1118 case X86_TYPE_CYRIX_GXm:
1119 mask_edx =
1120 CPUID_INTC_EDX_MSR |
1121 CPUID_INTC_EDX_CX8 |
1122 CPUID_INTC_EDX_CMOV |
1123 CPUID_INTC_EDX_MMX;
1124 break;
1125 case X86_TYPE_CYRIX_MediaGX:
1126 break;
1127 case X86_TYPE_CYRIX_MII:
1128 case X86_TYPE_VIA_CYRIX_III:
1129 mask_edx =
1130 CPUID_INTC_EDX_DE |
1131 CPUID_INTC_EDX_TSC |
1132 CPUID_INTC_EDX_MSR |
1133 CPUID_INTC_EDX_CX8 |
1134 CPUID_INTC_EDX_PGE |
1135 CPUID_INTC_EDX_CMOV |
1136 CPUID_INTC_EDX_MMX;
1137 break;
1138 default:
1139 break;
1140 }
1141 break;
1142 }
1143
1144 #if defined(__xpv)
1145 /*
1146 * Do not support MONITOR/MWAIT under a hypervisor
1147 */
1148 mask_ecx &= ~CPUID_INTC_ECX_MON;
1149 /*
1150 * Do not support XSAVE under a hypervisor for now
1151 */
1152 xsave_force_disable = B_TRUE;
1153
1154 #endif /* __xpv */
1155
1156 if (xsave_force_disable) {
1157 mask_ecx &= ~CPUID_INTC_ECX_XSAVE;
1158 mask_ecx &= ~CPUID_INTC_ECX_AVX;
1159 }
1160
1161 /*
1162 * Now we've figured out the masks that determine
1163 * which bits we choose to believe, apply the masks
1164 * to the feature words, then map the kernel's view
1165 * of these feature words into its feature word.
1166 */
1167 cp->cp_edx &= mask_edx;
1168 cp->cp_ecx &= mask_ecx;
1169
1170 /*
1171 * apply any platform restrictions (we don't call this
1172 * immediately after __cpuid_insn here, because we need the
1173 * workarounds applied above first)
1174 */
1175 platform_cpuid_mangle(cpi->cpi_vendor, 1, cp);
1176
1177 /*
1178 * fold in overrides from the "eeprom" mechanism
1179 */
1180 cp->cp_edx |= cpuid_feature_edx_include;
1181 cp->cp_edx &= ~cpuid_feature_edx_exclude;
1182
1183 cp->cp_ecx |= cpuid_feature_ecx_include;
1184 cp->cp_ecx &= ~cpuid_feature_ecx_exclude;
1185
1186 if (cp->cp_edx & CPUID_INTC_EDX_PSE) {
1187 add_x86_feature(featureset, X86FSET_LARGEPAGE);
1188 }
1189 if (cp->cp_edx & CPUID_INTC_EDX_TSC) {
1190 add_x86_feature(featureset, X86FSET_TSC);
1191 }
1192 if (cp->cp_edx & CPUID_INTC_EDX_MSR) {
1193 add_x86_feature(featureset, X86FSET_MSR);
1194 }
1195 if (cp->cp_edx & CPUID_INTC_EDX_MTRR) {
1196 add_x86_feature(featureset, X86FSET_MTRR);
1197 }
1198 if (cp->cp_edx & CPUID_INTC_EDX_PGE) {
1199 add_x86_feature(featureset, X86FSET_PGE);
1200 }
1201 if (cp->cp_edx & CPUID_INTC_EDX_CMOV) {
1202 add_x86_feature(featureset, X86FSET_CMOV);
1203 }
1204 if (cp->cp_edx & CPUID_INTC_EDX_MMX) {
1205 add_x86_feature(featureset, X86FSET_MMX);
1206 }
1207 if ((cp->cp_edx & CPUID_INTC_EDX_MCE) != 0 &&
1208 (cp->cp_edx & CPUID_INTC_EDX_MCA) != 0) {
1209 add_x86_feature(featureset, X86FSET_MCA);
1210 }
1211 if (cp->cp_edx & CPUID_INTC_EDX_PAE) {
1212 add_x86_feature(featureset, X86FSET_PAE);
1213 }
1214 if (cp->cp_edx & CPUID_INTC_EDX_CX8) {
1215 add_x86_feature(featureset, X86FSET_CX8);
1216 }
1217 if (cp->cp_ecx & CPUID_INTC_ECX_CX16) {
1218 add_x86_feature(featureset, X86FSET_CX16);
1219 }
1220 if (cp->cp_edx & CPUID_INTC_EDX_PAT) {
1221 add_x86_feature(featureset, X86FSET_PAT);
1222 }
1223 if (cp->cp_edx & CPUID_INTC_EDX_SEP) {
1224 add_x86_feature(featureset, X86FSET_SEP);
1225 }
1226 if (cp->cp_edx & CPUID_INTC_EDX_FXSR) {
1227 /*
1228 * In our implementation, fxsave/fxrstor
1229 * are prerequisites before we'll even
1230 * try and do SSE things.
1231 */
1232 if (cp->cp_edx & CPUID_INTC_EDX_SSE) {
1233 add_x86_feature(featureset, X86FSET_SSE);
1234 }
1235 if (cp->cp_edx & CPUID_INTC_EDX_SSE2) {
1236 add_x86_feature(featureset, X86FSET_SSE2);
1237 }
1238 if (cp->cp_ecx & CPUID_INTC_ECX_SSE3) {
1239 add_x86_feature(featureset, X86FSET_SSE3);
1240 }
1241 if (cp->cp_ecx & CPUID_INTC_ECX_SSSE3) {
1242 add_x86_feature(featureset, X86FSET_SSSE3);
1243 }
1244 if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_1) {
1245 add_x86_feature(featureset, X86FSET_SSE4_1);
1246 }
1247 if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_2) {
1248 add_x86_feature(featureset, X86FSET_SSE4_2);
1249 }
1250 if (cp->cp_ecx & CPUID_INTC_ECX_AES) {
1251 add_x86_feature(featureset, X86FSET_AES);
1252 }
1253 if (cp->cp_ecx & CPUID_INTC_ECX_PCLMULQDQ) {
1254 add_x86_feature(featureset, X86FSET_PCLMULQDQ);
1255 }
1256
1257 if (cp->cp_ecx & CPUID_INTC_ECX_XSAVE) {
1258 add_x86_feature(featureset, X86FSET_XSAVE);
1259 /* We only test AVX when there is XSAVE */
1260 if (cp->cp_ecx & CPUID_INTC_ECX_AVX) {
1261 add_x86_feature(featureset,
1262 X86FSET_AVX);
1263 }
1264 }
1265 }
1266 if (cp->cp_edx & CPUID_INTC_EDX_DE) {
1267 add_x86_feature(featureset, X86FSET_DE);
1268 }
1269 #if !defined(__xpv)
1270 if (cp->cp_ecx & CPUID_INTC_ECX_MON) {
1271
1272 /*
1273 * We require the CLFLUSH instruction for erratum workaround
1274 * to use MONITOR/MWAIT.
1275 */
1276 if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
1277 cpi->cpi_mwait.support |= MWAIT_SUPPORT;
1278 add_x86_feature(featureset, X86FSET_MWAIT);
1279 } else {
1280 extern int idle_cpu_assert_cflush_monitor;
1281
1282 /*
1283 * All processors we are aware of which have
1284 * MONITOR/MWAIT also have CLFLUSH.
1285 */
1286 if (idle_cpu_assert_cflush_monitor) {
1287 ASSERT((cp->cp_ecx & CPUID_INTC_ECX_MON) &&
1288 (cp->cp_edx & CPUID_INTC_EDX_CLFSH));
1289 }
1290 }
1291 }
1292 #endif /* __xpv */
1293
1294 if (cp->cp_ecx & CPUID_INTC_ECX_VMX) {
1295 add_x86_feature(featureset, X86FSET_VMX);
1296 }
1297
1298 /*
1299 * Only need it first time, rest of the cpus would follow suit.
1300 * we only capture this for the bootcpu.
1301 */
1302 if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
1303 add_x86_feature(featureset, X86FSET_CLFSH);
1304 x86_clflush_size = (BITX(cp->cp_ebx, 15, 8) * 8);
1305 }
1306 if (is_x86_feature(featureset, X86FSET_PAE))
1307 cpi->cpi_pabits = 36;
1308
1309 /*
1310 * Hyperthreading configuration is slightly tricky on Intel
1311 * and pure clones, and even trickier on AMD.
1312 *
1313 * (AMD chose to set the HTT bit on their CMP processors,
1314 * even though they're not actually hyperthreaded. Thus it
1315 * takes a bit more work to figure out what's really going
1316 * on ... see the handling of the CMP_LGCY bit below)
1317 */
1318 if (cp->cp_edx & CPUID_INTC_EDX_HTT) {
1319 cpi->cpi_ncpu_per_chip = CPI_CPU_COUNT(cpi);
1320 if (cpi->cpi_ncpu_per_chip > 1)
1321 add_x86_feature(featureset, X86FSET_HTT);
1322 } else {
1323 cpi->cpi_ncpu_per_chip = 1;
1324 }
1325
1326 /*
1327 * Work on the "extended" feature information, doing
1328 * some basic initialization for cpuid_pass2()
1329 */
1330 xcpuid = 0;
1331 switch (cpi->cpi_vendor) {
1332 case X86_VENDOR_Intel:
1333 if (IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf)
1334 xcpuid++;
1335 break;
1336 case X86_VENDOR_AMD:
1337 if (cpi->cpi_family > 5 ||
1338 (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
1339 xcpuid++;
1340 break;
1341 case X86_VENDOR_Cyrix:
1342 /*
1343 * Only these Cyrix CPUs are -known- to support
1344 * extended cpuid operations.
1345 */
1346 if (x86_type == X86_TYPE_VIA_CYRIX_III ||
1347 x86_type == X86_TYPE_CYRIX_GXm)
1348 xcpuid++;
1349 break;
1350 case X86_VENDOR_Centaur:
1351 case X86_VENDOR_TM:
1352 default:
1353 xcpuid++;
1354 break;
1355 }
1356
1357 if (xcpuid) {
1358 cp = &cpi->cpi_extd[0];
1359 cp->cp_eax = 0x80000000;
1360 cpi->cpi_xmaxeax = __cpuid_insn(cp);
1361 }
1362
1363 if (cpi->cpi_xmaxeax & 0x80000000) {
1364
1365 if (cpi->cpi_xmaxeax > CPI_XMAXEAX_MAX)
1366 cpi->cpi_xmaxeax = CPI_XMAXEAX_MAX;
1367
1368 switch (cpi->cpi_vendor) {
1369 case X86_VENDOR_Intel:
1370 case X86_VENDOR_AMD:
1371 if (cpi->cpi_xmaxeax < 0x80000001)
1372 break;
1373 cp = &cpi->cpi_extd[1];
1374 cp->cp_eax = 0x80000001;
1375 (void) __cpuid_insn(cp);
1376
1377 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
1378 cpi->cpi_family == 5 &&
1379 cpi->cpi_model == 6 &&
1380 cpi->cpi_step == 6) {
1381 /*
1382 * K6 model 6 uses bit 10 to indicate SYSC
1383 * Later models use bit 11. Fix it here.
1384 */
1385 if (cp->cp_edx & 0x400) {
1386 cp->cp_edx &= ~0x400;
1387 cp->cp_edx |= CPUID_AMD_EDX_SYSC;
1388 }
1389 }
1390
1391 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000001, cp);
1392
1393 /*
1394 * Compute the additions to the kernel's feature word.
1395 */
1396 if (cp->cp_edx & CPUID_AMD_EDX_NX) {
1397 add_x86_feature(featureset, X86FSET_NX);
1398 }
1399
1400 /*
1401 * Regardless whether or not we boot 64-bit,
1402 * we should have a way to identify whether
1403 * the CPU is capable of running 64-bit.
1404 */
1405 if (cp->cp_edx & CPUID_AMD_EDX_LM) {
1406 add_x86_feature(featureset, X86FSET_64);
1407 }
1408
1409 #if defined(__amd64)
1410 /* 1 GB large page - enable only for 64 bit kernel */
1411 if (cp->cp_edx & CPUID_AMD_EDX_1GPG) {
1412 add_x86_feature(featureset, X86FSET_1GPG);
1413 }
1414 #endif
1415
1416 if ((cpi->cpi_vendor == X86_VENDOR_AMD) &&
1417 (cpi->cpi_std[1].cp_edx & CPUID_INTC_EDX_FXSR) &&
1418 (cp->cp_ecx & CPUID_AMD_ECX_SSE4A)) {
1419 add_x86_feature(featureset, X86FSET_SSE4A);
1420 }
1421
1422 /*
1423 * If both the HTT and CMP_LGCY bits are set,
1424 * then we're not actually HyperThreaded. Read
1425 * "AMD CPUID Specification" for more details.
1426 */
1427 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
1428 is_x86_feature(featureset, X86FSET_HTT) &&
1429 (cp->cp_ecx & CPUID_AMD_ECX_CMP_LGCY)) {
1430 remove_x86_feature(featureset, X86FSET_HTT);
1431 add_x86_feature(featureset, X86FSET_CMP);
1432 }
1433 #if defined(__amd64)
1434 /*
1435 * It's really tricky to support syscall/sysret in
1436 * the i386 kernel; we rely on sysenter/sysexit
1437 * instead. In the amd64 kernel, things are -way-
1438 * better.
1439 */
1440 if (cp->cp_edx & CPUID_AMD_EDX_SYSC) {
1441 add_x86_feature(featureset, X86FSET_ASYSC);
1442 }
1443
1444 /*
1445 * While we're thinking about system calls, note
1446 * that AMD processors don't support sysenter
1447 * in long mode at all, so don't try to program them.
1448 */
1449 if (x86_vendor == X86_VENDOR_AMD) {
1450 remove_x86_feature(featureset, X86FSET_SEP);
1451 }
1452 #endif
1453 if (cp->cp_edx & CPUID_AMD_EDX_TSCP) {
1454 add_x86_feature(featureset, X86FSET_TSCP);
1455 }
1456
1457 if (cp->cp_ecx & CPUID_AMD_ECX_SVM) {
1458 add_x86_feature(featureset, X86FSET_SVM);
1459 }
1460
1461 if (cp->cp_ecx & CPUID_AMD_ECX_TOPOEXT) {
1462 add_x86_feature(featureset, X86FSET_TOPOEXT);
1463 }
1464 break;
1465 default:
1466 break;
1467 }
1468
1469 /*
1470 * Get CPUID data about processor cores and hyperthreads.
1471 */
1472 switch (cpi->cpi_vendor) {
1473 case X86_VENDOR_Intel:
1474 if (cpi->cpi_maxeax >= 4) {
1475 cp = &cpi->cpi_std[4];
1476 cp->cp_eax = 4;
1477 cp->cp_ecx = 0;
1478 (void) __cpuid_insn(cp);
1479 platform_cpuid_mangle(cpi->cpi_vendor, 4, cp);
1480 }
1481 /*FALLTHROUGH*/
1482 case X86_VENDOR_AMD:
1483 if (cpi->cpi_xmaxeax < 0x80000008)
1484 break;
1485 cp = &cpi->cpi_extd[8];
1486 cp->cp_eax = 0x80000008;
1487 (void) __cpuid_insn(cp);
1488 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000008, cp);
1489
1490 /*
1491 * Virtual and physical address limits from
1492 * cpuid override previously guessed values.
1493 */
1494 cpi->cpi_pabits = BITX(cp->cp_eax, 7, 0);
1495 cpi->cpi_vabits = BITX(cp->cp_eax, 15, 8);
1496 break;
1497 default:
1498 break;
1499 }
1500
1501 /*
1502 * Derive the number of cores per chip
1503 */
1504 switch (cpi->cpi_vendor) {
1505 case X86_VENDOR_Intel:
1506 if (cpi->cpi_maxeax < 4) {
1507 cpi->cpi_ncore_per_chip = 1;
1508 break;
1509 } else {
1510 cpi->cpi_ncore_per_chip =
1511 BITX((cpi)->cpi_std[4].cp_eax, 31, 26) + 1;
1512 }
1513 break;
1514 case X86_VENDOR_AMD:
1515 if (cpi->cpi_xmaxeax < 0x80000008) {
1516 cpi->cpi_ncore_per_chip = 1;
1517 break;
1518 } else {
1519 /*
1520 * On family 0xf cpuid fn 2 ECX[7:0] "NC" is
1521 * 1 less than the number of physical cores on
1522 * the chip. In family 0x10 this value can
1523 * be affected by "downcoring" - it reflects
1524 * 1 less than the number of cores actually
1525 * enabled on this node.
1526 */
1527 cpi->cpi_ncore_per_chip =
1528 BITX((cpi)->cpi_extd[8].cp_ecx, 7, 0) + 1;
1529 }
1530 break;
1531 default:
1532 cpi->cpi_ncore_per_chip = 1;
1533 break;
1534 }
1535
1536 /*
1537 * Get CPUID data about TSC Invariance in Deep C-State.
1538 */
1539 switch (cpi->cpi_vendor) {
1540 case X86_VENDOR_Intel:
1541 if (cpi->cpi_maxeax >= 7) {
1542 cp = &cpi->cpi_extd[7];
1543 cp->cp_eax = 0x80000007;
1544 cp->cp_ecx = 0;
1545 (void) __cpuid_insn(cp);
1546 }
1547 break;
1548 default:
1549 break;
1550 }
1551 } else {
1552 cpi->cpi_ncore_per_chip = 1;
1553 }
1554
1555 /*
1556 * If more than one core, then this processor is CMP.
1557 */
1558 if (cpi->cpi_ncore_per_chip > 1) {
1559 add_x86_feature(featureset, X86FSET_CMP);
1560 }
1561
1562 /*
1563 * If the number of cores is the same as the number
1564 * of CPUs, then we cannot have HyperThreading.
1565 */
1566 if (cpi->cpi_ncpu_per_chip == cpi->cpi_ncore_per_chip) {
1567 remove_x86_feature(featureset, X86FSET_HTT);
1568 }
1569
1570 cpi->cpi_apicid = CPI_APIC_ID(cpi);
1571 cpi->cpi_procnodes_per_pkg = 1;
1572 cpi->cpi_cores_per_compunit = 1;
1573 if (is_x86_feature(featureset, X86FSET_HTT) == B_FALSE &&
1574 is_x86_feature(featureset, X86FSET_CMP) == B_FALSE) {
1575 /*
1576 * Single-core single-threaded processors.
1577 */
1578 cpi->cpi_chipid = -1;
1579 cpi->cpi_clogid = 0;
1580 cpi->cpi_coreid = cpu->cpu_id;
1581 cpi->cpi_pkgcoreid = 0;
1582 if (cpi->cpi_vendor == X86_VENDOR_AMD)
1583 cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 3, 0);
1584 else
1585 cpi->cpi_procnodeid = cpi->cpi_chipid;
1586 } else if (cpi->cpi_ncpu_per_chip > 1) {
1587 if (cpi->cpi_vendor == X86_VENDOR_Intel)
1588 cpuid_intel_getids(cpu, featureset);
1589 else if (cpi->cpi_vendor == X86_VENDOR_AMD)
1590 cpuid_amd_getids(cpu);
1591 else {
1592 /*
1593 * All other processors are currently
1594 * assumed to have single cores.
1595 */
1596 cpi->cpi_coreid = cpi->cpi_chipid;
1597 cpi->cpi_pkgcoreid = 0;
1598 cpi->cpi_procnodeid = cpi->cpi_chipid;
1599 cpi->cpi_compunitid = cpi->cpi_chipid;
1600 }
1601 }
1602
1603 /*
1604 * Synthesize chip "revision" and socket type
1605 */
1606 cpi->cpi_chiprev = _cpuid_chiprev(cpi->cpi_vendor, cpi->cpi_family,
1607 cpi->cpi_model, cpi->cpi_step);
1608 cpi->cpi_chiprevstr = _cpuid_chiprevstr(cpi->cpi_vendor,
1609 cpi->cpi_family, cpi->cpi_model, cpi->cpi_step);
1610 cpi->cpi_socket = _cpuid_skt(cpi->cpi_vendor, cpi->cpi_family,
1611 cpi->cpi_model, cpi->cpi_step);
1612
1613 pass1_done:
1614 cpi->cpi_pass = 1;
1615 }
1616
1617 /*
1618 * Make copies of the cpuid table entries we depend on, in
1619 * part for ease of parsing now, in part so that we have only
1620 * one place to correct any of it, in part for ease of
1621 * later export to userland, and in part so we can look at
1622 * this stuff in a crash dump.
1623 */
1624
1625 /*ARGSUSED*/
1626 void
1627 cpuid_pass2(cpu_t *cpu)
1628 {
1629 uint_t n, nmax;
1630 int i;
1631 struct cpuid_regs *cp;
1632 uint8_t *dp;
1633 uint32_t *iptr;
1634 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
1635
1636 ASSERT(cpi->cpi_pass == 1);
1637
1638 if (cpi->cpi_maxeax < 1)
1639 goto pass2_done;
1640
1641 if ((nmax = cpi->cpi_maxeax + 1) > NMAX_CPI_STD)
1642 nmax = NMAX_CPI_STD;
1643 /*
1644 * (We already handled n == 0 and n == 1 in pass 1)
1645 */
1646 for (n = 2, cp = &cpi->cpi_std[2]; n < nmax; n++, cp++) {
1647 cp->cp_eax = n;
1648
1649 /*
1650 * CPUID function 4 expects %ecx to be initialized
1651 * with an index which indicates which cache to return
1652 * information about. The OS is expected to call function 4
1653 * with %ecx set to 0, 1, 2, ... until it returns with
1654 * EAX[4:0] set to 0, which indicates there are no more
1655 * caches.
1656 *
1657 * Here, populate cpi_std[4] with the information returned by
1658 * function 4 when %ecx == 0, and do the rest in cpuid_pass3()
1659 * when dynamic memory allocation becomes available.
1660 *
1661 * Note: we need to explicitly initialize %ecx here, since
1662 * function 4 may have been previously invoked.
1663 */
1664 if (n == 4)
1665 cp->cp_ecx = 0;
1666
1667 (void) __cpuid_insn(cp);
1668 platform_cpuid_mangle(cpi->cpi_vendor, n, cp);
1669 switch (n) {
1670 case 2:
1671 /*
1672 * "the lower 8 bits of the %eax register
1673 * contain a value that identifies the number
1674 * of times the cpuid [instruction] has to be
1675 * executed to obtain a complete image of the
1676 * processor's caching systems."
1677 *
1678 * How *do* they make this stuff up?
1679 */
1680 cpi->cpi_ncache = sizeof (*cp) *
1681 BITX(cp->cp_eax, 7, 0);
1682 if (cpi->cpi_ncache == 0)
1683 break;
1684 cpi->cpi_ncache--; /* skip count byte */
1685
1686 /*
1687 * Well, for now, rather than attempt to implement
1688 * this slightly dubious algorithm, we just look
1689 * at the first 15 ..
1690 */
1691 if (cpi->cpi_ncache > (sizeof (*cp) - 1))
1692 cpi->cpi_ncache = sizeof (*cp) - 1;
1693
1694 dp = cpi->cpi_cacheinfo;
1695 if (BITX(cp->cp_eax, 31, 31) == 0) {
1696 uint8_t *p = (void *)&cp->cp_eax;
1697 for (i = 1; i < 4; i++)
1698 if (p[i] != 0)
1699 *dp++ = p[i];
1700 }
1701 if (BITX(cp->cp_ebx, 31, 31) == 0) {
1702 uint8_t *p = (void *)&cp->cp_ebx;
1703 for (i = 0; i < 4; i++)
1704 if (p[i] != 0)
1705 *dp++ = p[i];
1706 }
1707 if (BITX(cp->cp_ecx, 31, 31) == 0) {
1708 uint8_t *p = (void *)&cp->cp_ecx;
1709 for (i = 0; i < 4; i++)
1710 if (p[i] != 0)
1711 *dp++ = p[i];
1712 }
1713 if (BITX(cp->cp_edx, 31, 31) == 0) {
1714 uint8_t *p = (void *)&cp->cp_edx;
1715 for (i = 0; i < 4; i++)
1716 if (p[i] != 0)
1717 *dp++ = p[i];
1718 }
1719 break;
1720
1721 case 3: /* Processor serial number, if PSN supported */
1722 break;
1723
1724 case 4: /* Deterministic cache parameters */
1725 break;
1726
1727 case 5: /* Monitor/Mwait parameters */
1728 {
1729 size_t mwait_size;
1730
1731 /*
1732 * check cpi_mwait.support which was set in cpuid_pass1
1733 */
1734 if (!(cpi->cpi_mwait.support & MWAIT_SUPPORT))
1735 break;
1736
1737 /*
1738 * Protect ourself from insane mwait line size.
1739 * Workaround for incomplete hardware emulator(s).
1740 */
1741 mwait_size = (size_t)MWAIT_SIZE_MAX(cpi);
1742 if (mwait_size < sizeof (uint32_t) ||
1743 !ISP2(mwait_size)) {
1744 #if DEBUG
1745 cmn_err(CE_NOTE, "Cannot handle cpu %d mwait "
1746 "size %ld", cpu->cpu_id, (long)mwait_size);
1747 #endif
1748 break;
1749 }
1750
1751 cpi->cpi_mwait.mon_min = (size_t)MWAIT_SIZE_MIN(cpi);
1752 cpi->cpi_mwait.mon_max = mwait_size;
1753 if (MWAIT_EXTENSION(cpi)) {
1754 cpi->cpi_mwait.support |= MWAIT_EXTENSIONS;
1755 if (MWAIT_INT_ENABLE(cpi))
1756 cpi->cpi_mwait.support |=
1757 MWAIT_ECX_INT_ENABLE;
1758 }
1759 break;
1760 }
1761 default:
1762 break;
1763 }
1764 }
1765
1766 if (cpi->cpi_maxeax >= 0xB && cpi->cpi_vendor == X86_VENDOR_Intel) {
1767 struct cpuid_regs regs;
1768
1769 cp = ®s;
1770 cp->cp_eax = 0xB;
1771 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
1772
1773 (void) __cpuid_insn(cp);
1774
1775 /*
1776 * Check CPUID.EAX=0BH, ECX=0H:EBX is non-zero, which
1777 * indicates that the extended topology enumeration leaf is
1778 * available.
1779 */
1780 if (cp->cp_ebx) {
1781 uint32_t x2apic_id;
1782 uint_t coreid_shift = 0;
1783 uint_t ncpu_per_core = 1;
1784 uint_t chipid_shift = 0;
1785 uint_t ncpu_per_chip = 1;
1786 uint_t i;
1787 uint_t level;
1788
1789 for (i = 0; i < CPI_FNB_ECX_MAX; i++) {
1790 cp->cp_eax = 0xB;
1791 cp->cp_ecx = i;
1792
1793 (void) __cpuid_insn(cp);
1794 level = CPI_CPU_LEVEL_TYPE(cp);
1795
1796 if (level == 1) {
1797 x2apic_id = cp->cp_edx;
1798 coreid_shift = BITX(cp->cp_eax, 4, 0);
1799 ncpu_per_core = BITX(cp->cp_ebx, 15, 0);
1800 } else if (level == 2) {
1801 x2apic_id = cp->cp_edx;
1802 chipid_shift = BITX(cp->cp_eax, 4, 0);
1803 ncpu_per_chip = BITX(cp->cp_ebx, 15, 0);
1804 }
1805 }
1806
1807 cpi->cpi_apicid = x2apic_id;
1808 cpi->cpi_ncpu_per_chip = ncpu_per_chip;
1809 cpi->cpi_ncore_per_chip = ncpu_per_chip /
1810 ncpu_per_core;
1811 cpi->cpi_chipid = x2apic_id >> chipid_shift;
1812 cpi->cpi_clogid = x2apic_id & ((1 << chipid_shift) - 1);
1813 cpi->cpi_coreid = x2apic_id >> coreid_shift;
1814 cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
1815 }
1816
1817 /* Make cp NULL so that we don't stumble on others */
1818 cp = NULL;
1819 }
1820
1821 /*
1822 * XSAVE enumeration
1823 */
1824 if (cpi->cpi_maxeax >= 0xD) {
1825 struct cpuid_regs regs;
1826 boolean_t cpuid_d_valid = B_TRUE;
1827
1828 cp = ®s;
1829 cp->cp_eax = 0xD;
1830 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
1831
1832 (void) __cpuid_insn(cp);
1833
1834 /*
1835 * Sanity checks for debug
1836 */
1837 if ((cp->cp_eax & XFEATURE_LEGACY_FP) == 0 ||
1838 (cp->cp_eax & XFEATURE_SSE) == 0) {
1839 cpuid_d_valid = B_FALSE;
1840 }
1841
1842 cpi->cpi_xsave.xsav_hw_features_low = cp->cp_eax;
1843 cpi->cpi_xsave.xsav_hw_features_high = cp->cp_edx;
1844 cpi->cpi_xsave.xsav_max_size = cp->cp_ecx;
1845
1846 /*
1847 * If the hw supports AVX, get the size and offset in the save
1848 * area for the ymm state.
1849 */
1850 if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_AVX) {
1851 cp->cp_eax = 0xD;
1852 cp->cp_ecx = 2;
1853 cp->cp_edx = cp->cp_ebx = 0;
1854
1855 (void) __cpuid_insn(cp);
1856
1857 if (cp->cp_ebx != CPUID_LEAFD_2_YMM_OFFSET ||
1858 cp->cp_eax != CPUID_LEAFD_2_YMM_SIZE) {
1859 cpuid_d_valid = B_FALSE;
1860 }
1861
1862 cpi->cpi_xsave.ymm_size = cp->cp_eax;
1863 cpi->cpi_xsave.ymm_offset = cp->cp_ebx;
1864 }
1865
1866 if (is_x86_feature(x86_featureset, X86FSET_XSAVE)) {
1867 xsave_state_size = 0;
1868 } else if (cpuid_d_valid) {
1869 xsave_state_size = cpi->cpi_xsave.xsav_max_size;
1870 } else {
1871 /* Broken CPUID 0xD, probably in HVM */
1872 cmn_err(CE_WARN, "cpu%d: CPUID.0xD returns invalid "
1873 "value: hw_low = %d, hw_high = %d, xsave_size = %d"
1874 ", ymm_size = %d, ymm_offset = %d\n",
1875 cpu->cpu_id, cpi->cpi_xsave.xsav_hw_features_low,
1876 cpi->cpi_xsave.xsav_hw_features_high,
1877 (int)cpi->cpi_xsave.xsav_max_size,
1878 (int)cpi->cpi_xsave.ymm_size,
1879 (int)cpi->cpi_xsave.ymm_offset);
1880
1881 if (xsave_state_size != 0) {
1882 /*
1883 * This must be a non-boot CPU. We cannot
1884 * continue, because boot cpu has already
1885 * enabled XSAVE.
1886 */
1887 ASSERT(cpu->cpu_id != 0);
1888 cmn_err(CE_PANIC, "cpu%d: we have already "
1889 "enabled XSAVE on boot cpu, cannot "
1890 "continue.", cpu->cpu_id);
1891 } else {
1892 /*
1893 * Must be from boot CPU, OK to disable XSAVE.
1894 */
1895 ASSERT(cpu->cpu_id == 0);
1896 remove_x86_feature(x86_featureset,
1897 X86FSET_XSAVE);
1898 remove_x86_feature(x86_featureset, X86FSET_AVX);
1899 CPI_FEATURES_ECX(cpi) &= ~CPUID_INTC_ECX_XSAVE;
1900 CPI_FEATURES_ECX(cpi) &= ~CPUID_INTC_ECX_AVX;
1901 xsave_force_disable = B_TRUE;
1902 }
1903 }
1904 }
1905
1906
1907 if ((cpi->cpi_xmaxeax & 0x80000000) == 0)
1908 goto pass2_done;
1909
1910 if ((nmax = cpi->cpi_xmaxeax - 0x80000000 + 1) > NMAX_CPI_EXTD)
1911 nmax = NMAX_CPI_EXTD;
1912 /*
1913 * Copy the extended properties, fixing them as we go.
1914 * (We already handled n == 0 and n == 1 in pass 1)
1915 */
1916 iptr = (void *)cpi->cpi_brandstr;
1917 for (n = 2, cp = &cpi->cpi_extd[2]; n < nmax; cp++, n++) {
1918 cp->cp_eax = 0x80000000 + n;
1919 (void) __cpuid_insn(cp);
1920 platform_cpuid_mangle(cpi->cpi_vendor, 0x80000000 + n, cp);
1921 switch (n) {
1922 case 2:
1923 case 3:
1924 case 4:
1925 /*
1926 * Extract the brand string
1927 */
1928 *iptr++ = cp->cp_eax;
1929 *iptr++ = cp->cp_ebx;
1930 *iptr++ = cp->cp_ecx;
1931 *iptr++ = cp->cp_edx;
1932 break;
1933 case 5:
1934 switch (cpi->cpi_vendor) {
1935 case X86_VENDOR_AMD:
1936 /*
1937 * The Athlon and Duron were the first
1938 * parts to report the sizes of the
1939 * TLB for large pages. Before then,
1940 * we don't trust the data.
1941 */
1942 if (cpi->cpi_family < 6 ||
1943 (cpi->cpi_family == 6 &&
1944 cpi->cpi_model < 1))
1945 cp->cp_eax = 0;
1946 break;
1947 default:
1948 break;
1949 }
1950 break;
1951 case 6:
1952 switch (cpi->cpi_vendor) {
1953 case X86_VENDOR_AMD:
1954 /*
1955 * The Athlon and Duron were the first
1956 * AMD parts with L2 TLB's.
1957 * Before then, don't trust the data.
1958 */
1959 if (cpi->cpi_family < 6 ||
1960 cpi->cpi_family == 6 &&
1961 cpi->cpi_model < 1)
1962 cp->cp_eax = cp->cp_ebx = 0;
1963 /*
1964 * AMD Duron rev A0 reports L2
1965 * cache size incorrectly as 1K
1966 * when it is really 64K
1967 */
1968 if (cpi->cpi_family == 6 &&
1969 cpi->cpi_model == 3 &&
1970 cpi->cpi_step == 0) {
1971 cp->cp_ecx &= 0xffff;
1972 cp->cp_ecx |= 0x400000;
1973 }
1974 break;
1975 case X86_VENDOR_Cyrix: /* VIA C3 */
1976 /*
1977 * VIA C3 processors are a bit messed
1978 * up w.r.t. encoding cache sizes in %ecx
1979 */
1980 if (cpi->cpi_family != 6)
1981 break;
1982 /*
1983 * model 7 and 8 were incorrectly encoded
1984 *
1985 * xxx is model 8 really broken?
1986 */
1987 if (cpi->cpi_model == 7 ||
1988 cpi->cpi_model == 8)
1989 cp->cp_ecx =
1990 BITX(cp->cp_ecx, 31, 24) << 16 |
1991 BITX(cp->cp_ecx, 23, 16) << 12 |
1992 BITX(cp->cp_ecx, 15, 8) << 8 |
1993 BITX(cp->cp_ecx, 7, 0);
1994 /*
1995 * model 9 stepping 1 has wrong associativity
1996 */
1997 if (cpi->cpi_model == 9 && cpi->cpi_step == 1)
1998 cp->cp_ecx |= 8 << 12;
1999 break;
2000 case X86_VENDOR_Intel:
2001 /*
2002 * Extended L2 Cache features function.
2003 * First appeared on Prescott.
2004 */
2005 default:
2006 break;
2007 }
2008 break;
2009 default:
2010 break;
2011 }
2012 }
2013
2014 pass2_done:
2015 cpi->cpi_pass = 2;
2016 }
2017
2018 static const char *
2019 intel_cpubrand(const struct cpuid_info *cpi)
2020 {
2021 int i;
2022
2023 if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2024 cpi->cpi_maxeax < 1 || cpi->cpi_family < 5)
2025 return ("i486");
2026
2027 switch (cpi->cpi_family) {
2028 case 5:
2029 return ("Intel Pentium(r)");
2030 case 6:
2031 switch (cpi->cpi_model) {
2032 uint_t celeron, xeon;
2033 const struct cpuid_regs *cp;
2034 case 0:
2035 case 1:
2036 case 2:
2037 return ("Intel Pentium(r) Pro");
2038 case 3:
2039 case 4:
2040 return ("Intel Pentium(r) II");
2041 case 6:
2042 return ("Intel Celeron(r)");
2043 case 5:
2044 case 7:
2045 celeron = xeon = 0;
2046 cp = &cpi->cpi_std[2]; /* cache info */
2047
2048 for (i = 1; i < 4; i++) {
2049 uint_t tmp;
2050
2051 tmp = (cp->cp_eax >> (8 * i)) & 0xff;
2052 if (tmp == 0x40)
2053 celeron++;
2054 if (tmp >= 0x44 && tmp <= 0x45)
2055 xeon++;
2056 }
2057
2058 for (i = 0; i < 2; i++) {
2059 uint_t tmp;
2060
2061 tmp = (cp->cp_ebx >> (8 * i)) & 0xff;
2062 if (tmp == 0x40)
2063 celeron++;
2064 else if (tmp >= 0x44 && tmp <= 0x45)
2065 xeon++;
2066 }
2067
2068 for (i = 0; i < 4; i++) {
2069 uint_t tmp;
2070
2071 tmp = (cp->cp_ecx >> (8 * i)) & 0xff;
2072 if (tmp == 0x40)
2073 celeron++;
2074 else if (tmp >= 0x44 && tmp <= 0x45)
2075 xeon++;
2076 }
2077
2078 for (i = 0; i < 4; i++) {
2079 uint_t tmp;
2080
2081 tmp = (cp->cp_edx >> (8 * i)) & 0xff;
2082 if (tmp == 0x40)
2083 celeron++;
2084 else if (tmp >= 0x44 && tmp <= 0x45)
2085 xeon++;
2086 }
2087
2088 if (celeron)
2089 return ("Intel Celeron(r)");
2090 if (xeon)
2091 return (cpi->cpi_model == 5 ?
2092 "Intel Pentium(r) II Xeon(tm)" :
2093 "Intel Pentium(r) III Xeon(tm)");
2094 return (cpi->cpi_model == 5 ?
2095 "Intel Pentium(r) II or Pentium(r) II Xeon(tm)" :
2096 "Intel Pentium(r) III or Pentium(r) III Xeon(tm)");
2097 default:
2098 break;
2099 }
2100 default:
2101 break;
2102 }
2103
2104 /* BrandID is present if the field is nonzero */
2105 if (cpi->cpi_brandid != 0) {
2106 static const struct {
2107 uint_t bt_bid;
2108 const char *bt_str;
2109 } brand_tbl[] = {
2110 { 0x1, "Intel(r) Celeron(r)" },
2111 { 0x2, "Intel(r) Pentium(r) III" },
2112 { 0x3, "Intel(r) Pentium(r) III Xeon(tm)" },
2113 { 0x4, "Intel(r) Pentium(r) III" },
2114 { 0x6, "Mobile Intel(r) Pentium(r) III" },
2115 { 0x7, "Mobile Intel(r) Celeron(r)" },
2116 { 0x8, "Intel(r) Pentium(r) 4" },
2117 { 0x9, "Intel(r) Pentium(r) 4" },
2118 { 0xa, "Intel(r) Celeron(r)" },
2119 { 0xb, "Intel(r) Xeon(tm)" },
2120 { 0xc, "Intel(r) Xeon(tm) MP" },
2121 { 0xe, "Mobile Intel(r) Pentium(r) 4" },
2122 { 0xf, "Mobile Intel(r) Celeron(r)" },
2123 { 0x11, "Mobile Genuine Intel(r)" },
2124 { 0x12, "Intel(r) Celeron(r) M" },
2125 { 0x13, "Mobile Intel(r) Celeron(r)" },
2126 { 0x14, "Intel(r) Celeron(r)" },
2127 { 0x15, "Mobile Genuine Intel(r)" },
2128 { 0x16, "Intel(r) Pentium(r) M" },
2129 { 0x17, "Mobile Intel(r) Celeron(r)" }
2130 };
2131 uint_t btblmax = sizeof (brand_tbl) / sizeof (brand_tbl[0]);
2132 uint_t sgn;
2133
2134 sgn = (cpi->cpi_family << 8) |
2135 (cpi->cpi_model << 4) | cpi->cpi_step;
2136
2137 for (i = 0; i < btblmax; i++)
2138 if (brand_tbl[i].bt_bid == cpi->cpi_brandid)
2139 break;
2140 if (i < btblmax) {
2141 if (sgn == 0x6b1 && cpi->cpi_brandid == 3)
2142 return ("Intel(r) Celeron(r)");
2143 if (sgn < 0xf13 && cpi->cpi_brandid == 0xb)
2144 return ("Intel(r) Xeon(tm) MP");
2145 if (sgn < 0xf13 && cpi->cpi_brandid == 0xe)
2146 return ("Intel(r) Xeon(tm)");
2147 return (brand_tbl[i].bt_str);
2148 }
2149 }
2150
2151 return (NULL);
2152 }
2153
2154 static const char *
2155 amd_cpubrand(const struct cpuid_info *cpi)
2156 {
2157 if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2158 cpi->cpi_maxeax < 1 || cpi->cpi_family < 5)
2159 return ("i486 compatible");
2160
2161 switch (cpi->cpi_family) {
2162 case 5:
2163 switch (cpi->cpi_model) {
2164 case 0:
2165 case 1:
2166 case 2:
2167 case 3:
2168 case 4:
2169 case 5:
2170 return ("AMD-K5(r)");
2171 case 6:
2172 case 7:
2173 return ("AMD-K6(r)");
2174 case 8:
2175 return ("AMD-K6(r)-2");
2176 case 9:
2177 return ("AMD-K6(r)-III");
2178 default:
2179 return ("AMD (family 5)");
2180 }
2181 case 6:
2182 switch (cpi->cpi_model) {
2183 case 1:
2184 return ("AMD-K7(tm)");
2185 case 0:
2186 case 2:
2187 case 4:
2188 return ("AMD Athlon(tm)");
2189 case 3:
2190 case 7:
2191 return ("AMD Duron(tm)");
2192 case 6:
2193 case 8:
2194 case 10:
2195 /*
2196 * Use the L2 cache size to distinguish
2197 */
2198 return ((cpi->cpi_extd[6].cp_ecx >> 16) >= 256 ?
2199 "AMD Athlon(tm)" : "AMD Duron(tm)");
2200 default:
2201 return ("AMD (family 6)");
2202 }
2203 default:
2204 break;
2205 }
2206
2207 if (cpi->cpi_family == 0xf && cpi->cpi_model == 5 &&
2208 cpi->cpi_brandid != 0) {
2209 switch (BITX(cpi->cpi_brandid, 7, 5)) {
2210 case 3:
2211 return ("AMD Opteron(tm) UP 1xx");
2212 case 4:
2213 return ("AMD Opteron(tm) DP 2xx");
2214 case 5:
2215 return ("AMD Opteron(tm) MP 8xx");
2216 default:
2217 return ("AMD Opteron(tm)");
2218 }
2219 }
2220
2221 return (NULL);
2222 }
2223
2224 static const char *
2225 cyrix_cpubrand(struct cpuid_info *cpi, uint_t type)
2226 {
2227 if (!is_x86_feature(x86_featureset, X86FSET_CPUID) ||
2228 cpi->cpi_maxeax < 1 || cpi->cpi_family < 5 ||
2229 type == X86_TYPE_CYRIX_486)
2230 return ("i486 compatible");
2231
2232 switch (type) {
2233 case X86_TYPE_CYRIX_6x86:
2234 return ("Cyrix 6x86");
2235 case X86_TYPE_CYRIX_6x86L:
2236 return ("Cyrix 6x86L");
2237 case X86_TYPE_CYRIX_6x86MX:
2238 return ("Cyrix 6x86MX");
2239 case X86_TYPE_CYRIX_GXm:
2240 return ("Cyrix GXm");
2241 case X86_TYPE_CYRIX_MediaGX:
2242 return ("Cyrix MediaGX");
2243 case X86_TYPE_CYRIX_MII:
2244 return ("Cyrix M2");
2245 case X86_TYPE_VIA_CYRIX_III:
2246 return ("VIA Cyrix M3");
2247 default:
2248 /*
2249 * Have another wild guess ..
2250 */
2251 if (cpi->cpi_family == 4 && cpi->cpi_model == 9)
2252 return ("Cyrix 5x86");
2253 else if (cpi->cpi_family == 5) {
2254 switch (cpi->cpi_model) {
2255 case 2:
2256 return ("Cyrix 6x86"); /* Cyrix M1 */
2257 case 4:
2258 return ("Cyrix MediaGX");
2259 default:
2260 break;
2261 }
2262 } else if (cpi->cpi_family == 6) {
2263 switch (cpi->cpi_model) {
2264 case 0:
2265 return ("Cyrix 6x86MX"); /* Cyrix M2? */
2266 case 5:
2267 case 6:
2268 case 7:
2269 case 8:
2270 case 9:
2271 return ("VIA C3");
2272 default:
2273 break;
2274 }
2275 }
2276 break;
2277 }
2278 return (NULL);
2279 }
2280
2281 /*
2282 * This only gets called in the case that the CPU extended
2283 * feature brand string (0x80000002, 0x80000003, 0x80000004)
2284 * aren't available, or contain null bytes for some reason.
2285 */
2286 static void
2287 fabricate_brandstr(struct cpuid_info *cpi)
2288 {
2289 const char *brand = NULL;
2290
2291 switch (cpi->cpi_vendor) {
2292 case X86_VENDOR_Intel:
2293 brand = intel_cpubrand(cpi);
2294 break;
2295 case X86_VENDOR_AMD:
2296 brand = amd_cpubrand(cpi);
2297 break;
2298 case X86_VENDOR_Cyrix:
2299 brand = cyrix_cpubrand(cpi, x86_type);
2300 break;
2301 case X86_VENDOR_NexGen:
2302 if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
2303 brand = "NexGen Nx586";
2304 break;
2305 case X86_VENDOR_Centaur:
2306 if (cpi->cpi_family == 5)
2307 switch (cpi->cpi_model) {
2308 case 4:
2309 brand = "Centaur C6";
2310 break;
2311 case 8:
2312 brand = "Centaur C2";
2313 break;
2314 case 9:
2315 brand = "Centaur C3";
2316 break;
2317 default:
2318 break;
2319 }
2320 break;
2321 case X86_VENDOR_Rise:
2322 if (cpi->cpi_family == 5 &&
2323 (cpi->cpi_model == 0 || cpi->cpi_model == 2))
2324 brand = "Rise mP6";
2325 break;
2326 case X86_VENDOR_SiS:
2327 if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
2328 brand = "SiS 55x";
2329 break;
2330 case X86_VENDOR_TM:
2331 if (cpi->cpi_family == 5 && cpi->cpi_model == 4)
2332 brand = "Transmeta Crusoe TM3x00 or TM5x00";
2333 break;
2334 case X86_VENDOR_NSC:
2335 case X86_VENDOR_UMC:
2336 default:
2337 break;
2338 }
2339 if (brand) {
2340 (void) strcpy((char *)cpi->cpi_brandstr, brand);
2341 return;
2342 }
2343
2344 /*
2345 * If all else fails ...
2346 */
2347 (void) snprintf(cpi->cpi_brandstr, sizeof (cpi->cpi_brandstr),
2348 "%s %d.%d.%d", cpi->cpi_vendorstr, cpi->cpi_family,
2349 cpi->cpi_model, cpi->cpi_step);
2350 }
2351
2352 /*
2353 * This routine is called just after kernel memory allocation
2354 * becomes available on cpu0, and as part of mp_startup() on
2355 * the other cpus.
2356 *
2357 * Fixup the brand string, and collect any information from cpuid
2358 * that requires dynamicically allocated storage to represent.
2359 */
2360 /*ARGSUSED*/
2361 void
2362 cpuid_pass3(cpu_t *cpu)
2363 {
2364 int i, max, shft, level, size;
2365 struct cpuid_regs regs;
2366 struct cpuid_regs *cp;
2367 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2368
2369 ASSERT(cpi->cpi_pass == 2);
2370
2371 /*
2372 * Function 4: Deterministic cache parameters
2373 *
2374 * Take this opportunity to detect the number of threads
2375 * sharing the last level cache, and construct a corresponding
2376 * cache id. The respective cpuid_info members are initialized
2377 * to the default case of "no last level cache sharing".
2378 */
2379 cpi->cpi_ncpu_shr_last_cache = 1;
2380 cpi->cpi_last_lvl_cacheid = cpu->cpu_id;
2381
2382 if (cpi->cpi_maxeax >= 4 && cpi->cpi_vendor == X86_VENDOR_Intel) {
2383
2384 /*
2385 * Find the # of elements (size) returned by fn 4, and along
2386 * the way detect last level cache sharing details.
2387 */
2388 bzero(®s, sizeof (regs));
2389 cp = ®s;
2390 for (i = 0, max = 0; i < CPI_FN4_ECX_MAX; i++) {
2391 cp->cp_eax = 4;
2392 cp->cp_ecx = i;
2393
2394 (void) __cpuid_insn(cp);
2395
2396 if (CPI_CACHE_TYPE(cp) == 0)
2397 break;
2398 level = CPI_CACHE_LVL(cp);
2399 if (level > max) {
2400 max = level;
2401 cpi->cpi_ncpu_shr_last_cache =
2402 CPI_NTHR_SHR_CACHE(cp) + 1;
2403 }
2404 }
2405 cpi->cpi_std_4_size = size = i;
2406
2407 /*
2408 * Allocate the cpi_std_4 array. The first element
2409 * references the regs for fn 4, %ecx == 0, which
2410 * cpuid_pass2() stashed in cpi->cpi_std[4].
2411 */
2412 if (size > 0) {
2413 cpi->cpi_std_4 =
2414 kmem_alloc(size * sizeof (cp), KM_SLEEP);
2415 cpi->cpi_std_4[0] = &cpi->cpi_std[4];
2416
2417 /*
2418 * Allocate storage to hold the additional regs
2419 * for function 4, %ecx == 1 .. cpi_std_4_size.
2420 *
2421 * The regs for fn 4, %ecx == 0 has already
2422 * been allocated as indicated above.
2423 */
2424 for (i = 1; i < size; i++) {
2425 cp = cpi->cpi_std_4[i] =
2426 kmem_zalloc(sizeof (regs), KM_SLEEP);
2427 cp->cp_eax = 4;
2428 cp->cp_ecx = i;
2429
2430 (void) __cpuid_insn(cp);
2431 }
2432 }
2433 /*
2434 * Determine the number of bits needed to represent
2435 * the number of CPUs sharing the last level cache.
2436 *
2437 * Shift off that number of bits from the APIC id to
2438 * derive the cache id.
2439 */
2440 shft = 0;
2441 for (i = 1; i < cpi->cpi_ncpu_shr_last_cache; i <<= 1)
2442 shft++;
2443 cpi->cpi_last_lvl_cacheid = cpi->cpi_apicid >> shft;
2444 }
2445
2446 /*
2447 * Now fixup the brand string
2448 */
2449 if ((cpi->cpi_xmaxeax & 0x80000000) == 0) {
2450 fabricate_brandstr(cpi);
2451 } else {
2452
2453 /*
2454 * If we successfully extracted a brand string from the cpuid
2455 * instruction, clean it up by removing leading spaces and
2456 * similar junk.
2457 */
2458 if (cpi->cpi_brandstr[0]) {
2459 size_t maxlen = sizeof (cpi->cpi_brandstr);
2460 char *src, *dst;
2461
2462 dst = src = (char *)cpi->cpi_brandstr;
2463 src[maxlen - 1] = '\0';
2464 /*
2465 * strip leading spaces
2466 */
2467 while (*src == ' ')
2468 src++;
2469 /*
2470 * Remove any 'Genuine' or "Authentic" prefixes
2471 */
2472 if (strncmp(src, "Genuine ", 8) == 0)
2473 src += 8;
2474 if (strncmp(src, "Authentic ", 10) == 0)
2475 src += 10;
2476
2477 /*
2478 * Now do an in-place copy.
2479 * Map (R) to (r) and (TM) to (tm).
2480 * The era of teletypes is long gone, and there's
2481 * -really- no need to shout.
2482 */
2483 while (*src != '\0') {
2484 if (src[0] == '(') {
2485 if (strncmp(src + 1, "R)", 2) == 0) {
2486 (void) strncpy(dst, "(r)", 3);
2487 src += 3;
2488 dst += 3;
2489 continue;
2490 }
2491 if (strncmp(src + 1, "TM)", 3) == 0) {
2492 (void) strncpy(dst, "(tm)", 4);
2493 src += 4;
2494 dst += 4;
2495 continue;
2496 }
2497 }
2498 *dst++ = *src++;
2499 }
2500 *dst = '\0';
2501
2502 /*
2503 * Finally, remove any trailing spaces
2504 */
2505 while (--dst > cpi->cpi_brandstr)
2506 if (*dst == ' ')
2507 *dst = '\0';
2508 else
2509 break;
2510 } else
2511 fabricate_brandstr(cpi);
2512 }
2513 cpi->cpi_pass = 3;
2514 }
2515
2516 /*
2517 * This routine is called out of bind_hwcap() much later in the life
2518 * of the kernel (post_startup()). The job of this routine is to resolve
2519 * the hardware feature support and kernel support for those features into
2520 * what we're actually going to tell applications via the aux vector.
2521 */
2522 uint_t
2523 cpuid_pass4(cpu_t *cpu)
2524 {
2525 struct cpuid_info *cpi;
2526 uint_t hwcap_flags = 0;
2527
2528 if (cpu == NULL)
2529 cpu = CPU;
2530 cpi = cpu->cpu_m.mcpu_cpi;
2531
2532 ASSERT(cpi->cpi_pass == 3);
2533
2534 if (cpi->cpi_maxeax >= 1) {
2535 uint32_t *edx = &cpi->cpi_support[STD_EDX_FEATURES];
2536 uint32_t *ecx = &cpi->cpi_support[STD_ECX_FEATURES];
2537
2538 *edx = CPI_FEATURES_EDX(cpi);
2539 *ecx = CPI_FEATURES_ECX(cpi);
2540
2541 /*
2542 * [these require explicit kernel support]
2543 */
2544 if (!is_x86_feature(x86_featureset, X86FSET_SEP))
2545 *edx &= ~CPUID_INTC_EDX_SEP;
2546
2547 if (!is_x86_feature(x86_featureset, X86FSET_SSE))
2548 *edx &= ~(CPUID_INTC_EDX_FXSR|CPUID_INTC_EDX_SSE);
2549 if (!is_x86_feature(x86_featureset, X86FSET_SSE2))
2550 *edx &= ~CPUID_INTC_EDX_SSE2;
2551
2552 if (!is_x86_feature(x86_featureset, X86FSET_HTT))
2553 *edx &= ~CPUID_INTC_EDX_HTT;
2554
2555 if (!is_x86_feature(x86_featureset, X86FSET_SSE3))
2556 *ecx &= ~CPUID_INTC_ECX_SSE3;
2557
2558 if (!is_x86_feature(x86_featureset, X86FSET_SSSE3))
2559 *ecx &= ~CPUID_INTC_ECX_SSSE3;
2560 if (!is_x86_feature(x86_featureset, X86FSET_SSE4_1))
2561 *ecx &= ~CPUID_INTC_ECX_SSE4_1;
2562 if (!is_x86_feature(x86_featureset, X86FSET_SSE4_2))
2563 *ecx &= ~CPUID_INTC_ECX_SSE4_2;
2564 if (!is_x86_feature(x86_featureset, X86FSET_AES))
2565 *ecx &= ~CPUID_INTC_ECX_AES;
2566 if (!is_x86_feature(x86_featureset, X86FSET_PCLMULQDQ))
2567 *ecx &= ~CPUID_INTC_ECX_PCLMULQDQ;
2568 if (!is_x86_feature(x86_featureset, X86FSET_XSAVE))
2569 *ecx &= ~(CPUID_INTC_ECX_XSAVE |
2570 CPUID_INTC_ECX_OSXSAVE);
2571 if (!is_x86_feature(x86_featureset, X86FSET_AVX))
2572 *ecx &= ~CPUID_INTC_ECX_AVX;
2573
2574 /*
2575 * [no explicit support required beyond x87 fp context]
2576 */
2577 if (!fpu_exists)
2578 *edx &= ~(CPUID_INTC_EDX_FPU | CPUID_INTC_EDX_MMX);
2579
2580 /*
2581 * Now map the supported feature vector to things that we
2582 * think userland will care about.
2583 */
2584 if (*edx & CPUID_INTC_EDX_SEP)
2585 hwcap_flags |= AV_386_SEP;
2586 if (*edx & CPUID_INTC_EDX_SSE)
2587 hwcap_flags |= AV_386_FXSR | AV_386_SSE;
2588 if (*edx & CPUID_INTC_EDX_SSE2)
2589 hwcap_flags |= AV_386_SSE2;
2590 if (*ecx & CPUID_INTC_ECX_SSE3)
2591 hwcap_flags |= AV_386_SSE3;
2592 if (*ecx & CPUID_INTC_ECX_SSSE3)
2593 hwcap_flags |= AV_386_SSSE3;
2594 if (*ecx & CPUID_INTC_ECX_SSE4_1)
2595 hwcap_flags |= AV_386_SSE4_1;
2596 if (*ecx & CPUID_INTC_ECX_SSE4_2)
2597 hwcap_flags |= AV_386_SSE4_2;
2598 if (*ecx & CPUID_INTC_ECX_MOVBE)
2599 hwcap_flags |= AV_386_MOVBE;
2600 if (*ecx & CPUID_INTC_ECX_AES)
2601 hwcap_flags |= AV_386_AES;
2602 if (*ecx & CPUID_INTC_ECX_PCLMULQDQ)
2603 hwcap_flags |= AV_386_PCLMULQDQ;
2604 if ((*ecx & CPUID_INTC_ECX_XSAVE) &&
2605 (*ecx & CPUID_INTC_ECX_OSXSAVE))
2606 hwcap_flags |= AV_386_XSAVE;
2607 if (*ecx & CPUID_INTC_ECX_VMX)
2608 hwcap_flags |= AV_386_VMX;
2609 if (*ecx & CPUID_INTC_ECX_POPCNT)
2610 hwcap_flags |= AV_386_POPCNT;
2611 if (*edx & CPUID_INTC_EDX_FPU)
2612 hwcap_flags |= AV_386_FPU;
2613 if (*edx & CPUID_INTC_EDX_MMX)
2614 hwcap_flags |= AV_386_MMX;
2615
2616 if (*edx & CPUID_INTC_EDX_TSC)
2617 hwcap_flags |= AV_386_TSC;
2618 if (*edx & CPUID_INTC_EDX_CX8)
2619 hwcap_flags |= AV_386_CX8;
2620 if (*edx & CPUID_INTC_EDX_CMOV)
2621 hwcap_flags |= AV_386_CMOV;
2622 if (*ecx & CPUID_INTC_ECX_CX16)
2623 hwcap_flags |= AV_386_CX16;
2624 }
2625
2626 if (cpi->cpi_xmaxeax < 0x80000001)
2627 goto pass4_done;
2628
2629 switch (cpi->cpi_vendor) {
2630 struct cpuid_regs cp;
2631 uint32_t *edx, *ecx;
2632
2633 case X86_VENDOR_Intel:
2634 /*
2635 * Seems like Intel duplicated what we necessary
2636 * here to make the initial crop of 64-bit OS's work.
2637 * Hopefully, those are the only "extended" bits
2638 * they'll add.
2639 */
2640 /*FALLTHROUGH*/
2641
2642 case X86_VENDOR_AMD:
2643 edx = &cpi->cpi_support[AMD_EDX_FEATURES];
2644 ecx = &cpi->cpi_support[AMD_ECX_FEATURES];
2645
2646 *edx = CPI_FEATURES_XTD_EDX(cpi);
2647 *ecx = CPI_FEATURES_XTD_ECX(cpi);
2648
2649 /*
2650 * [these features require explicit kernel support]
2651 */
2652 switch (cpi->cpi_vendor) {
2653 case X86_VENDOR_Intel:
2654 if (!is_x86_feature(x86_featureset, X86FSET_TSCP))
2655 *edx &= ~CPUID_AMD_EDX_TSCP;
2656 break;
2657
2658 case X86_VENDOR_AMD:
2659 if (!is_x86_feature(x86_featureset, X86FSET_TSCP))
2660 *edx &= ~CPUID_AMD_EDX_TSCP;
2661 if (!is_x86_feature(x86_featureset, X86FSET_SSE4A))
2662 *ecx &= ~CPUID_AMD_ECX_SSE4A;
2663 break;
2664
2665 default:
2666 break;
2667 }
2668
2669 /*
2670 * [no explicit support required beyond
2671 * x87 fp context and exception handlers]
2672 */
2673 if (!fpu_exists)
2674 *edx &= ~(CPUID_AMD_EDX_MMXamd |
2675 CPUID_AMD_EDX_3DNow | CPUID_AMD_EDX_3DNowx);
2676
2677 if (!is_x86_feature(x86_featureset, X86FSET_NX))
2678 *edx &= ~CPUID_AMD_EDX_NX;
2679 #if !defined(__amd64)
2680 *edx &= ~CPUID_AMD_EDX_LM;
2681 #endif
2682 /*
2683 * Now map the supported feature vector to
2684 * things that we think userland will care about.
2685 */
2686 #if defined(__amd64)
2687 if (*edx & CPUID_AMD_EDX_SYSC)
2688 hwcap_flags |= AV_386_AMD_SYSC;
2689 #endif
2690 if (*edx & CPUID_AMD_EDX_MMXamd)
2691 hwcap_flags |= AV_386_AMD_MMX;
2692 if (*edx & CPUID_AMD_EDX_3DNow)
2693 hwcap_flags |= AV_386_AMD_3DNow;
2694 if (*edx & CPUID_AMD_EDX_3DNowx)
2695 hwcap_flags |= AV_386_AMD_3DNowx;
2696 if (*ecx & CPUID_AMD_ECX_SVM)
2697 hwcap_flags |= AV_386_AMD_SVM;
2698
2699 switch (cpi->cpi_vendor) {
2700 case X86_VENDOR_AMD:
2701 if (*edx & CPUID_AMD_EDX_TSCP)
2702 hwcap_flags |= AV_386_TSCP;
2703 if (*ecx & CPUID_AMD_ECX_AHF64)
2704 hwcap_flags |= AV_386_AHF;
2705 if (*ecx & CPUID_AMD_ECX_SSE4A)
2706 hwcap_flags |= AV_386_AMD_SSE4A;
2707 if (*ecx & CPUID_AMD_ECX_LZCNT)
2708 hwcap_flags |= AV_386_AMD_LZCNT;
2709 break;
2710
2711 case X86_VENDOR_Intel:
2712 if (*edx & CPUID_AMD_EDX_TSCP)
2713 hwcap_flags |= AV_386_TSCP;
2714 /*
2715 * Aarrgh.
2716 * Intel uses a different bit in the same word.
2717 */
2718 if (*ecx & CPUID_INTC_ECX_AHF64)
2719 hwcap_flags |= AV_386_AHF;
2720 break;
2721
2722 default:
2723 break;
2724 }
2725 break;
2726
2727 case X86_VENDOR_TM:
2728 cp.cp_eax = 0x80860001;
2729 (void) __cpuid_insn(&cp);
2730 cpi->cpi_support[TM_EDX_FEATURES] = cp.cp_edx;
2731 break;
2732
2733 default:
2734 break;
2735 }
2736
2737 pass4_done:
2738 cpi->cpi_pass = 4;
2739 return (hwcap_flags);
2740 }
2741
2742
2743 /*
2744 * Simulate the cpuid instruction using the data we previously
2745 * captured about this CPU. We try our best to return the truth
2746 * about the hardware, independently of kernel support.
2747 */
2748 uint32_t
2749 cpuid_insn(cpu_t *cpu, struct cpuid_regs *cp)
2750 {
2751 struct cpuid_info *cpi;
2752 struct cpuid_regs *xcp;
2753
2754 if (cpu == NULL)
2755 cpu = CPU;
2756 cpi = cpu->cpu_m.mcpu_cpi;
2757
2758 ASSERT(cpuid_checkpass(cpu, 3));
2759
2760 /*
2761 * CPUID data is cached in two separate places: cpi_std for standard
2762 * CPUID functions, and cpi_extd for extended CPUID functions.
2763 */
2764 if (cp->cp_eax <= cpi->cpi_maxeax && cp->cp_eax < NMAX_CPI_STD)
2765 xcp = &cpi->cpi_std[cp->cp_eax];
2766 else if (cp->cp_eax >= 0x80000000 && cp->cp_eax <= cpi->cpi_xmaxeax &&
2767 cp->cp_eax < 0x80000000 + NMAX_CPI_EXTD)
2768 xcp = &cpi->cpi_extd[cp->cp_eax - 0x80000000];
2769 else
2770 /*
2771 * The caller is asking for data from an input parameter which
2772 * the kernel has not cached. In this case we go fetch from
2773 * the hardware and return the data directly to the user.
2774 */
2775 return (__cpuid_insn(cp));
2776
2777 cp->cp_eax = xcp->cp_eax;
2778 cp->cp_ebx = xcp->cp_ebx;
2779 cp->cp_ecx = xcp->cp_ecx;
2780 cp->cp_edx = xcp->cp_edx;
2781 return (cp->cp_eax);
2782 }
2783
2784 int
2785 cpuid_checkpass(cpu_t *cpu, int pass)
2786 {
2787 return (cpu != NULL && cpu->cpu_m.mcpu_cpi != NULL &&
2788 cpu->cpu_m.mcpu_cpi->cpi_pass >= pass);
2789 }
2790
2791 int
2792 cpuid_getbrandstr(cpu_t *cpu, char *s, size_t n)
2793 {
2794 ASSERT(cpuid_checkpass(cpu, 3));
2795
2796 return (snprintf(s, n, "%s", cpu->cpu_m.mcpu_cpi->cpi_brandstr));
2797 }
2798
2799 int
2800 cpuid_is_cmt(cpu_t *cpu)
2801 {
2802 if (cpu == NULL)
2803 cpu = CPU;
2804
2805 ASSERT(cpuid_checkpass(cpu, 1));
2806
2807 return (cpu->cpu_m.mcpu_cpi->cpi_chipid >= 0);
2808 }
2809
2810 /*
2811 * AMD and Intel both implement the 64-bit variant of the syscall
2812 * instruction (syscallq), so if there's -any- support for syscall,
2813 * cpuid currently says "yes, we support this".
2814 *
2815 * However, Intel decided to -not- implement the 32-bit variant of the
2816 * syscall instruction, so we provide a predicate to allow our caller
2817 * to test that subtlety here.
2818 *
2819 * XXPV Currently, 32-bit syscall instructions don't work via the hypervisor,
2820 * even in the case where the hardware would in fact support it.
2821 */
2822 /*ARGSUSED*/
2823 int
2824 cpuid_syscall32_insn(cpu_t *cpu)
2825 {
2826 ASSERT(cpuid_checkpass((cpu == NULL ? CPU : cpu), 1));
2827
2828 #if !defined(__xpv)
2829 if (cpu == NULL)
2830 cpu = CPU;
2831
2832 /*CSTYLED*/
2833 {
2834 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2835
2836 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
2837 cpi->cpi_xmaxeax >= 0x80000001 &&
2838 (CPI_FEATURES_XTD_EDX(cpi) & CPUID_AMD_EDX_SYSC))
2839 return (1);
2840 }
2841 #endif
2842 return (0);
2843 }
2844
2845 int
2846 cpuid_getidstr(cpu_t *cpu, char *s, size_t n)
2847 {
2848 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2849
2850 static const char fmt[] =
2851 "x86 (%s %X family %d model %d step %d clock %d MHz)";
2852 static const char fmt_ht[] =
2853 "x86 (chipid 0x%x %s %X family %d model %d step %d clock %d MHz)";
2854
2855 ASSERT(cpuid_checkpass(cpu, 1));
2856
2857 if (cpuid_is_cmt(cpu))
2858 return (snprintf(s, n, fmt_ht, cpi->cpi_chipid,
2859 cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
2860 cpi->cpi_family, cpi->cpi_model,
2861 cpi->cpi_step, cpu->cpu_type_info.pi_clock));
2862 return (snprintf(s, n, fmt,
2863 cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
2864 cpi->cpi_family, cpi->cpi_model,
2865 cpi->cpi_step, cpu->cpu_type_info.pi_clock));
2866 }
2867
2868 const char *
2869 cpuid_getvendorstr(cpu_t *cpu)
2870 {
2871 ASSERT(cpuid_checkpass(cpu, 1));
2872 return ((const char *)cpu->cpu_m.mcpu_cpi->cpi_vendorstr);
2873 }
2874
2875 uint_t
2876 cpuid_getvendor(cpu_t *cpu)
2877 {
2878 ASSERT(cpuid_checkpass(cpu, 1));
2879 return (cpu->cpu_m.mcpu_cpi->cpi_vendor);
2880 }
2881
2882 uint_t
2883 cpuid_getfamily(cpu_t *cpu)
2884 {
2885 ASSERT(cpuid_checkpass(cpu, 1));
2886 return (cpu->cpu_m.mcpu_cpi->cpi_family);
2887 }
2888
2889 uint_t
2890 cpuid_getmodel(cpu_t *cpu)
2891 {
2892 ASSERT(cpuid_checkpass(cpu, 1));
2893 return (cpu->cpu_m.mcpu_cpi->cpi_model);
2894 }
2895
2896 uint_t
2897 cpuid_get_ncpu_per_chip(cpu_t *cpu)
2898 {
2899 ASSERT(cpuid_checkpass(cpu, 1));
2900 return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_per_chip);
2901 }
2902
2903 uint_t
2904 cpuid_get_ncore_per_chip(cpu_t *cpu)
2905 {
2906 ASSERT(cpuid_checkpass(cpu, 1));
2907 return (cpu->cpu_m.mcpu_cpi->cpi_ncore_per_chip);
2908 }
2909
2910 uint_t
2911 cpuid_get_ncpu_sharing_last_cache(cpu_t *cpu)
2912 {
2913 ASSERT(cpuid_checkpass(cpu, 2));
2914 return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_shr_last_cache);
2915 }
2916
2917 id_t
2918 cpuid_get_last_lvl_cacheid(cpu_t *cpu)
2919 {
2920 ASSERT(cpuid_checkpass(cpu, 2));
2921 return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
2922 }
2923
2924 uint_t
2925 cpuid_getstep(cpu_t *cpu)
2926 {
2927 ASSERT(cpuid_checkpass(cpu, 1));
2928 return (cpu->cpu_m.mcpu_cpi->cpi_step);
2929 }
2930
2931 uint_t
2932 cpuid_getsig(struct cpu *cpu)
2933 {
2934 ASSERT(cpuid_checkpass(cpu, 1));
2935 return (cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_eax);
2936 }
2937
2938 uint32_t
2939 cpuid_getchiprev(struct cpu *cpu)
2940 {
2941 ASSERT(cpuid_checkpass(cpu, 1));
2942 return (cpu->cpu_m.mcpu_cpi->cpi_chiprev);
2943 }
2944
2945 const char *
2946 cpuid_getchiprevstr(struct cpu *cpu)
2947 {
2948 ASSERT(cpuid_checkpass(cpu, 1));
2949 return (cpu->cpu_m.mcpu_cpi->cpi_chiprevstr);
2950 }
2951
2952 uint32_t
2953 cpuid_getsockettype(struct cpu *cpu)
2954 {
2955 ASSERT(cpuid_checkpass(cpu, 1));
2956 return (cpu->cpu_m.mcpu_cpi->cpi_socket);
2957 }
2958
2959 const char *
2960 cpuid_getsocketstr(cpu_t *cpu)
2961 {
2962 static const char *socketstr = NULL;
2963 struct cpuid_info *cpi;
2964
2965 ASSERT(cpuid_checkpass(cpu, 1));
2966 cpi = cpu->cpu_m.mcpu_cpi;
2967
2968 /* Assume that socket types are the same across the system */
2969 if (socketstr == NULL)
2970 socketstr = _cpuid_sktstr(cpi->cpi_vendor, cpi->cpi_family,
2971 cpi->cpi_model, cpi->cpi_step);
2972
2973
2974 return (socketstr);
2975 }
2976
2977 int
2978 cpuid_get_chipid(cpu_t *cpu)
2979 {
2980 ASSERT(cpuid_checkpass(cpu, 1));
2981
2982 if (cpuid_is_cmt(cpu))
2983 return (cpu->cpu_m.mcpu_cpi->cpi_chipid);
2984 return (cpu->cpu_id);
2985 }
2986
2987 id_t
2988 cpuid_get_coreid(cpu_t *cpu)
2989 {
2990 ASSERT(cpuid_checkpass(cpu, 1));
2991 return (cpu->cpu_m.mcpu_cpi->cpi_coreid);
2992 }
2993
2994 int
2995 cpuid_get_pkgcoreid(cpu_t *cpu)
2996 {
2997 ASSERT(cpuid_checkpass(cpu, 1));
2998 return (cpu->cpu_m.mcpu_cpi->cpi_pkgcoreid);
2999 }
3000
3001 int
3002 cpuid_get_clogid(cpu_t *cpu)
3003 {
3004 ASSERT(cpuid_checkpass(cpu, 1));
3005 return (cpu->cpu_m.mcpu_cpi->cpi_clogid);
3006 }
3007
3008 int
3009 cpuid_get_cacheid(cpu_t *cpu)
3010 {
3011 ASSERT(cpuid_checkpass(cpu, 1));
3012 return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
3013 }
3014
3015 uint_t
3016 cpuid_get_procnodeid(cpu_t *cpu)
3017 {
3018 ASSERT(cpuid_checkpass(cpu, 1));
3019 return (cpu->cpu_m.mcpu_cpi->cpi_procnodeid);
3020 }
3021
3022 uint_t
3023 cpuid_get_procnodes_per_pkg(cpu_t *cpu)
3024 {
3025 ASSERT(cpuid_checkpass(cpu, 1));
3026 return (cpu->cpu_m.mcpu_cpi->cpi_procnodes_per_pkg);
3027 }
3028
3029 uint_t
3030 cpuid_get_compunitid(cpu_t *cpu)
3031 {
3032 ASSERT(cpuid_checkpass(cpu, 1));
3033 return (cpu->cpu_m.mcpu_cpi->cpi_compunitid);
3034 }
3035
3036 uint_t
3037 cpuid_get_cores_per_compunit(cpu_t *cpu)
3038 {
3039 ASSERT(cpuid_checkpass(cpu, 1));
3040 return (cpu->cpu_m.mcpu_cpi->cpi_cores_per_compunit);
3041 }
3042
3043 /*ARGSUSED*/
3044 int
3045 cpuid_have_cr8access(cpu_t *cpu)
3046 {
3047 #if defined(__amd64)
3048 return (1);
3049 #else
3050 struct cpuid_info *cpi;
3051
3052 ASSERT(cpu != NULL);
3053 cpi = cpu->cpu_m.mcpu_cpi;
3054 if (cpi->cpi_vendor == X86_VENDOR_AMD && cpi->cpi_maxeax >= 1 &&
3055 (CPI_FEATURES_XTD_ECX(cpi) & CPUID_AMD_ECX_CR8D) != 0)
3056 return (1);
3057 return (0);
3058 #endif
3059 }
3060
3061 uint32_t
3062 cpuid_get_apicid(cpu_t *cpu)
3063 {
3064 ASSERT(cpuid_checkpass(cpu, 1));
3065 if (cpu->cpu_m.mcpu_cpi->cpi_maxeax < 1) {
3066 return (UINT32_MAX);
3067 } else {
3068 return (cpu->cpu_m.mcpu_cpi->cpi_apicid);
3069 }
3070 }
3071
3072 void
3073 cpuid_get_addrsize(cpu_t *cpu, uint_t *pabits, uint_t *vabits)
3074 {
3075 struct cpuid_info *cpi;
3076
3077 if (cpu == NULL)
3078 cpu = CPU;
3079 cpi = cpu->cpu_m.mcpu_cpi;
3080
3081 ASSERT(cpuid_checkpass(cpu, 1));
3082
3083 if (pabits)
3084 *pabits = cpi->cpi_pabits;
3085 if (vabits)
3086 *vabits = cpi->cpi_vabits;
3087 }
3088
3089 /*
3090 * Returns the number of data TLB entries for a corresponding
3091 * pagesize. If it can't be computed, or isn't known, the
3092 * routine returns zero. If you ask about an architecturally
3093 * impossible pagesize, the routine will panic (so that the
3094 * hat implementor knows that things are inconsistent.)
3095 */
3096 uint_t
3097 cpuid_get_dtlb_nent(cpu_t *cpu, size_t pagesize)
3098 {
3099 struct cpuid_info *cpi;
3100 uint_t dtlb_nent = 0;
3101
3102 if (cpu == NULL)
3103 cpu = CPU;
3104 cpi = cpu->cpu_m.mcpu_cpi;
3105
3106 ASSERT(cpuid_checkpass(cpu, 1));
3107
3108 /*
3109 * Check the L2 TLB info
3110 */
3111 if (cpi->cpi_xmaxeax >= 0x80000006) {
3112 struct cpuid_regs *cp = &cpi->cpi_extd[6];
3113
3114 switch (pagesize) {
3115
3116 case 4 * 1024:
3117 /*
3118 * All zero in the top 16 bits of the register
3119 * indicates a unified TLB. Size is in low 16 bits.
3120 */
3121 if ((cp->cp_ebx & 0xffff0000) == 0)
3122 dtlb_nent = cp->cp_ebx & 0x0000ffff;
3123 else
3124 dtlb_nent = BITX(cp->cp_ebx, 27, 16);
3125 break;
3126
3127 case 2 * 1024 * 1024:
3128 if ((cp->cp_eax & 0xffff0000) == 0)
3129 dtlb_nent = cp->cp_eax & 0x0000ffff;
3130 else
3131 dtlb_nent = BITX(cp->cp_eax, 27, 16);
3132 break;
3133
3134 default:
3135 panic("unknown L2 pagesize");
3136 /*NOTREACHED*/
3137 }
3138 }
3139
3140 if (dtlb_nent != 0)
3141 return (dtlb_nent);
3142
3143 /*
3144 * No L2 TLB support for this size, try L1.
3145 */
3146 if (cpi->cpi_xmaxeax >= 0x80000005) {
3147 struct cpuid_regs *cp = &cpi->cpi_extd[5];
3148
3149 switch (pagesize) {
3150 case 4 * 1024:
3151 dtlb_nent = BITX(cp->cp_ebx, 23, 16);
3152 break;
3153 case 2 * 1024 * 1024:
3154 dtlb_nent = BITX(cp->cp_eax, 23, 16);
3155 break;
3156 default:
3157 panic("unknown L1 d-TLB pagesize");
3158 /*NOTREACHED*/
3159 }
3160 }
3161
3162 return (dtlb_nent);
3163 }
3164
3165 /*
3166 * Return 0 if the erratum is not present or not applicable, positive
3167 * if it is, and negative if the status of the erratum is unknown.
3168 *
3169 * See "Revision Guide for AMD Athlon(tm) 64 and AMD Opteron(tm)
3170 * Processors" #25759, Rev 3.57, August 2005
3171 */
3172 int
3173 cpuid_opteron_erratum(cpu_t *cpu, uint_t erratum)
3174 {
3175 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3176 uint_t eax;
3177
3178 /*
3179 * Bail out if this CPU isn't an AMD CPU, or if it's
3180 * a legacy (32-bit) AMD CPU.
3181 */
3182 if (cpi->cpi_vendor != X86_VENDOR_AMD ||
3183 cpi->cpi_family == 4 || cpi->cpi_family == 5 ||
3184 cpi->cpi_family == 6)
3185
3186 return (0);
3187
3188 eax = cpi->cpi_std[1].cp_eax;
3189
3190 #define SH_B0(eax) (eax == 0xf40 || eax == 0xf50)
3191 #define SH_B3(eax) (eax == 0xf51)
3192 #define B(eax) (SH_B0(eax) || SH_B3(eax))
3193
3194 #define SH_C0(eax) (eax == 0xf48 || eax == 0xf58)
3195
3196 #define SH_CG(eax) (eax == 0xf4a || eax == 0xf5a || eax == 0xf7a)
3197 #define DH_CG(eax) (eax == 0xfc0 || eax == 0xfe0 || eax == 0xff0)
3198 #define CH_CG(eax) (eax == 0xf82 || eax == 0xfb2)
3199 #define CG(eax) (SH_CG(eax) || DH_CG(eax) || CH_CG(eax))
3200
3201 #define SH_D0(eax) (eax == 0x10f40 || eax == 0x10f50 || eax == 0x10f70)
3202 #define DH_D0(eax) (eax == 0x10fc0 || eax == 0x10ff0)
3203 #define CH_D0(eax) (eax == 0x10f80 || eax == 0x10fb0)
3204 #define D0(eax) (SH_D0(eax) || DH_D0(eax) || CH_D0(eax))
3205
3206 #define SH_E0(eax) (eax == 0x20f50 || eax == 0x20f40 || eax == 0x20f70)
3207 #define JH_E1(eax) (eax == 0x20f10) /* JH8_E0 had 0x20f30 */
3208 #define DH_E3(eax) (eax == 0x20fc0 || eax == 0x20ff0)
3209 #define SH_E4(eax) (eax == 0x20f51 || eax == 0x20f71)
3210 #define BH_E4(eax) (eax == 0x20fb1)
3211 #define SH_E5(eax) (eax == 0x20f42)
3212 #define DH_E6(eax) (eax == 0x20ff2 || eax == 0x20fc2)
3213 #define JH_E6(eax) (eax == 0x20f12 || eax == 0x20f32)
3214 #define EX(eax) (SH_E0(eax) || JH_E1(eax) || DH_E3(eax) || \
3215 SH_E4(eax) || BH_E4(eax) || SH_E5(eax) || \
3216 DH_E6(eax) || JH_E6(eax))
3217
3218 #define DR_AX(eax) (eax == 0x100f00 || eax == 0x100f01 || eax == 0x100f02)
3219 #define DR_B0(eax) (eax == 0x100f20)
3220 #define DR_B1(eax) (eax == 0x100f21)
3221 #define DR_BA(eax) (eax == 0x100f2a)
3222 #define DR_B2(eax) (eax == 0x100f22)
3223 #define DR_B3(eax) (eax == 0x100f23)
3224 #define RB_C0(eax) (eax == 0x100f40)
3225
3226 switch (erratum) {
3227 case 1:
3228 return (cpi->cpi_family < 0x10);
3229 case 51: /* what does the asterisk mean? */
3230 return (B(eax) || SH_C0(eax) || CG(eax));
3231 case 52:
3232 return (B(eax));
3233 case 57:
3234 return (cpi->cpi_family <= 0x11);
3235 case 58:
3236 return (B(eax));
3237 case 60:
3238 return (cpi->cpi_family <= 0x11);
3239 case 61:
3240 case 62:
3241 case 63:
3242 case 64:
3243 case 65:
3244 case 66:
3245 case 68:
3246 case 69:
3247 case 70:
3248 case 71:
3249 return (B(eax));
3250 case 72:
3251 return (SH_B0(eax));
3252 case 74:
3253 return (B(eax));
3254 case 75:
3255 return (cpi->cpi_family < 0x10);
3256 case 76:
3257 return (B(eax));
3258 case 77:
3259 return (cpi->cpi_family <= 0x11);
3260 case 78:
3261 return (B(eax) || SH_C0(eax));
3262 case 79:
3263 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3264 case 80:
3265 case 81:
3266 case 82:
3267 return (B(eax));
3268 case 83:
3269 return (B(eax) || SH_C0(eax) || CG(eax));
3270 case 85:
3271 return (cpi->cpi_family < 0x10);
3272 case 86:
3273 return (SH_C0(eax) || CG(eax));
3274 case 88:
3275 #if !defined(__amd64)
3276 return (0);
3277 #else
3278 return (B(eax) || SH_C0(eax));
3279 #endif
3280 case 89:
3281 return (cpi->cpi_family < 0x10);
3282 case 90:
3283 return (B(eax) || SH_C0(eax) || CG(eax));
3284 case 91:
3285 case 92:
3286 return (B(eax) || SH_C0(eax));
3287 case 93:
3288 return (SH_C0(eax));
3289 case 94:
3290 return (B(eax) || SH_C0(eax) || CG(eax));
3291 case 95:
3292 #if !defined(__amd64)
3293 return (0);
3294 #else
3295 return (B(eax) || SH_C0(eax));
3296 #endif
3297 case 96:
3298 return (B(eax) || SH_C0(eax) || CG(eax));
3299 case 97:
3300 case 98:
3301 return (SH_C0(eax) || CG(eax));
3302 case 99:
3303 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3304 case 100:
3305 return (B(eax) || SH_C0(eax));
3306 case 101:
3307 case 103:
3308 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3309 case 104:
3310 return (SH_C0(eax) || CG(eax) || D0(eax));
3311 case 105:
3312 case 106:
3313 case 107:
3314 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3315 case 108:
3316 return (DH_CG(eax));
3317 case 109:
3318 return (SH_C0(eax) || CG(eax) || D0(eax));
3319 case 110:
3320 return (D0(eax) || EX(eax));
3321 case 111:
3322 return (CG(eax));
3323 case 112:
3324 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3325 case 113:
3326 return (eax == 0x20fc0);
3327 case 114:
3328 return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
3329 case 115:
3330 return (SH_E0(eax) || JH_E1(eax));
3331 case 116:
3332 return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
3333 case 117:
3334 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
3335 case 118:
3336 return (SH_E0(eax) || JH_E1(eax) || SH_E4(eax) || BH_E4(eax) ||
3337 JH_E6(eax));
3338 case 121:
3339 return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
3340 case 122:
3341 return (cpi->cpi_family < 0x10 || cpi->cpi_family == 0x11);
3342 case 123:
3343 return (JH_E1(eax) || BH_E4(eax) || JH_E6(eax));
3344 case 131:
3345 return (cpi->cpi_family < 0x10);
3346 case 6336786:
3347 /*
3348 * Test for AdvPowerMgmtInfo.TscPStateInvariant
3349 * if this is a K8 family or newer processor
3350 */
3351 if (CPI_FAMILY(cpi) == 0xf) {
3352 struct cpuid_regs regs;
3353 regs.cp_eax = 0x80000007;
3354 (void) __cpuid_insn(®s);
3355 return (!(regs.cp_edx & 0x100));
3356 }
3357 return (0);
3358 case 6323525:
3359 return (((((eax >> 12) & 0xff00) + (eax & 0xf00)) |
3360 (((eax >> 4) & 0xf) | ((eax >> 12) & 0xf0))) < 0xf40);
3361
3362 case 6671130:
3363 /*
3364 * check for processors (pre-Shanghai) that do not provide
3365 * optimal management of 1gb ptes in its tlb.
3366 */
3367 return (cpi->cpi_family == 0x10 && cpi->cpi_model < 4);
3368
3369 case 298:
3370 return (DR_AX(eax) || DR_B0(eax) || DR_B1(eax) || DR_BA(eax) ||
3371 DR_B2(eax) || RB_C0(eax));
3372
3373 case 721:
3374 #if defined(__amd64)
3375 return (cpi->cpi_family == 0x10 || cpi->cpi_family == 0x12);
3376 #else
3377 return (0);
3378 #endif
3379
3380 default:
3381 return (-1);
3382
3383 }
3384 }
3385
3386 /*
3387 * Determine if specified erratum is present via OSVW (OS Visible Workaround).
3388 * Return 1 if erratum is present, 0 if not present and -1 if indeterminate.
3389 */
3390 int
3391 osvw_opteron_erratum(cpu_t *cpu, uint_t erratum)
3392 {
3393 struct cpuid_info *cpi;
3394 uint_t osvwid;
3395 static int osvwfeature = -1;
3396 uint64_t osvwlength;
3397
3398
3399 cpi = cpu->cpu_m.mcpu_cpi;
3400
3401 /* confirm OSVW supported */
3402 if (osvwfeature == -1) {
3403 osvwfeature = cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW;
3404 } else {
3405 /* assert that osvw feature setting is consistent on all cpus */
3406 ASSERT(osvwfeature ==
3407 (cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW));
3408 }
3409 if (!osvwfeature)
3410 return (-1);
3411
3412 osvwlength = rdmsr(MSR_AMD_OSVW_ID_LEN) & OSVW_ID_LEN_MASK;
3413
3414 switch (erratum) {
3415 case 298: /* osvwid is 0 */
3416 osvwid = 0;
3417 if (osvwlength <= (uint64_t)osvwid) {
3418 /* osvwid 0 is unknown */
3419 return (-1);
3420 }
3421
3422 /*
3423 * Check the OSVW STATUS MSR to determine the state
3424 * of the erratum where:
3425 * 0 - fixed by HW
3426 * 1 - BIOS has applied the workaround when BIOS
3427 * workaround is available. (Or for other errata,
3428 * OS workaround is required.)
3429 * For a value of 1, caller will confirm that the
3430 * erratum 298 workaround has indeed been applied by BIOS.
3431 *
3432 * A 1 may be set in cpus that have a HW fix
3433 * in a mixed cpu system. Regarding erratum 298:
3434 * In a multiprocessor platform, the workaround above
3435 * should be applied to all processors regardless of
3436 * silicon revision when an affected processor is
3437 * present.
3438 */
3439
3440 return (rdmsr(MSR_AMD_OSVW_STATUS +
3441 (osvwid / OSVW_ID_CNT_PER_MSR)) &
3442 (1ULL << (osvwid % OSVW_ID_CNT_PER_MSR)));
3443
3444 default:
3445 return (-1);
3446 }
3447 }
3448
3449 static const char assoc_str[] = "associativity";
3450 static const char line_str[] = "line-size";
3451 static const char size_str[] = "size";
3452
3453 static void
3454 add_cache_prop(dev_info_t *devi, const char *label, const char *type,
3455 uint32_t val)
3456 {
3457 char buf[128];
3458
3459 /*
3460 * ndi_prop_update_int() is used because it is desirable for
3461 * DDI_PROP_HW_DEF and DDI_PROP_DONTSLEEP to be set.
3462 */
3463 if (snprintf(buf, sizeof (buf), "%s-%s", label, type) < sizeof (buf))
3464 (void) ndi_prop_update_int(DDI_DEV_T_NONE, devi, buf, val);
3465 }
3466
3467 /*
3468 * Intel-style cache/tlb description
3469 *
3470 * Standard cpuid level 2 gives a randomly ordered
3471 * selection of tags that index into a table that describes
3472 * cache and tlb properties.
3473 */
3474
3475 static const char l1_icache_str[] = "l1-icache";
3476 static const char l1_dcache_str[] = "l1-dcache";
3477 static const char l2_cache_str[] = "l2-cache";
3478 static const char l3_cache_str[] = "l3-cache";
3479 static const char itlb4k_str[] = "itlb-4K";
3480 static const char dtlb4k_str[] = "dtlb-4K";
3481 static const char itlb2M_str[] = "itlb-2M";
3482 static const char itlb4M_str[] = "itlb-4M";
3483 static const char dtlb4M_str[] = "dtlb-4M";
3484 static const char dtlb24_str[] = "dtlb0-2M-4M";
3485 static const char itlb424_str[] = "itlb-4K-2M-4M";
3486 static const char itlb24_str[] = "itlb-2M-4M";
3487 static const char dtlb44_str[] = "dtlb-4K-4M";
3488 static const char sl1_dcache_str[] = "sectored-l1-dcache";
3489 static const char sl2_cache_str[] = "sectored-l2-cache";
3490 static const char itrace_str[] = "itrace-cache";
3491 static const char sl3_cache_str[] = "sectored-l3-cache";
3492 static const char sh_l2_tlb4k_str[] = "shared-l2-tlb-4k";
3493
3494 static const struct cachetab {
3495 uint8_t ct_code;
3496 uint8_t ct_assoc;
3497 uint16_t ct_line_size;
3498 size_t ct_size;
3499 const char *ct_label;
3500 } intel_ctab[] = {
3501 /*
3502 * maintain descending order!
3503 *
3504 * Codes ignored - Reason
3505 * ----------------------
3506 * 40H - intel_cpuid_4_cache_info() disambiguates l2/l3 cache
3507 * f0H/f1H - Currently we do not interpret prefetch size by design
3508 */
3509 { 0xe4, 16, 64, 8*1024*1024, l3_cache_str},
3510 { 0xe3, 16, 64, 4*1024*1024, l3_cache_str},
3511 { 0xe2, 16, 64, 2*1024*1024, l3_cache_str},
3512 { 0xde, 12, 64, 6*1024*1024, l3_cache_str},
3513 { 0xdd, 12, 64, 3*1024*1024, l3_cache_str},
3514 { 0xdc, 12, 64, ((1*1024*1024)+(512*1024)), l3_cache_str},
3515 { 0xd8, 8, 64, 4*1024*1024, l3_cache_str},
3516 { 0xd7, 8, 64, 2*1024*1024, l3_cache_str},
3517 { 0xd6, 8, 64, 1*1024*1024, l3_cache_str},
3518 { 0xd2, 4, 64, 2*1024*1024, l3_cache_str},
3519 { 0xd1, 4, 64, 1*1024*1024, l3_cache_str},
3520 { 0xd0, 4, 64, 512*1024, l3_cache_str},
3521 { 0xca, 4, 0, 512, sh_l2_tlb4k_str},
3522 { 0xc0, 4, 0, 8, dtlb44_str },
3523 { 0xba, 4, 0, 64, dtlb4k_str },
3524 { 0xb4, 4, 0, 256, dtlb4k_str },
3525 { 0xb3, 4, 0, 128, dtlb4k_str },
3526 { 0xb2, 4, 0, 64, itlb4k_str },
3527 { 0xb0, 4, 0, 128, itlb4k_str },
3528 { 0x87, 8, 64, 1024*1024, l2_cache_str},
3529 { 0x86, 4, 64, 512*1024, l2_cache_str},
3530 { 0x85, 8, 32, 2*1024*1024, l2_cache_str},
3531 { 0x84, 8, 32, 1024*1024, l2_cache_str},
3532 { 0x83, 8, 32, 512*1024, l2_cache_str},
3533 { 0x82, 8, 32, 256*1024, l2_cache_str},
3534 { 0x80, 8, 64, 512*1024, l2_cache_str},
3535 { 0x7f, 2, 64, 512*1024, l2_cache_str},
3536 { 0x7d, 8, 64, 2*1024*1024, sl2_cache_str},
3537 { 0x7c, 8, 64, 1024*1024, sl2_cache_str},
3538 { 0x7b, 8, 64, 512*1024, sl2_cache_str},
3539 { 0x7a, 8, 64, 256*1024, sl2_cache_str},
3540 { 0x79, 8, 64, 128*1024, sl2_cache_str},
3541 { 0x78, 8, 64, 1024*1024, l2_cache_str},
3542 { 0x73, 8, 0, 64*1024, itrace_str},
3543 { 0x72, 8, 0, 32*1024, itrace_str},
3544 { 0x71, 8, 0, 16*1024, itrace_str},
3545 { 0x70, 8, 0, 12*1024, itrace_str},
3546 { 0x68, 4, 64, 32*1024, sl1_dcache_str},
3547 { 0x67, 4, 64, 16*1024, sl1_dcache_str},
3548 { 0x66, 4, 64, 8*1024, sl1_dcache_str},
3549 { 0x60, 8, 64, 16*1024, sl1_dcache_str},
3550 { 0x5d, 0, 0, 256, dtlb44_str},
3551 { 0x5c, 0, 0, 128, dtlb44_str},
3552 { 0x5b, 0, 0, 64, dtlb44_str},
3553 { 0x5a, 4, 0, 32, dtlb24_str},
3554 { 0x59, 0, 0, 16, dtlb4k_str},
3555 { 0x57, 4, 0, 16, dtlb4k_str},
3556 { 0x56, 4, 0, 16, dtlb4M_str},
3557 { 0x55, 0, 0, 7, itlb24_str},
3558 { 0x52, 0, 0, 256, itlb424_str},
3559 { 0x51, 0, 0, 128, itlb424_str},
3560 { 0x50, 0, 0, 64, itlb424_str},
3561 { 0x4f, 0, 0, 32, itlb4k_str},
3562 { 0x4e, 24, 64, 6*1024*1024, l2_cache_str},
3563 { 0x4d, 16, 64, 16*1024*1024, l3_cache_str},
3564 { 0x4c, 12, 64, 12*1024*1024, l3_cache_str},
3565 { 0x4b, 16, 64, 8*1024*1024, l3_cache_str},
3566 { 0x4a, 12, 64, 6*1024*1024, l3_cache_str},
3567 { 0x49, 16, 64, 4*1024*1024, l3_cache_str},
3568 { 0x48, 12, 64, 3*1024*1024, l2_cache_str},
3569 { 0x47, 8, 64, 8*1024*1024, l3_cache_str},
3570 { 0x46, 4, 64, 4*1024*1024, l3_cache_str},
3571 { 0x45, 4, 32, 2*1024*1024, l2_cache_str},
3572 { 0x44, 4, 32, 1024*1024, l2_cache_str},
3573 { 0x43, 4, 32, 512*1024, l2_cache_str},
3574 { 0x42, 4, 32, 256*1024, l2_cache_str},
3575 { 0x41, 4, 32, 128*1024, l2_cache_str},
3576 { 0x3e, 4, 64, 512*1024, sl2_cache_str},
3577 { 0x3d, 6, 64, 384*1024, sl2_cache_str},
3578 { 0x3c, 4, 64, 256*1024, sl2_cache_str},
3579 { 0x3b, 2, 64, 128*1024, sl2_cache_str},
3580 { 0x3a, 6, 64, 192*1024, sl2_cache_str},
3581 { 0x39, 4, 64, 128*1024, sl2_cache_str},
3582 { 0x30, 8, 64, 32*1024, l1_icache_str},
3583 { 0x2c, 8, 64, 32*1024, l1_dcache_str},
3584 { 0x29, 8, 64, 4096*1024, sl3_cache_str},
3585 { 0x25, 8, 64, 2048*1024, sl3_cache_str},
3586 { 0x23, 8, 64, 1024*1024, sl3_cache_str},
3587 { 0x22, 4, 64, 512*1024, sl3_cache_str},
3588 { 0x0e, 6, 64, 24*1024, l1_dcache_str},
3589 { 0x0d, 4, 32, 16*1024, l1_dcache_str},
3590 { 0x0c, 4, 32, 16*1024, l1_dcache_str},
3591 { 0x0b, 4, 0, 4, itlb4M_str},
3592 { 0x0a, 2, 32, 8*1024, l1_dcache_str},
3593 { 0x08, 4, 32, 16*1024, l1_icache_str},
3594 { 0x06, 4, 32, 8*1024, l1_icache_str},
3595 { 0x05, 4, 0, 32, dtlb4M_str},
3596 { 0x04, 4, 0, 8, dtlb4M_str},
3597 { 0x03, 4, 0, 64, dtlb4k_str},
3598 { 0x02, 4, 0, 2, itlb4M_str},
3599 { 0x01, 4, 0, 32, itlb4k_str},
3600 { 0 }
3601 };
3602
3603 static const struct cachetab cyrix_ctab[] = {
3604 { 0x70, 4, 0, 32, "tlb-4K" },
3605 { 0x80, 4, 16, 16*1024, "l1-cache" },
3606 { 0 }
3607 };
3608
3609 /*
3610 * Search a cache table for a matching entry
3611 */
3612 static const struct cachetab *
3613 find_cacheent(const struct cachetab *ct, uint_t code)
3614 {
3615 if (code != 0) {
3616 for (; ct->ct_code != 0; ct++)
3617 if (ct->ct_code <= code)
3618 break;
3619 if (ct->ct_code == code)
3620 return (ct);
3621 }
3622 return (NULL);
3623 }
3624
3625 /*
3626 * Populate cachetab entry with L2 or L3 cache-information using
3627 * cpuid function 4. This function is called from intel_walk_cacheinfo()
3628 * when descriptor 0x49 is encountered. It returns 0 if no such cache
3629 * information is found.
3630 */
3631 static int
3632 intel_cpuid_4_cache_info(struct cachetab *ct, struct cpuid_info *cpi)
3633 {
3634 uint32_t level, i;
3635 int ret = 0;
3636
3637 for (i = 0; i < cpi->cpi_std_4_size; i++) {
3638 level = CPI_CACHE_LVL(cpi->cpi_std_4[i]);
3639
3640 if (level == 2 || level == 3) {
3641 ct->ct_assoc = CPI_CACHE_WAYS(cpi->cpi_std_4[i]) + 1;
3642 ct->ct_line_size =
3643 CPI_CACHE_COH_LN_SZ(cpi->cpi_std_4[i]) + 1;
3644 ct->ct_size = ct->ct_assoc *
3645 (CPI_CACHE_PARTS(cpi->cpi_std_4[i]) + 1) *
3646 ct->ct_line_size *
3647 (cpi->cpi_std_4[i]->cp_ecx + 1);
3648
3649 if (level == 2) {
3650 ct->ct_label = l2_cache_str;
3651 } else if (level == 3) {
3652 ct->ct_label = l3_cache_str;
3653 }
3654 ret = 1;
3655 }
3656 }
3657
3658 return (ret);
3659 }
3660
3661 /*
3662 * Walk the cacheinfo descriptor, applying 'func' to every valid element
3663 * The walk is terminated if the walker returns non-zero.
3664 */
3665 static void
3666 intel_walk_cacheinfo(struct cpuid_info *cpi,
3667 void *arg, int (*func)(void *, const struct cachetab *))
3668 {
3669 const struct cachetab *ct;
3670 struct cachetab des_49_ct, des_b1_ct;
3671 uint8_t *dp;
3672 int i;
3673
3674 if ((dp = cpi->cpi_cacheinfo) == NULL)
3675 return;
3676 for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
3677 /*
3678 * For overloaded descriptor 0x49 we use cpuid function 4
3679 * if supported by the current processor, to create
3680 * cache information.
3681 * For overloaded descriptor 0xb1 we use X86_PAE flag
3682 * to disambiguate the cache information.
3683 */
3684 if (*dp == 0x49 && cpi->cpi_maxeax >= 0x4 &&
3685 intel_cpuid_4_cache_info(&des_49_ct, cpi) == 1) {
3686 ct = &des_49_ct;
3687 } else if (*dp == 0xb1) {
3688 des_b1_ct.ct_code = 0xb1;
3689 des_b1_ct.ct_assoc = 4;
3690 des_b1_ct.ct_line_size = 0;
3691 if (is_x86_feature(x86_featureset, X86FSET_PAE)) {
3692 des_b1_ct.ct_size = 8;
3693 des_b1_ct.ct_label = itlb2M_str;
3694 } else {
3695 des_b1_ct.ct_size = 4;
3696 des_b1_ct.ct_label = itlb4M_str;
3697 }
3698 ct = &des_b1_ct;
3699 } else {
3700 if ((ct = find_cacheent(intel_ctab, *dp)) == NULL) {
3701 continue;
3702 }
3703 }
3704
3705 if (func(arg, ct) != 0) {
3706 break;
3707 }
3708 }
3709 }
3710
3711 /*
3712 * (Like the Intel one, except for Cyrix CPUs)
3713 */
3714 static void
3715 cyrix_walk_cacheinfo(struct cpuid_info *cpi,
3716 void *arg, int (*func)(void *, const struct cachetab *))
3717 {
3718 const struct cachetab *ct;
3719 uint8_t *dp;
3720 int i;
3721
3722 if ((dp = cpi->cpi_cacheinfo) == NULL)
3723 return;
3724 for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
3725 /*
3726 * Search Cyrix-specific descriptor table first ..
3727 */
3728 if ((ct = find_cacheent(cyrix_ctab, *dp)) != NULL) {
3729 if (func(arg, ct) != 0)
3730 break;
3731 continue;
3732 }
3733 /*
3734 * .. else fall back to the Intel one
3735 */
3736 if ((ct = find_cacheent(intel_ctab, *dp)) != NULL) {
3737 if (func(arg, ct) != 0)
3738 break;
3739 continue;
3740 }
3741 }
3742 }
3743
3744 /*
3745 * A cacheinfo walker that adds associativity, line-size, and size properties
3746 * to the devinfo node it is passed as an argument.
3747 */
3748 static int
3749 add_cacheent_props(void *arg, const struct cachetab *ct)
3750 {
3751 dev_info_t *devi = arg;
3752
3753 add_cache_prop(devi, ct->ct_label, assoc_str, ct->ct_assoc);
3754 if (ct->ct_line_size != 0)
3755 add_cache_prop(devi, ct->ct_label, line_str,
3756 ct->ct_line_size);
3757 add_cache_prop(devi, ct->ct_label, size_str, ct->ct_size);
3758 return (0);
3759 }
3760
3761
3762 static const char fully_assoc[] = "fully-associative?";
3763
3764 /*
3765 * AMD style cache/tlb description
3766 *
3767 * Extended functions 5 and 6 directly describe properties of
3768 * tlbs and various cache levels.
3769 */
3770 static void
3771 add_amd_assoc(dev_info_t *devi, const char *label, uint_t assoc)
3772 {
3773 switch (assoc) {
3774 case 0: /* reserved; ignore */
3775 break;
3776 default:
3777 add_cache_prop(devi, label, assoc_str, assoc);
3778 break;
3779 case 0xff:
3780 add_cache_prop(devi, label, fully_assoc, 1);
3781 break;
3782 }
3783 }
3784
3785 static void
3786 add_amd_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
3787 {
3788 if (size == 0)
3789 return;
3790 add_cache_prop(devi, label, size_str, size);
3791 add_amd_assoc(devi, label, assoc);
3792 }
3793
3794 static void
3795 add_amd_cache(dev_info_t *devi, const char *label,
3796 uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
3797 {
3798 if (size == 0 || line_size == 0)
3799 return;
3800 add_amd_assoc(devi, label, assoc);
3801 /*
3802 * Most AMD parts have a sectored cache. Multiple cache lines are
3803 * associated with each tag. A sector consists of all cache lines
3804 * associated with a tag. For example, the AMD K6-III has a sector
3805 * size of 2 cache lines per tag.
3806 */
3807 if (lines_per_tag != 0)
3808 add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
3809 add_cache_prop(devi, label, line_str, line_size);
3810 add_cache_prop(devi, label, size_str, size * 1024);
3811 }
3812
3813 static void
3814 add_amd_l2_assoc(dev_info_t *devi, const char *label, uint_t assoc)
3815 {
3816 switch (assoc) {
3817 case 0: /* off */
3818 break;
3819 case 1:
3820 case 2:
3821 case 4:
3822 add_cache_prop(devi, label, assoc_str, assoc);
3823 break;
3824 case 6:
3825 add_cache_prop(devi, label, assoc_str, 8);
3826 break;
3827 case 8:
3828 add_cache_prop(devi, label, assoc_str, 16);
3829 break;
3830 case 0xf:
3831 add_cache_prop(devi, label, fully_assoc, 1);
3832 break;
3833 default: /* reserved; ignore */
3834 break;
3835 }
3836 }
3837
3838 static void
3839 add_amd_l2_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
3840 {
3841 if (size == 0 || assoc == 0)
3842 return;
3843 add_amd_l2_assoc(devi, label, assoc);
3844 add_cache_prop(devi, label, size_str, size);
3845 }
3846
3847 static void
3848 add_amd_l2_cache(dev_info_t *devi, const char *label,
3849 uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
3850 {
3851 if (size == 0 || assoc == 0 || line_size == 0)
3852 return;
3853 add_amd_l2_assoc(devi, label, assoc);
3854 if (lines_per_tag != 0)
3855 add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
3856 add_cache_prop(devi, label, line_str, line_size);
3857 add_cache_prop(devi, label, size_str, size * 1024);
3858 }
3859
3860 static void
3861 amd_cache_info(struct cpuid_info *cpi, dev_info_t *devi)
3862 {
3863 struct cpuid_regs *cp;
3864
3865 if (cpi->cpi_xmaxeax < 0x80000005)
3866 return;
3867 cp = &cpi->cpi_extd[5];
3868
3869 /*
3870 * 4M/2M L1 TLB configuration
3871 *
3872 * We report the size for 2M pages because AMD uses two
3873 * TLB entries for one 4M page.
3874 */
3875 add_amd_tlb(devi, "dtlb-2M",
3876 BITX(cp->cp_eax, 31, 24), BITX(cp->cp_eax, 23, 16));
3877 add_amd_tlb(devi, "itlb-2M",
3878 BITX(cp->cp_eax, 15, 8), BITX(cp->cp_eax, 7, 0));
3879
3880 /*
3881 * 4K L1 TLB configuration
3882 */
3883
3884 switch (cpi->cpi_vendor) {
3885 uint_t nentries;
3886 case X86_VENDOR_TM:
3887 if (cpi->cpi_family >= 5) {
3888 /*
3889 * Crusoe processors have 256 TLB entries, but
3890 * cpuid data format constrains them to only
3891 * reporting 255 of them.
3892 */
3893 if ((nentries = BITX(cp->cp_ebx, 23, 16)) == 255)
3894 nentries = 256;
3895 /*
3896 * Crusoe processors also have a unified TLB
3897 */
3898 add_amd_tlb(devi, "tlb-4K", BITX(cp->cp_ebx, 31, 24),
3899 nentries);
3900 break;
3901 }
3902 /*FALLTHROUGH*/
3903 default:
3904 add_amd_tlb(devi, itlb4k_str,
3905 BITX(cp->cp_ebx, 31, 24), BITX(cp->cp_ebx, 23, 16));
3906 add_amd_tlb(devi, dtlb4k_str,
3907 BITX(cp->cp_ebx, 15, 8), BITX(cp->cp_ebx, 7, 0));
3908 break;
3909 }
3910
3911 /*
3912 * data L1 cache configuration
3913 */
3914
3915 add_amd_cache(devi, l1_dcache_str,
3916 BITX(cp->cp_ecx, 31, 24), BITX(cp->cp_ecx, 23, 16),
3917 BITX(cp->cp_ecx, 15, 8), BITX(cp->cp_ecx, 7, 0));
3918
3919 /*
3920 * code L1 cache configuration
3921 */
3922
3923 add_amd_cache(devi, l1_icache_str,
3924 BITX(cp->cp_edx, 31, 24), BITX(cp->cp_edx, 23, 16),
3925 BITX(cp->cp_edx, 15, 8), BITX(cp->cp_edx, 7, 0));
3926
3927 if (cpi->cpi_xmaxeax < 0x80000006)
3928 return;
3929 cp = &cpi->cpi_extd[6];
3930
3931 /* Check for a unified L2 TLB for large pages */
3932
3933 if (BITX(cp->cp_eax, 31, 16) == 0)
3934 add_amd_l2_tlb(devi, "l2-tlb-2M",
3935 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
3936 else {
3937 add_amd_l2_tlb(devi, "l2-dtlb-2M",
3938 BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
3939 add_amd_l2_tlb(devi, "l2-itlb-2M",
3940 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
3941 }
3942
3943 /* Check for a unified L2 TLB for 4K pages */
3944
3945 if (BITX(cp->cp_ebx, 31, 16) == 0) {
3946 add_amd_l2_tlb(devi, "l2-tlb-4K",
3947 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
3948 } else {
3949 add_amd_l2_tlb(devi, "l2-dtlb-4K",
3950 BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
3951 add_amd_l2_tlb(devi, "l2-itlb-4K",
3952 BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
3953 }
3954
3955 add_amd_l2_cache(devi, l2_cache_str,
3956 BITX(cp->cp_ecx, 31, 16), BITX(cp->cp_ecx, 15, 12),
3957 BITX(cp->cp_ecx, 11, 8), BITX(cp->cp_ecx, 7, 0));
3958 }
3959
3960 /*
3961 * There are two basic ways that the x86 world describes it cache
3962 * and tlb architecture - Intel's way and AMD's way.
3963 *
3964 * Return which flavor of cache architecture we should use
3965 */
3966 static int
3967 x86_which_cacheinfo(struct cpuid_info *cpi)
3968 {
3969 switch (cpi->cpi_vendor) {
3970 case X86_VENDOR_Intel:
3971 if (cpi->cpi_maxeax >= 2)
3972 return (X86_VENDOR_Intel);
3973 break;
3974 case X86_VENDOR_AMD:
3975 /*
3976 * The K5 model 1 was the first part from AMD that reported
3977 * cache sizes via extended cpuid functions.
3978 */
3979 if (cpi->cpi_family > 5 ||
3980 (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
3981 return (X86_VENDOR_AMD);
3982 break;
3983 case X86_VENDOR_TM:
3984 if (cpi->cpi_family >= 5)
3985 return (X86_VENDOR_AMD);
3986 /*FALLTHROUGH*/
3987 default:
3988 /*
3989 * If they have extended CPU data for 0x80000005
3990 * then we assume they have AMD-format cache
3991 * information.
3992 *
3993 * If not, and the vendor happens to be Cyrix,
3994 * then try our-Cyrix specific handler.
3995 *
3996 * If we're not Cyrix, then assume we're using Intel's
3997 * table-driven format instead.
3998 */
3999 if (cpi->cpi_xmaxeax >= 0x80000005)
4000 return (X86_VENDOR_AMD);
4001 else if (cpi->cpi_vendor == X86_VENDOR_Cyrix)
4002 return (X86_VENDOR_Cyrix);
4003 else if (cpi->cpi_maxeax >= 2)
4004 return (X86_VENDOR_Intel);
4005 break;
4006 }
4007 return (-1);
4008 }
4009
4010 void
4011 cpuid_set_cpu_properties(void *dip, processorid_t cpu_id,
4012 struct cpuid_info *cpi)
4013 {
4014 dev_info_t *cpu_devi;
4015 int create;
4016
4017 cpu_devi = (dev_info_t *)dip;
4018
4019 /* device_type */
4020 (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4021 "device_type", "cpu");
4022
4023 /* reg */
4024 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4025 "reg", cpu_id);
4026
4027 /* cpu-mhz, and clock-frequency */
4028 if (cpu_freq > 0) {
4029 long long mul;
4030
4031 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4032 "cpu-mhz", cpu_freq);
4033 if ((mul = cpu_freq * 1000000LL) <= INT_MAX)
4034 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4035 "clock-frequency", (int)mul);
4036 }
4037
4038 if (!is_x86_feature(x86_featureset, X86FSET_CPUID)) {
4039 return;
4040 }
4041
4042 /* vendor-id */
4043 (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4044 "vendor-id", cpi->cpi_vendorstr);
4045
4046 if (cpi->cpi_maxeax == 0) {
4047 return;
4048 }
4049
4050 /*
4051 * family, model, and step
4052 */
4053 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4054 "family", CPI_FAMILY(cpi));
4055 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4056 "cpu-model", CPI_MODEL(cpi));
4057 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4058 "stepping-id", CPI_STEP(cpi));
4059
4060 /* type */
4061 switch (cpi->cpi_vendor) {
4062 case X86_VENDOR_Intel:
4063 create = 1;
4064 break;
4065 default:
4066 create = 0;
4067 break;
4068 }
4069 if (create)
4070 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4071 "type", CPI_TYPE(cpi));
4072
4073 /* ext-family */
4074 switch (cpi->cpi_vendor) {
4075 case X86_VENDOR_Intel:
4076 case X86_VENDOR_AMD:
4077 create = cpi->cpi_family >= 0xf;
4078 break;
4079 default:
4080 create = 0;
4081 break;
4082 }
4083 if (create)
4084 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4085 "ext-family", CPI_FAMILY_XTD(cpi));
4086
4087 /* ext-model */
4088 switch (cpi->cpi_vendor) {
4089 case X86_VENDOR_Intel:
4090 create = IS_EXTENDED_MODEL_INTEL(cpi);
4091 break;
4092 case X86_VENDOR_AMD:
4093 create = CPI_FAMILY(cpi) == 0xf;
4094 break;
4095 default:
4096 create = 0;
4097 break;
4098 }
4099 if (create)
4100 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4101 "ext-model", CPI_MODEL_XTD(cpi));
4102
4103 /* generation */
4104 switch (cpi->cpi_vendor) {
4105 case X86_VENDOR_AMD:
4106 /*
4107 * AMD K5 model 1 was the first part to support this
4108 */
4109 create = cpi->cpi_xmaxeax >= 0x80000001;
4110 break;
4111 default:
4112 create = 0;
4113 break;
4114 }
4115 if (create)
4116 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4117 "generation", BITX((cpi)->cpi_extd[1].cp_eax, 11, 8));
4118
4119 /* brand-id */
4120 switch (cpi->cpi_vendor) {
4121 case X86_VENDOR_Intel:
4122 /*
4123 * brand id first appeared on Pentium III Xeon model 8,
4124 * and Celeron model 8 processors and Opteron
4125 */
4126 create = cpi->cpi_family > 6 ||
4127 (cpi->cpi_family == 6 && cpi->cpi_model >= 8);
4128 break;
4129 case X86_VENDOR_AMD:
4130 create = cpi->cpi_family >= 0xf;
4131 break;
4132 default:
4133 create = 0;
4134 break;
4135 }
4136 if (create && cpi->cpi_brandid != 0) {
4137 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4138 "brand-id", cpi->cpi_brandid);
4139 }
4140
4141 /* chunks, and apic-id */
4142 switch (cpi->cpi_vendor) {
4143 /*
4144 * first available on Pentium IV and Opteron (K8)
4145 */
4146 case X86_VENDOR_Intel:
4147 create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
4148 break;
4149 case X86_VENDOR_AMD:
4150 create = cpi->cpi_family >= 0xf;
4151 break;
4152 default:
4153 create = 0;
4154 break;
4155 }
4156 if (create) {
4157 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4158 "chunks", CPI_CHUNKS(cpi));
4159 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4160 "apic-id", cpi->cpi_apicid);
4161 if (cpi->cpi_chipid >= 0) {
4162 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4163 "chip#", cpi->cpi_chipid);
4164 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4165 "clog#", cpi->cpi_clogid);
4166 }
4167 }
4168
4169 /* cpuid-features */
4170 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4171 "cpuid-features", CPI_FEATURES_EDX(cpi));
4172
4173
4174 /* cpuid-features-ecx */
4175 switch (cpi->cpi_vendor) {
4176 case X86_VENDOR_Intel:
4177 create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
4178 break;
4179 case X86_VENDOR_AMD:
4180 create = cpi->cpi_family >= 0xf;
4181 break;
4182 default:
4183 create = 0;
4184 break;
4185 }
4186 if (create)
4187 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4188 "cpuid-features-ecx", CPI_FEATURES_ECX(cpi));
4189
4190 /* ext-cpuid-features */
4191 switch (cpi->cpi_vendor) {
4192 case X86_VENDOR_Intel:
4193 case X86_VENDOR_AMD:
4194 case X86_VENDOR_Cyrix:
4195 case X86_VENDOR_TM:
4196 case X86_VENDOR_Centaur:
4197 create = cpi->cpi_xmaxeax >= 0x80000001;
4198 break;
4199 default:
4200 create = 0;
4201 break;
4202 }
4203 if (create) {
4204 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4205 "ext-cpuid-features", CPI_FEATURES_XTD_EDX(cpi));
4206 (void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
4207 "ext-cpuid-features-ecx", CPI_FEATURES_XTD_ECX(cpi));
4208 }
4209
4210 /*
4211 * Brand String first appeared in Intel Pentium IV, AMD K5
4212 * model 1, and Cyrix GXm. On earlier models we try and
4213 * simulate something similar .. so this string should always
4214 * same -something- about the processor, however lame.
4215 */
4216 (void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
4217 "brand-string", cpi->cpi_brandstr);
4218
4219 /*
4220 * Finally, cache and tlb information
4221 */
4222 switch (x86_which_cacheinfo(cpi)) {
4223 case X86_VENDOR_Intel:
4224 intel_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
4225 break;
4226 case X86_VENDOR_Cyrix:
4227 cyrix_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
4228 break;
4229 case X86_VENDOR_AMD:
4230 amd_cache_info(cpi, cpu_devi);
4231 break;
4232 default:
4233 break;
4234 }
4235 }
4236
4237 struct l2info {
4238 int *l2i_csz;
4239 int *l2i_lsz;
4240 int *l2i_assoc;
4241 int l2i_ret;
4242 };
4243
4244 /*
4245 * A cacheinfo walker that fetches the size, line-size and associativity
4246 * of the L2 cache
4247 */
4248 static int
4249 intel_l2cinfo(void *arg, const struct cachetab *ct)
4250 {
4251 struct l2info *l2i = arg;
4252 int *ip;
4253
4254 if (ct->ct_label != l2_cache_str &&
4255 ct->ct_label != sl2_cache_str)
4256 return (0); /* not an L2 -- keep walking */
4257
4258 if ((ip = l2i->l2i_csz) != NULL)
4259 *ip = ct->ct_size;
4260 if ((ip = l2i->l2i_lsz) != NULL)
4261 *ip = ct->ct_line_size;
4262 if ((ip = l2i->l2i_assoc) != NULL)
4263 *ip = ct->ct_assoc;
4264 l2i->l2i_ret = ct->ct_size;
4265 return (1); /* was an L2 -- terminate walk */
4266 }
4267
4268 /*
4269 * AMD L2/L3 Cache and TLB Associativity Field Definition:
4270 *
4271 * Unlike the associativity for the L1 cache and tlb where the 8 bit
4272 * value is the associativity, the associativity for the L2 cache and
4273 * tlb is encoded in the following table. The 4 bit L2 value serves as
4274 * an index into the amd_afd[] array to determine the associativity.
4275 * -1 is undefined. 0 is fully associative.
4276 */
4277
4278 static int amd_afd[] =
4279 {-1, 1, 2, -1, 4, -1, 8, -1, 16, -1, 32, 48, 64, 96, 128, 0};
4280
4281 static void
4282 amd_l2cacheinfo(struct cpuid_info *cpi, struct l2info *l2i)
4283 {
4284 struct cpuid_regs *cp;
4285 uint_t size, assoc;
4286 int i;
4287 int *ip;
4288
4289 if (cpi->cpi_xmaxeax < 0x80000006)
4290 return;
4291 cp = &cpi->cpi_extd[6];
4292
4293 if ((i = BITX(cp->cp_ecx, 15, 12)) != 0 &&
4294 (size = BITX(cp->cp_ecx, 31, 16)) != 0) {
4295 uint_t cachesz = size * 1024;
4296 assoc = amd_afd[i];
4297
4298 ASSERT(assoc != -1);
4299
4300 if ((ip = l2i->l2i_csz) != NULL)
4301 *ip = cachesz;
4302 if ((ip = l2i->l2i_lsz) != NULL)
4303 *ip = BITX(cp->cp_ecx, 7, 0);
4304 if ((ip = l2i->l2i_assoc) != NULL)
4305 *ip = assoc;
4306 l2i->l2i_ret = cachesz;
4307 }
4308 }
4309
4310 int
4311 getl2cacheinfo(cpu_t *cpu, int *csz, int *lsz, int *assoc)
4312 {
4313 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
4314 struct l2info __l2info, *l2i = &__l2info;
4315
4316 l2i->l2i_csz = csz;
4317 l2i->l2i_lsz = lsz;
4318 l2i->l2i_assoc = assoc;
4319 l2i->l2i_ret = -1;
4320
4321 switch (x86_which_cacheinfo(cpi)) {
4322 case X86_VENDOR_Intel:
4323 intel_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
4324 break;
4325 case X86_VENDOR_Cyrix:
4326 cyrix_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
4327 break;
4328 case X86_VENDOR_AMD:
4329 amd_l2cacheinfo(cpi, l2i);
4330 break;
4331 default:
4332 break;
4333 }
4334 return (l2i->l2i_ret);
4335 }
4336
4337 #if !defined(__xpv)
4338
4339 uint32_t *
4340 cpuid_mwait_alloc(cpu_t *cpu)
4341 {
4342 uint32_t *ret;
4343 size_t mwait_size;
4344
4345 ASSERT(cpuid_checkpass(CPU, 2));
4346
4347 mwait_size = CPU->cpu_m.mcpu_cpi->cpi_mwait.mon_max;
4348 if (mwait_size == 0)
4349 return (NULL);
4350
4351 /*
4352 * kmem_alloc() returns cache line size aligned data for mwait_size
4353 * allocations. mwait_size is currently cache line sized. Neither
4354 * of these implementation details are guarantied to be true in the
4355 * future.
4356 *
4357 * First try allocating mwait_size as kmem_alloc() currently returns
4358 * correctly aligned memory. If kmem_alloc() does not return
4359 * mwait_size aligned memory, then use mwait_size ROUNDUP.
4360 *
4361 * Set cpi_mwait.buf_actual and cpi_mwait.size_actual in case we
4362 * decide to free this memory.
4363 */
4364 ret = kmem_zalloc(mwait_size, KM_SLEEP);
4365 if (ret == (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size)) {
4366 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = ret;
4367 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = mwait_size;
4368 *ret = MWAIT_RUNNING;
4369 return (ret);
4370 } else {
4371 kmem_free(ret, mwait_size);
4372 ret = kmem_zalloc(mwait_size * 2, KM_SLEEP);
4373 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = ret;
4374 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = mwait_size * 2;
4375 ret = (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size);
4376 *ret = MWAIT_RUNNING;
4377 return (ret);
4378 }
4379 }
4380
4381 void
4382 cpuid_mwait_free(cpu_t *cpu)
4383 {
4384 if (cpu->cpu_m.mcpu_cpi == NULL) {
4385 return;
4386 }
4387
4388 if (cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual != NULL &&
4389 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual > 0) {
4390 kmem_free(cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual,
4391 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual);
4392 }
4393
4394 cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = NULL;
4395 cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = 0;
4396 }
4397
4398 void
4399 patch_tsc_read(int flag)
4400 {
4401 size_t cnt;
4402
4403 switch (flag) {
4404 case X86_NO_TSC:
4405 cnt = &_no_rdtsc_end - &_no_rdtsc_start;
4406 (void) memcpy((void *)tsc_read, (void *)&_no_rdtsc_start, cnt);
4407 break;
4408 case X86_HAVE_TSCP:
4409 cnt = &_tscp_end - &_tscp_start;
4410 (void) memcpy((void *)tsc_read, (void *)&_tscp_start, cnt);
4411 break;
4412 case X86_TSC_MFENCE:
4413 cnt = &_tsc_mfence_end - &_tsc_mfence_start;
4414 (void) memcpy((void *)tsc_read,
4415 (void *)&_tsc_mfence_start, cnt);
4416 break;
4417 case X86_TSC_LFENCE:
4418 cnt = &_tsc_lfence_end - &_tsc_lfence_start;
4419 (void) memcpy((void *)tsc_read,
4420 (void *)&_tsc_lfence_start, cnt);
4421 break;
4422 default:
4423 break;
4424 }
4425 }
4426
4427 int
4428 cpuid_deep_cstates_supported(void)
4429 {
4430 struct cpuid_info *cpi;
4431 struct cpuid_regs regs;
4432
4433 ASSERT(cpuid_checkpass(CPU, 1));
4434
4435 cpi = CPU->cpu_m.mcpu_cpi;
4436
4437 if (!is_x86_feature(x86_featureset, X86FSET_CPUID))
4438 return (0);
4439
4440 switch (cpi->cpi_vendor) {
4441 case X86_VENDOR_Intel:
4442 if (cpi->cpi_xmaxeax < 0x80000007)
4443 return (0);
4444
4445 /*
4446 * TSC run at a constant rate in all ACPI C-states?
4447 */
4448 regs.cp_eax = 0x80000007;
4449 (void) __cpuid_insn(®s);
4450 return (regs.cp_edx & CPUID_TSC_CSTATE_INVARIANCE);
4451
4452 default:
4453 return (0);
4454 }
4455 }
4456
4457 #endif /* !__xpv */
4458
4459 void
4460 post_startup_cpu_fixups(void)
4461 {
4462 #ifndef __xpv
4463 /*
4464 * Some AMD processors support C1E state. Entering this state will
4465 * cause the local APIC timer to stop, which we can't deal with at
4466 * this time.
4467 */
4468 if (cpuid_getvendor(CPU) == X86_VENDOR_AMD) {
4469 on_trap_data_t otd;
4470 uint64_t reg;
4471
4472 if (!on_trap(&otd, OT_DATA_ACCESS)) {
4473 reg = rdmsr(MSR_AMD_INT_PENDING_CMP_HALT);
4474 /* Disable C1E state if it is enabled by BIOS */
4475 if ((reg >> AMD_ACTONCMPHALT_SHIFT) &
4476 AMD_ACTONCMPHALT_MASK) {
4477 reg &= ~(AMD_ACTONCMPHALT_MASK <<
4478 AMD_ACTONCMPHALT_SHIFT);
4479 wrmsr(MSR_AMD_INT_PENDING_CMP_HALT, reg);
4480 }
4481 }
4482 no_trap();
4483 }
4484 #endif /* !__xpv */
4485 }
4486
4487 /*
4488 * Setup necessary registers to enable XSAVE feature on this processor.
4489 * This function needs to be called early enough, so that no xsave/xrstor
4490 * ops will execute on the processor before the MSRs are properly set up.
4491 *
4492 * Current implementation has the following assumption:
4493 * - cpuid_pass1() is done, so that X86 features are known.
4494 * - fpu_probe() is done, so that fp_save_mech is chosen.
4495 */
4496 void
4497 xsave_setup_msr(cpu_t *cpu)
4498 {
4499 ASSERT(fp_save_mech == FP_XSAVE);
4500 ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
4501
4502 /* Enable OSXSAVE in CR4. */
4503 setcr4(getcr4() | CR4_OSXSAVE);
4504 /*
4505 * Update SW copy of ECX, so that /dev/cpu/self/cpuid will report
4506 * correct value.
4507 */
4508 cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_ecx |= CPUID_INTC_ECX_OSXSAVE;
4509 setup_xfem();
4510 }
4511
4512 /*
4513 * Starting with the Westmere processor the local
4514 * APIC timer will continue running in all C-states,
4515 * including the deepest C-states.
4516 */
4517 int
4518 cpuid_arat_supported(void)
4519 {
4520 struct cpuid_info *cpi;
4521 struct cpuid_regs regs;
4522
4523 ASSERT(cpuid_checkpass(CPU, 1));
4524 ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
4525
4526 cpi = CPU->cpu_m.mcpu_cpi;
4527
4528 switch (cpi->cpi_vendor) {
4529 case X86_VENDOR_Intel:
4530 /*
4531 * Always-running Local APIC Timer is
4532 * indicated by CPUID.6.EAX[2].
4533 */
4534 if (cpi->cpi_maxeax >= 6) {
4535 regs.cp_eax = 6;
4536 (void) cpuid_insn(NULL, ®s);
4537 return (regs.cp_eax & CPUID_CSTATE_ARAT);
4538 } else {
4539 return (0);
4540 }
4541 default:
4542 return (0);
4543 }
4544 }
4545
4546 /*
4547 * Check support for Intel ENERGY_PERF_BIAS feature
4548 */
4549 int
4550 cpuid_iepb_supported(struct cpu *cp)
4551 {
4552 struct cpuid_info *cpi = cp->cpu_m.mcpu_cpi;
4553 struct cpuid_regs regs;
4554
4555 ASSERT(cpuid_checkpass(cp, 1));
4556
4557 if (!(is_x86_feature(x86_featureset, X86FSET_CPUID)) ||
4558 !(is_x86_feature(x86_featureset, X86FSET_MSR))) {
4559 return (0);
4560 }
4561
4562 /*
4563 * Intel ENERGY_PERF_BIAS MSR is indicated by
4564 * capability bit CPUID.6.ECX.3
4565 */
4566 if ((cpi->cpi_vendor != X86_VENDOR_Intel) || (cpi->cpi_maxeax < 6))
4567 return (0);
4568
4569 regs.cp_eax = 0x6;
4570 (void) cpuid_insn(NULL, ®s);
4571 return (regs.cp_ecx & CPUID_EPB_SUPPORT);
4572 }
4573
4574 /*
4575 * Check support for TSC deadline timer
4576 *
4577 * TSC deadline timer provides a superior software programming
4578 * model over local APIC timer that eliminates "time drifts".
4579 * Instead of specifying a relative time, software specifies an
4580 * absolute time as the target at which the processor should
4581 * generate a timer event.
4582 */
4583 int
4584 cpuid_deadline_tsc_supported(void)
4585 {
4586 struct cpuid_info *cpi = CPU->cpu_m.mcpu_cpi;
4587 struct cpuid_regs regs;
4588
4589 ASSERT(cpuid_checkpass(CPU, 1));
4590 ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
4591
4592 switch (cpi->cpi_vendor) {
4593 case X86_VENDOR_Intel:
4594 if (cpi->cpi_maxeax >= 1) {
4595 regs.cp_eax = 1;
4596 (void) cpuid_insn(NULL, ®s);
4597 return (regs.cp_ecx & CPUID_DEADLINE_TSC);
4598 } else {
4599 return (0);
4600 }
4601 default:
4602 return (0);
4603 }
4604 }
4605
4606 #if defined(__amd64) && !defined(__xpv)
4607 /*
4608 * Patch in versions of bcopy for high performance Intel Nhm processors
4609 * and later...
4610 */
4611 void
4612 patch_memops(uint_t vendor)
4613 {
4614 size_t cnt, i;
4615 caddr_t to, from;
4616
4617 if ((vendor == X86_VENDOR_Intel) &&
4618 is_x86_feature(x86_featureset, X86FSET_SSE4_2)) {
4619 cnt = &bcopy_patch_end - &bcopy_patch_start;
4620 to = &bcopy_ck_size;
4621 from = &bcopy_patch_start;
4622 for (i = 0; i < cnt; i++) {
4623 *to++ = *from++;
4624 }
4625 }
4626 }
4627 #endif /* __amd64 && !__xpv */
4628
4629 /*
4630 * This function finds the number of bits to represent the number of cores per
4631 * chip and the number of strands per core for the Intel platforms.
4632 * It re-uses the x2APIC cpuid code of the cpuid_pass2().
4633 */
4634 void
4635 cpuid_get_ext_topo(uint_t vendor, uint_t *core_nbits, uint_t *strand_nbits)
4636 {
4637 struct cpuid_regs regs;
4638 struct cpuid_regs *cp = ®s;
4639
4640 if (vendor != X86_VENDOR_Intel) {
4641 return;
4642 }
4643
4644 /* if the cpuid level is 0xB, extended topo is available. */
4645 cp->cp_eax = 0;
4646 if (__cpuid_insn(cp) >= 0xB) {
4647
4648 cp->cp_eax = 0xB;
4649 cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
4650 (void) __cpuid_insn(cp);
4651
4652 /*
4653 * Check CPUID.EAX=0BH, ECX=0H:EBX is non-zero, which
4654 * indicates that the extended topology enumeration leaf is
4655 * available.
4656 */
4657 if (cp->cp_ebx) {
4658 uint_t coreid_shift = 0;
4659 uint_t chipid_shift = 0;
4660 uint_t i;
4661 uint_t level;
4662
4663 for (i = 0; i < CPI_FNB_ECX_MAX; i++) {
4664 cp->cp_eax = 0xB;
4665 cp->cp_ecx = i;
4666
4667 (void) __cpuid_insn(cp);
4668 level = CPI_CPU_LEVEL_TYPE(cp);
4669
4670 if (level == 1) {
4671 /*
4672 * Thread level processor topology
4673 * Number of bits shift right APIC ID
4674 * to get the coreid.
4675 */
4676 coreid_shift = BITX(cp->cp_eax, 4, 0);
4677 } else if (level == 2) {
4678 /*
4679 * Core level processor topology
4680 * Number of bits shift right APIC ID
4681 * to get the chipid.
4682 */
4683 chipid_shift = BITX(cp->cp_eax, 4, 0);
4684 }
4685 }
4686
4687 if (coreid_shift > 0 && chipid_shift > coreid_shift) {
4688 *strand_nbits = coreid_shift;
4689 *core_nbits = chipid_shift - coreid_shift;
4690 }
4691 }
4692 }
4693 }