Print this page
*** NO COMMENTS ***
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/i86pc/os/mp_machdep.c
+++ new/usr/src/uts/i86pc/os/mp_machdep.c
1 1
2 2 /*
3 3 * CDDL HEADER START
4 4 *
5 5 * The contents of this file are subject to the terms of the
6 6 * Common Development and Distribution License (the "License").
7 7 * You may not use this file except in compliance with the License.
8 8 *
9 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 10 * or http://www.opensolaris.org/os/licensing.
11 11 * See the License for the specific language governing permissions
12 12 * and limitations under the License.
13 13 *
14 14 * When distributing Covered Code, include this CDDL HEADER in each
15 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 16 * If applicable, add the following below this CDDL HEADER, with the
17 17 * fields enclosed by brackets "[]" replaced with your own identifying
18 18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 19 *
20 20 * CDDL HEADER END
21 21 */
22 22 /*
23 23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 24 */
25 25 /*
26 26 * Copyright (c) 2009-2010, Intel Corporation.
27 27 * All rights reserved.
28 28 */
29 29
30 30 #define PSMI_1_7
31 31 #include <sys/smp_impldefs.h>
32 32 #include <sys/psm.h>
33 33 #include <sys/psm_modctl.h>
34 34 #include <sys/pit.h>
35 35 #include <sys/cmn_err.h>
36 36 #include <sys/strlog.h>
37 37 #include <sys/clock.h>
38 38 #include <sys/debug.h>
39 39 #include <sys/rtc.h>
40 40 #include <sys/x86_archext.h>
41 41 #include <sys/cpupart.h>
42 42 #include <sys/cpuvar.h>
43 43 #include <sys/cpu_event.h>
44 44 #include <sys/cmt.h>
45 45 #include <sys/cpu.h>
46 46 #include <sys/disp.h>
47 47 #include <sys/archsystm.h>
48 48 #include <sys/machsystm.h>
49 49 #include <sys/sysmacros.h>
50 50 #include <sys/memlist.h>
51 51 #include <sys/param.h>
52 52 #include <sys/promif.h>
53 53 #include <sys/cpu_pm.h>
54 54 #if defined(__xpv)
55 55 #include <sys/hypervisor.h>
56 56 #endif
57 57 #include <sys/mach_intr.h>
58 58 #include <vm/hat_i86.h>
59 59 #include <sys/kdi_machimpl.h>
60 60 #include <sys/sdt.h>
61 61 #include <sys/hpet.h>
62 62 #include <sys/sunddi.h>
63 63 #include <sys/sunndi.h>
64 64 #include <sys/cpc_pcbe.h>
65 65
66 66 #define OFFSETOF(s, m) (size_t)(&(((s *)0)->m))
67 67
68 68 /*
69 69 * Local function prototypes
70 70 */
71 71 static int mp_disable_intr(processorid_t cpun);
72 72 static void mp_enable_intr(processorid_t cpun);
73 73 static void mach_init();
74 74 static void mach_picinit();
75 75 static int machhztomhz(uint64_t cpu_freq_hz);
76 76 static uint64_t mach_getcpufreq(void);
77 77 static void mach_fixcpufreq(void);
78 78 static int mach_clkinit(int, int *);
79 79 static void mach_smpinit(void);
80 80 static int mach_softlvl_to_vect(int ipl);
81 81 static void mach_get_platform(int owner);
82 82 static void mach_construct_info();
83 83 static int mach_translate_irq(dev_info_t *dip, int irqno);
84 84 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *,
85 85 psm_intr_op_t, int *);
86 86 static void mach_notify_error(int level, char *errmsg);
87 87 static hrtime_t dummy_hrtime(void);
88 88 static void dummy_scalehrtime(hrtime_t *);
89 89 static uint64_t dummy_unscalehrtime(hrtime_t);
90 90 void cpu_idle(void);
91 91 static void cpu_wakeup(cpu_t *, int);
92 92 #ifndef __xpv
93 93 void cpu_idle_mwait(void);
94 94 static void cpu_wakeup_mwait(cpu_t *, int);
95 95 #endif
96 96 static int mach_cpu_create_devinfo(cpu_t *cp, dev_info_t **dipp);
97 97
98 98 /*
99 99 * External reference functions
100 100 */
101 101 extern void return_instr();
102 102 extern uint64_t freq_tsc(uint32_t *);
103 103 #if defined(__i386)
104 104 extern uint64_t freq_notsc(uint32_t *);
105 105 #endif
106 106 extern void pc_gethrestime(timestruc_t *);
107 107 extern int cpuid_get_coreid(cpu_t *);
108 108 extern int cpuid_get_chipid(cpu_t *);
109 109
110 110 /*
111 111 * PSM functions initialization
112 112 */
113 113 void (*psm_shutdownf)(int, int) = (void (*)(int, int))return_instr;
114 114 void (*psm_preshutdownf)(int, int) = (void (*)(int, int))return_instr;
115 115 void (*psm_notifyf)(int) = (void (*)(int))return_instr;
116 116 void (*psm_set_idle_cpuf)(int) = (void (*)(int))return_instr;
117 117 void (*psm_unset_idle_cpuf)(int) = (void (*)(int))return_instr;
118 118 void (*psminitf)() = mach_init;
119 119 void (*picinitf)() = return_instr;
120 120 int (*clkinitf)(int, int *) = (int (*)(int, int *))return_instr;
121 121 int (*ap_mlsetup)() = (int (*)(void))return_instr;
122 122 void (*send_dirintf)() = return_instr;
123 123 void (*setspl)(int) = (void (*)(int))return_instr;
124 124 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr;
125 125 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr;
126 126 int (*get_pending_spl)(void) = (int (*)(void))return_instr;
127 127 int (*addintr)(void *, int, avfunc, char *, int, caddr_t, caddr_t,
128 128 uint64_t *, dev_info_t *) = NULL;
129 129 void (*remintr)(void *, int, avfunc, int) = NULL;
130 130 void (*kdisetsoftint)(int, struct av_softinfo *)=
131 131 (void (*)(int, struct av_softinfo *))return_instr;
132 132 void (*setsoftint)(int, struct av_softinfo *)=
133 133 (void (*)(int, struct av_softinfo *))return_instr;
134 134 int (*slvltovect)(int) = (int (*)(int))return_instr;
135 135 int (*setlvl)(int, int *) = (int (*)(int, int *))return_instr;
136 136 void (*setlvlx)(int, int) = (void (*)(int, int))return_instr;
137 137 int (*psm_disable_intr)(int) = mp_disable_intr;
138 138 void (*psm_enable_intr)(int) = mp_enable_intr;
139 139 hrtime_t (*gethrtimef)(void) = dummy_hrtime;
140 140 hrtime_t (*gethrtimeunscaledf)(void) = dummy_hrtime;
141 141 void (*scalehrtimef)(hrtime_t *) = dummy_scalehrtime;
142 142 uint64_t (*unscalehrtimef)(hrtime_t) = dummy_unscalehrtime;
143 143 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq;
144 144 void (*gethrestimef)(timestruc_t *) = pc_gethrestime;
145 145 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL;
146 146 int (*psm_get_clockirq)(int) = NULL;
147 147 int (*psm_get_ipivect)(int, int) = NULL;
148 148 uchar_t (*psm_get_ioapicid)(uchar_t) = NULL;
149 149 uint32_t (*psm_get_localapicid)(uint32_t) = NULL;
150 150 uchar_t (*psm_xlate_vector_by_irq)(uchar_t) = NULL;
151 151
152 152 int (*psm_clkinit)(int) = NULL;
153 153 void (*psm_timer_reprogram)(hrtime_t) = NULL;
154 154 void (*psm_timer_enable)(void) = NULL;
155 155 void (*psm_timer_disable)(void) = NULL;
156 156 void (*psm_post_cyclic_setup)(void *arg) = NULL;
157 157 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t,
158 158 int *) = mach_intr_ops;
159 159 int (*psm_state)(psm_state_request_t *) = (int (*)(psm_state_request_t *))
160 160 return_instr;
161 161
162 162 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr;
163 163 void (*hrtime_tick)(void) = return_instr;
164 164
165 165 int (*psm_cpu_create_devinfo)(cpu_t *, dev_info_t **) = mach_cpu_create_devinfo;
166 166 int (*psm_cpu_get_devinfo)(cpu_t *, dev_info_t **) = NULL;
167 167
168 168 /* global IRM pool for APIX (PSM) module */
169 169 ddi_irm_pool_t *apix_irm_pool_p = NULL;
170 170
171 171 /*
172 172 * True if the generic TSC code is our source of hrtime, rather than whatever
173 173 * the PSM can provide.
174 174 */
175 175 #ifdef __xpv
176 176 int tsc_gethrtime_enable = 0;
177 177 #else
178 178 int tsc_gethrtime_enable = 1;
179 179 #endif
180 180 int tsc_gethrtime_initted = 0;
181 181
182 182 /*
183 183 * True if the hrtime implementation is "hires"; namely, better than microdata.
184 184 */
185 185 int gethrtime_hires = 0;
186 186
187 187 /*
188 188 * Local Static Data
189 189 */
190 190 static struct psm_ops mach_ops;
191 191 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL};
192 192 static ushort_t mach_ver[4] = {0, 0, 0, 0};
193 193
194 194 /*
195 195 * virtualization support for psm
196 196 */
197 197 void *psm_vt_ops = NULL;
198 198 /*
199 199 * If non-zero, idle cpus will become "halted" when there's
200 200 * no work to do.
201 201 */
202 202 int idle_cpu_use_hlt = 1;
203 203
204 204 #ifndef __xpv
205 205 /*
206 206 * If non-zero, idle cpus will use mwait if available to halt instead of hlt.
207 207 */
208 208 int idle_cpu_prefer_mwait = 1;
209 209 /*
210 210 * Set to 0 to avoid MONITOR+CLFLUSH assertion.
211 211 */
212 212 int idle_cpu_assert_cflush_monitor = 1;
213 213
214 214 /*
215 215 * If non-zero, idle cpus will not use power saving Deep C-States idle loop.
216 216 */
217 217 int idle_cpu_no_deep_c = 0;
218 218 /*
219 219 * Non-power saving idle loop and wakeup pointers.
220 220 * Allows user to toggle Deep Idle power saving feature on/off.
221 221 */
222 222 void (*non_deep_idle_cpu)() = cpu_idle;
223 223 void (*non_deep_idle_disp_enq_thread)(cpu_t *, int);
224 224
225 225 /*
226 226 * Object for the kernel to access the HPET.
227 227 */
228 228 hpet_t hpet;
229 229
230 230 #endif /* ifndef __xpv */
231 231
232 232 uint_t cp_haltset_fanout = 0;
233 233
234 234 /*ARGSUSED*/
235 235 int
236 236 pg_plat_hw_shared(cpu_t *cp, pghw_type_t hw)
237 237 {
|
↓ open down ↓ |
237 lines elided |
↑ open up ↑ |
238 238 switch (hw) {
239 239 case PGHW_IPIPE:
240 240 if (is_x86_feature(x86_featureset, X86FSET_HTT)) {
241 241 /*
242 242 * Hyper-threading is SMT
243 243 */
244 244 return (1);
245 245 } else {
246 246 return (0);
247 247 }
248 + case PGHW_FPU:
249 + if (cpuid_get_cores_per_compunit(cp) > 1)
250 + return (1);
251 + else
252 + return (0);
248 253 case PGHW_PROCNODE:
249 254 if (cpuid_get_procnodes_per_pkg(cp) > 1)
250 255 return (1);
251 256 else
252 257 return (0);
253 258 case PGHW_CHIP:
254 259 if (is_x86_feature(x86_featureset, X86FSET_CMP) ||
255 260 is_x86_feature(x86_featureset, X86FSET_HTT))
256 261 return (1);
257 262 else
258 263 return (0);
259 264 case PGHW_CACHE:
260 265 if (cpuid_get_ncpu_sharing_last_cache(cp) > 1)
261 266 return (1);
262 267 else
263 268 return (0);
264 269 case PGHW_POW_ACTIVE:
265 270 if (cpupm_domain_id(cp, CPUPM_DTYPE_ACTIVE) != (id_t)-1)
266 271 return (1);
267 272 else
268 273 return (0);
269 274 case PGHW_POW_IDLE:
270 275 if (cpupm_domain_id(cp, CPUPM_DTYPE_IDLE) != (id_t)-1)
271 276 return (1);
272 277 else
273 278 return (0);
274 279 default:
275 280 return (0);
276 281 }
277 282 }
278 283
279 284 /*
280 285 * Compare two CPUs and see if they have a pghw_type_t sharing relationship
281 286 * If pghw_type_t is an unsupported hardware type, then return -1
282 287 */
283 288 int
284 289 pg_plat_cpus_share(cpu_t *cpu_a, cpu_t *cpu_b, pghw_type_t hw)
285 290 {
286 291 id_t pgp_a, pgp_b;
287 292
288 293 pgp_a = pg_plat_hw_instance_id(cpu_a, hw);
289 294 pgp_b = pg_plat_hw_instance_id(cpu_b, hw);
290 295
291 296 if (pgp_a == -1 || pgp_b == -1)
292 297 return (-1);
293 298
294 299 return (pgp_a == pgp_b);
295 300 }
296 301
297 302 /*
298 303 * Return a physical instance identifier for known hardware sharing
|
↓ open down ↓ |
41 lines elided |
↑ open up ↑ |
299 304 * relationships
300 305 */
301 306 id_t
302 307 pg_plat_hw_instance_id(cpu_t *cpu, pghw_type_t hw)
303 308 {
304 309 switch (hw) {
305 310 case PGHW_IPIPE:
306 311 return (cpuid_get_coreid(cpu));
307 312 case PGHW_CACHE:
308 313 return (cpuid_get_last_lvl_cacheid(cpu));
314 + case PGHW_FPU:
315 + return (cpuid_get_compunitid(cpu));
309 316 case PGHW_PROCNODE:
310 317 return (cpuid_get_procnodeid(cpu));
311 318 case PGHW_CHIP:
312 319 return (cpuid_get_chipid(cpu));
313 320 case PGHW_POW_ACTIVE:
314 321 return (cpupm_domain_id(cpu, CPUPM_DTYPE_ACTIVE));
315 322 case PGHW_POW_IDLE:
316 323 return (cpupm_domain_id(cpu, CPUPM_DTYPE_IDLE));
317 324 default:
318 325 return (-1);
319 326 }
320 327 }
321 328
322 329 /*
323 330 * Express preference for optimizing for sharing relationship
|
↓ open down ↓ |
5 lines elided |
↑ open up ↑ |
324 331 * hw1 vs hw2
325 332 */
326 333 pghw_type_t
327 334 pg_plat_hw_rank(pghw_type_t hw1, pghw_type_t hw2)
328 335 {
329 336 int i, rank1, rank2;
330 337
331 338 static pghw_type_t hw_hier[] = {
332 339 PGHW_IPIPE,
333 340 PGHW_CACHE,
341 + PGHW_FPU,
334 342 PGHW_PROCNODE,
335 343 PGHW_CHIP,
336 344 PGHW_POW_IDLE,
337 345 PGHW_POW_ACTIVE,
338 346 PGHW_NUM_COMPONENTS
339 347 };
340 348
341 349 for (i = 0; hw_hier[i] != PGHW_NUM_COMPONENTS; i++) {
342 350 if (hw_hier[i] == hw1)
343 351 rank1 = i;
344 352 if (hw_hier[i] == hw2)
345 353 rank2 = i;
346 354 }
347 355
348 356 if (rank1 > rank2)
349 357 return (hw1);
350 358 else
351 359 return (hw2);
352 360 }
353 361
|
↓ open down ↓ |
10 lines elided |
↑ open up ↑ |
354 362 /*
355 363 * Override the default CMT dispatcher policy for the specified
356 364 * hardware sharing relationship
357 365 */
358 366 pg_cmt_policy_t
359 367 pg_plat_cmt_policy(pghw_type_t hw)
360 368 {
361 369 /*
362 370 * For shared caches, also load balance across them to
363 371 * maximize aggregate cache capacity
372 + *
373 + * On AMD family 0x15 CPUs, cores come in pairs called
374 + * compute units, sharing the FPU and the L1I and L2
375 + * caches. Use balancing and cache affinity.
364 376 */
365 377 switch (hw) {
378 + case PGHW_FPU:
366 379 case PGHW_CACHE:
367 380 return (CMT_BALANCE|CMT_AFFINITY);
368 381 default:
369 382 return (CMT_NO_POLICY);
370 383 }
371 384 }
372 385
373 386 id_t
374 387 pg_plat_get_core_id(cpu_t *cpu)
375 388 {
376 389 return ((id_t)cpuid_get_coreid(cpu));
377 390 }
378 391
379 392 void
380 393 cmp_set_nosteal_interval(void)
381 394 {
382 395 /* Set the nosteal interval (used by disp_getbest()) to 100us */
383 396 nosteal_nsec = 100000UL;
384 397 }
385 398
386 399 /*
387 400 * Routine to ensure initial callers to hrtime gets 0 as return
388 401 */
389 402 static hrtime_t
390 403 dummy_hrtime(void)
391 404 {
392 405 return (0);
393 406 }
394 407
395 408 /* ARGSUSED */
396 409 static void
397 410 dummy_scalehrtime(hrtime_t *ticks)
398 411 {}
399 412
400 413 static uint64_t
401 414 dummy_unscalehrtime(hrtime_t nsecs)
402 415 {
403 416 return ((uint64_t)nsecs);
404 417 }
405 418
406 419 /*
407 420 * Supports Deep C-State power saving idle loop.
408 421 */
409 422 void
410 423 cpu_idle_adaptive(void)
411 424 {
412 425 (*CPU->cpu_m.mcpu_idle_cpu)();
413 426 }
414 427
415 428 /*
416 429 * Function called by CPU idle notification framework to check whether CPU
417 430 * has been awakened. It will be called with interrupt disabled.
418 431 * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle
419 432 * notification framework.
420 433 */
421 434 /*ARGSUSED*/
422 435 static void
423 436 cpu_idle_check_wakeup(void *arg)
424 437 {
425 438 /*
426 439 * Toggle interrupt flag to detect pending interrupts.
427 440 * If interrupt happened, do_interrupt() will notify CPU idle
428 441 * notification framework so no need to call cpu_idle_exit() here.
429 442 */
430 443 sti();
431 444 SMT_PAUSE();
432 445 cli();
433 446 }
434 447
435 448 /*
436 449 * Idle the present CPU until wakened via an interrupt
437 450 */
438 451 void
439 452 cpu_idle(void)
440 453 {
441 454 cpu_t *cpup = CPU;
442 455 processorid_t cpu_sid = cpup->cpu_seqid;
443 456 cpupart_t *cp = cpup->cpu_part;
444 457 int hset_update = 1;
445 458
446 459 /*
447 460 * If this CPU is online, and there's multiple CPUs
448 461 * in the system, then we should notate our halting
449 462 * by adding ourselves to the partition's halted CPU
450 463 * bitmap. This allows other CPUs to find/awaken us when
451 464 * work becomes available.
452 465 */
453 466 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1)
454 467 hset_update = 0;
455 468
456 469 /*
457 470 * Add ourselves to the partition's halted CPUs bitmap
458 471 * and set our HALTED flag, if necessary.
459 472 *
460 473 * When a thread becomes runnable, it is placed on the queue
461 474 * and then the halted CPU bitmap is checked to determine who
462 475 * (if anyone) should be awakened. We therefore need to first
463 476 * add ourselves to the bitmap, and and then check if there
464 477 * is any work available. The order is important to prevent a race
465 478 * that can lead to work languishing on a run queue somewhere while
466 479 * this CPU remains halted.
467 480 *
468 481 * Either the producing CPU will see we're halted and will awaken us,
469 482 * or this CPU will see the work available in disp_anywork().
470 483 *
471 484 * Note that memory barriers after updating the HALTED flag
472 485 * are not necessary since an atomic operation (updating the bitset)
473 486 * immediately follows. On x86 the atomic operation acts as a
474 487 * memory barrier for the update of cpu_disp_flags.
475 488 */
476 489 if (hset_update) {
477 490 cpup->cpu_disp_flags |= CPU_DISP_HALTED;
478 491 bitset_atomic_add(&cp->cp_haltset, cpu_sid);
479 492 }
480 493
481 494 /*
482 495 * Check to make sure there's really nothing to do.
483 496 * Work destined for this CPU may become available after
484 497 * this check. We'll be notified through the clearing of our
485 498 * bit in the halted CPU bitmap, and a poke.
486 499 */
487 500 if (disp_anywork()) {
488 501 if (hset_update) {
489 502 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
490 503 bitset_atomic_del(&cp->cp_haltset, cpu_sid);
491 504 }
492 505 return;
493 506 }
494 507
495 508 /*
496 509 * We're on our way to being halted.
497 510 *
498 511 * Disable interrupts now, so that we'll awaken immediately
499 512 * after halting if someone tries to poke us between now and
500 513 * the time we actually halt.
501 514 *
502 515 * We check for the presence of our bit after disabling interrupts.
503 516 * If it's cleared, we'll return. If the bit is cleared after
504 517 * we check then the poke will pop us out of the halted state.
505 518 *
506 519 * This means that the ordering of the poke and the clearing
507 520 * of the bit by cpu_wakeup is important.
508 521 * cpu_wakeup() must clear, then poke.
509 522 * cpu_idle() must disable interrupts, then check for the bit.
510 523 */
511 524 cli();
512 525
513 526 if (hset_update && bitset_in_set(&cp->cp_haltset, cpu_sid) == 0) {
514 527 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
515 528 sti();
516 529 return;
517 530 }
518 531
519 532 /*
520 533 * The check for anything locally runnable is here for performance
521 534 * and isn't needed for correctness. disp_nrunnable ought to be
522 535 * in our cache still, so it's inexpensive to check, and if there
523 536 * is anything runnable we won't have to wait for the poke.
524 537 */
525 538 if (cpup->cpu_disp->disp_nrunnable != 0) {
526 539 if (hset_update) {
527 540 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
528 541 bitset_atomic_del(&cp->cp_haltset, cpu_sid);
529 542 }
530 543 sti();
531 544 return;
532 545 }
533 546
534 547 if (cpu_idle_enter(IDLE_STATE_C1, 0,
535 548 cpu_idle_check_wakeup, NULL) == 0) {
536 549 mach_cpu_idle();
537 550 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE);
538 551 }
539 552
540 553 /*
541 554 * We're no longer halted
542 555 */
543 556 if (hset_update) {
544 557 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
545 558 bitset_atomic_del(&cp->cp_haltset, cpu_sid);
546 559 }
547 560 }
548 561
549 562
550 563 /*
551 564 * If "cpu" is halted, then wake it up clearing its halted bit in advance.
552 565 * Otherwise, see if other CPUs in the cpu partition are halted and need to
553 566 * be woken up so that they can steal the thread we placed on this CPU.
554 567 * This function is only used on MP systems.
555 568 */
556 569 static void
557 570 cpu_wakeup(cpu_t *cpu, int bound)
558 571 {
559 572 uint_t cpu_found;
560 573 processorid_t cpu_sid;
561 574 cpupart_t *cp;
562 575
563 576 cp = cpu->cpu_part;
564 577 cpu_sid = cpu->cpu_seqid;
565 578 if (bitset_in_set(&cp->cp_haltset, cpu_sid)) {
566 579 /*
567 580 * Clear the halted bit for that CPU since it will be
568 581 * poked in a moment.
569 582 */
570 583 bitset_atomic_del(&cp->cp_haltset, cpu_sid);
571 584 /*
572 585 * We may find the current CPU present in the halted cpuset
573 586 * if we're in the context of an interrupt that occurred
574 587 * before we had a chance to clear our bit in cpu_idle().
575 588 * Poking ourself is obviously unnecessary, since if
576 589 * we're here, we're not halted.
577 590 */
578 591 if (cpu != CPU)
579 592 poke_cpu(cpu->cpu_id);
580 593 return;
581 594 } else {
582 595 /*
583 596 * This cpu isn't halted, but it's idle or undergoing a
584 597 * context switch. No need to awaken anyone else.
585 598 */
586 599 if (cpu->cpu_thread == cpu->cpu_idle_thread ||
587 600 cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL)
588 601 return;
589 602 }
590 603
591 604 /*
592 605 * No need to wake up other CPUs if this is for a bound thread.
593 606 */
594 607 if (bound)
595 608 return;
596 609
597 610 /*
598 611 * The CPU specified for wakeup isn't currently halted, so check
599 612 * to see if there are any other halted CPUs in the partition,
600 613 * and if there are then awaken one.
601 614 */
602 615 do {
603 616 cpu_found = bitset_find(&cp->cp_haltset);
604 617 if (cpu_found == (uint_t)-1)
605 618 return;
606 619 } while (bitset_atomic_test_and_del(&cp->cp_haltset, cpu_found) < 0);
607 620
608 621 if (cpu_found != CPU->cpu_seqid) {
609 622 poke_cpu(cpu_seq[cpu_found]->cpu_id);
610 623 }
611 624 }
612 625
613 626 #ifndef __xpv
614 627 /*
615 628 * Function called by CPU idle notification framework to check whether CPU
616 629 * has been awakened. It will be called with interrupt disabled.
617 630 * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle
618 631 * notification framework.
619 632 */
620 633 static void
621 634 cpu_idle_mwait_check_wakeup(void *arg)
622 635 {
623 636 volatile uint32_t *mcpu_mwait = (volatile uint32_t *)arg;
624 637
625 638 ASSERT(arg != NULL);
626 639 if (*mcpu_mwait != MWAIT_HALTED) {
627 640 /*
628 641 * CPU has been awakened, notify CPU idle notification system.
629 642 */
630 643 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE);
631 644 } else {
632 645 /*
633 646 * Toggle interrupt flag to detect pending interrupts.
634 647 * If interrupt happened, do_interrupt() will notify CPU idle
635 648 * notification framework so no need to call cpu_idle_exit()
636 649 * here.
637 650 */
638 651 sti();
639 652 SMT_PAUSE();
640 653 cli();
641 654 }
642 655 }
643 656
644 657 /*
645 658 * Idle the present CPU until awakened via touching its monitored line
646 659 */
647 660 void
648 661 cpu_idle_mwait(void)
649 662 {
650 663 volatile uint32_t *mcpu_mwait = CPU->cpu_m.mcpu_mwait;
651 664 cpu_t *cpup = CPU;
652 665 processorid_t cpu_sid = cpup->cpu_seqid;
653 666 cpupart_t *cp = cpup->cpu_part;
654 667 int hset_update = 1;
655 668
656 669 /*
657 670 * Set our mcpu_mwait here, so we can tell if anyone tries to
658 671 * wake us between now and when we call mwait. No other cpu will
659 672 * attempt to set our mcpu_mwait until we add ourself to the halted
660 673 * CPU bitmap.
661 674 */
662 675 *mcpu_mwait = MWAIT_HALTED;
663 676
664 677 /*
665 678 * If this CPU is online, and there's multiple CPUs
666 679 * in the system, then we should note our halting
667 680 * by adding ourselves to the partition's halted CPU
668 681 * bitmap. This allows other CPUs to find/awaken us when
669 682 * work becomes available.
670 683 */
671 684 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1)
672 685 hset_update = 0;
673 686
674 687 /*
675 688 * Add ourselves to the partition's halted CPUs bitmap
676 689 * and set our HALTED flag, if necessary.
677 690 *
678 691 * When a thread becomes runnable, it is placed on the queue
679 692 * and then the halted CPU bitmap is checked to determine who
680 693 * (if anyone) should be awakened. We therefore need to first
681 694 * add ourselves to the bitmap, and and then check if there
682 695 * is any work available.
683 696 *
684 697 * Note that memory barriers after updating the HALTED flag
685 698 * are not necessary since an atomic operation (updating the bitmap)
686 699 * immediately follows. On x86 the atomic operation acts as a
687 700 * memory barrier for the update of cpu_disp_flags.
688 701 */
689 702 if (hset_update) {
690 703 cpup->cpu_disp_flags |= CPU_DISP_HALTED;
691 704 bitset_atomic_add(&cp->cp_haltset, cpu_sid);
692 705 }
693 706
694 707 /*
695 708 * Check to make sure there's really nothing to do.
696 709 * Work destined for this CPU may become available after
697 710 * this check. We'll be notified through the clearing of our
698 711 * bit in the halted CPU bitmap, and a write to our mcpu_mwait.
699 712 *
700 713 * disp_anywork() checks disp_nrunnable, so we do not have to later.
701 714 */
702 715 if (disp_anywork()) {
703 716 if (hset_update) {
704 717 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
705 718 bitset_atomic_del(&cp->cp_haltset, cpu_sid);
706 719 }
707 720 return;
708 721 }
709 722
710 723 /*
711 724 * We're on our way to being halted.
712 725 * To avoid a lost wakeup, arm the monitor before checking if another
713 726 * cpu wrote to mcpu_mwait to wake us up.
714 727 */
715 728 i86_monitor(mcpu_mwait, 0, 0);
716 729 if (*mcpu_mwait == MWAIT_HALTED) {
717 730 if (cpu_idle_enter(IDLE_STATE_C1, 0,
718 731 cpu_idle_mwait_check_wakeup, (void *)mcpu_mwait) == 0) {
719 732 if (*mcpu_mwait == MWAIT_HALTED) {
720 733 i86_mwait(0, 0);
721 734 }
722 735 cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE);
723 736 }
724 737 }
725 738
726 739 /*
727 740 * We're no longer halted
728 741 */
729 742 if (hset_update) {
730 743 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
731 744 bitset_atomic_del(&cp->cp_haltset, cpu_sid);
732 745 }
733 746 }
734 747
735 748 /*
736 749 * If "cpu" is halted in mwait, then wake it up clearing its halted bit in
737 750 * advance. Otherwise, see if other CPUs in the cpu partition are halted and
738 751 * need to be woken up so that they can steal the thread we placed on this CPU.
739 752 * This function is only used on MP systems.
740 753 */
741 754 static void
742 755 cpu_wakeup_mwait(cpu_t *cp, int bound)
743 756 {
744 757 cpupart_t *cpu_part;
745 758 uint_t cpu_found;
746 759 processorid_t cpu_sid;
747 760
748 761 cpu_part = cp->cpu_part;
749 762 cpu_sid = cp->cpu_seqid;
750 763
751 764 /*
752 765 * Clear the halted bit for that CPU since it will be woken up
753 766 * in a moment.
754 767 */
755 768 if (bitset_in_set(&cpu_part->cp_haltset, cpu_sid)) {
756 769 /*
757 770 * Clear the halted bit for that CPU since it will be
758 771 * poked in a moment.
759 772 */
760 773 bitset_atomic_del(&cpu_part->cp_haltset, cpu_sid);
761 774 /*
762 775 * We may find the current CPU present in the halted cpuset
763 776 * if we're in the context of an interrupt that occurred
764 777 * before we had a chance to clear our bit in cpu_idle().
765 778 * Waking ourself is obviously unnecessary, since if
766 779 * we're here, we're not halted.
767 780 *
768 781 * monitor/mwait wakeup via writing to our cache line is
769 782 * harmless and less expensive than always checking if we
770 783 * are waking ourself which is an uncommon case.
771 784 */
772 785 MWAIT_WAKEUP(cp); /* write to monitored line */
773 786 return;
774 787 } else {
775 788 /*
776 789 * This cpu isn't halted, but it's idle or undergoing a
777 790 * context switch. No need to awaken anyone else.
778 791 */
779 792 if (cp->cpu_thread == cp->cpu_idle_thread ||
780 793 cp->cpu_disp_flags & CPU_DISP_DONTSTEAL)
781 794 return;
782 795 }
783 796
784 797 /*
785 798 * No need to wake up other CPUs if the thread we just enqueued
786 799 * is bound.
787 800 */
788 801 if (bound || ncpus == 1)
789 802 return;
790 803
791 804 /*
792 805 * See if there's any other halted CPUs. If there are, then
793 806 * select one, and awaken it.
794 807 * It's possible that after we find a CPU, somebody else
795 808 * will awaken it before we get the chance.
796 809 * In that case, look again.
797 810 */
798 811 do {
799 812 cpu_found = bitset_find(&cpu_part->cp_haltset);
800 813 if (cpu_found == (uint_t)-1)
801 814 return;
802 815 } while (bitset_atomic_test_and_del(&cpu_part->cp_haltset,
803 816 cpu_found) < 0);
804 817
805 818 /*
806 819 * Do not check if cpu_found is ourself as monitor/mwait
807 820 * wakeup is cheap.
808 821 */
809 822 MWAIT_WAKEUP(cpu_seq[cpu_found]); /* write to monitored line */
810 823 }
811 824
812 825 #endif
813 826
814 827 void (*cpu_pause_handler)(volatile char *) = NULL;
815 828
816 829 static int
817 830 mp_disable_intr(int cpun)
818 831 {
819 832 /*
820 833 * switch to the offline cpu
821 834 */
822 835 affinity_set(cpun);
823 836 /*
824 837 * raise ipl to just below cross call
825 838 */
826 839 splx(XC_SYS_PIL - 1);
827 840 /*
828 841 * set base spl to prevent the next swtch to idle from
829 842 * lowering back to ipl 0
830 843 */
831 844 CPU->cpu_intr_actv |= (1 << (XC_SYS_PIL - 1));
832 845 set_base_spl();
833 846 affinity_clear();
834 847 return (DDI_SUCCESS);
835 848 }
836 849
837 850 static void
838 851 mp_enable_intr(int cpun)
839 852 {
840 853 /*
841 854 * switch to the online cpu
842 855 */
843 856 affinity_set(cpun);
844 857 /*
845 858 * clear the interrupt active mask
846 859 */
847 860 CPU->cpu_intr_actv &= ~(1 << (XC_SYS_PIL - 1));
848 861 set_base_spl();
849 862 (void) spl0();
850 863 affinity_clear();
851 864 }
852 865
853 866 static void
854 867 mach_get_platform(int owner)
855 868 {
856 869 void **srv_opsp;
857 870 void **clt_opsp;
858 871 int i;
859 872 int total_ops;
860 873
861 874 /* fix up psm ops */
862 875 srv_opsp = (void **)mach_set[0];
863 876 clt_opsp = (void **)mach_set[owner];
864 877 if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01)
865 878 total_ops = sizeof (struct psm_ops_ver01) /
866 879 sizeof (void (*)(void));
867 880 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1)
868 881 /* no psm_notify_func */
869 882 total_ops = OFFSETOF(struct psm_ops, psm_notify_func) /
870 883 sizeof (void (*)(void));
871 884 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2)
872 885 /* no psm_timer funcs */
873 886 total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) /
874 887 sizeof (void (*)(void));
875 888 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3)
876 889 /* no psm_preshutdown function */
877 890 total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) /
878 891 sizeof (void (*)(void));
879 892 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4)
880 893 /* no psm_intr_ops function */
881 894 total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) /
882 895 sizeof (void (*)(void));
883 896 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_5)
884 897 /* no psm_state function */
885 898 total_ops = OFFSETOF(struct psm_ops, psm_state) /
886 899 sizeof (void (*)(void));
887 900 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_6)
888 901 /* no psm_cpu_ops function */
889 902 total_ops = OFFSETOF(struct psm_ops, psm_cpu_ops) /
890 903 sizeof (void (*)(void));
891 904 else
892 905 total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void));
893 906
894 907 /*
895 908 * Save the version of the PSM module, in case we need to
896 909 * behave differently based on version.
897 910 */
898 911 mach_ver[0] = mach_ver[owner];
899 912
900 913 for (i = 0; i < total_ops; i++)
901 914 if (clt_opsp[i] != NULL)
902 915 srv_opsp[i] = clt_opsp[i];
903 916 }
904 917
905 918 static void
906 919 mach_construct_info()
907 920 {
908 921 struct psm_sw *swp;
909 922 int mach_cnt[PSM_OWN_OVERRIDE+1] = {0};
910 923 int conflict_owner = 0;
911 924
912 925 if (psmsw->psw_forw == psmsw)
913 926 panic("No valid PSM modules found");
914 927 mutex_enter(&psmsw_lock);
915 928 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) {
916 929 if (!(swp->psw_flag & PSM_MOD_IDENTIFY))
917 930 continue;
918 931 mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops;
919 932 mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version;
920 933 mach_cnt[swp->psw_infop->p_owner]++;
921 934 }
922 935 mutex_exit(&psmsw_lock);
923 936
924 937 mach_get_platform(PSM_OWN_SYS_DEFAULT);
925 938
926 939 /* check to see are there any conflicts */
927 940 if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1)
928 941 conflict_owner = PSM_OWN_EXCLUSIVE;
929 942 if (mach_cnt[PSM_OWN_OVERRIDE] > 1)
930 943 conflict_owner = PSM_OWN_OVERRIDE;
931 944 if (conflict_owner) {
932 945 /* remove all psm modules except uppc */
933 946 cmn_err(CE_WARN,
934 947 "Conflicts detected on the following PSM modules:");
935 948 mutex_enter(&psmsw_lock);
936 949 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) {
937 950 if (swp->psw_infop->p_owner == conflict_owner)
938 951 cmn_err(CE_WARN, "%s ",
939 952 swp->psw_infop->p_mach_idstring);
940 953 }
941 954 mutex_exit(&psmsw_lock);
942 955 cmn_err(CE_WARN,
943 956 "Setting the system back to SINGLE processor mode!");
944 957 cmn_err(CE_WARN,
945 958 "Please edit /etc/mach to remove the invalid PSM module.");
946 959 return;
947 960 }
948 961
949 962 if (mach_set[PSM_OWN_EXCLUSIVE])
950 963 mach_get_platform(PSM_OWN_EXCLUSIVE);
951 964
952 965 if (mach_set[PSM_OWN_OVERRIDE])
953 966 mach_get_platform(PSM_OWN_OVERRIDE);
954 967 }
955 968
956 969 static void
957 970 mach_init()
958 971 {
959 972 struct psm_ops *pops;
960 973
961 974 mach_construct_info();
962 975
963 976 pops = mach_set[0];
964 977
965 978 /* register the interrupt and clock initialization rotuines */
966 979 picinitf = mach_picinit;
967 980 clkinitf = mach_clkinit;
968 981 psm_get_clockirq = pops->psm_get_clockirq;
969 982
970 983 /* register the interrupt setup code */
971 984 slvltovect = mach_softlvl_to_vect;
972 985 addspl = pops->psm_addspl;
973 986 delspl = pops->psm_delspl;
974 987
975 988 if (pops->psm_translate_irq)
976 989 psm_translate_irq = pops->psm_translate_irq;
977 990 if (pops->psm_intr_ops)
978 991 psm_intr_ops = pops->psm_intr_ops;
979 992
980 993 #if defined(PSMI_1_2) || defined(PSMI_1_3) || defined(PSMI_1_4)
981 994 /*
982 995 * Time-of-day functionality now handled in TOD modules.
983 996 * (Warn about PSM modules that think that we're going to use
984 997 * their ops vectors.)
985 998 */
986 999 if (pops->psm_tod_get)
987 1000 cmn_err(CE_WARN, "obsolete psm_tod_get op %p",
988 1001 (void *)pops->psm_tod_get);
989 1002
990 1003 if (pops->psm_tod_set)
991 1004 cmn_err(CE_WARN, "obsolete psm_tod_set op %p",
992 1005 (void *)pops->psm_tod_set);
993 1006 #endif
994 1007
995 1008 if (pops->psm_notify_error) {
996 1009 psm_notify_error = mach_notify_error;
997 1010 notify_error = pops->psm_notify_error;
998 1011 }
999 1012
1000 1013 (*pops->psm_softinit)();
1001 1014
1002 1015 /*
1003 1016 * Initialize the dispatcher's function hooks to enable CPU halting
1004 1017 * when idle. Set both the deep-idle and non-deep-idle hooks.
1005 1018 *
1006 1019 * Assume we can use power saving deep-idle loop cpu_idle_adaptive.
1007 1020 * Platform deep-idle driver will reset our idle loop to
1008 1021 * non_deep_idle_cpu if power saving deep-idle feature is not available.
1009 1022 *
1010 1023 * Do not use monitor/mwait if idle_cpu_use_hlt is not set(spin idle)
1011 1024 * or idle_cpu_prefer_mwait is not set.
1012 1025 * Allocate monitor/mwait buffer for cpu0.
1013 1026 */
1014 1027 #ifndef __xpv
1015 1028 non_deep_idle_disp_enq_thread = disp_enq_thread;
1016 1029 #endif
1017 1030 if (idle_cpu_use_hlt) {
1018 1031 idle_cpu = cpu_idle_adaptive;
1019 1032 CPU->cpu_m.mcpu_idle_cpu = cpu_idle;
1020 1033 #ifndef __xpv
1021 1034 if (is_x86_feature(x86_featureset, X86FSET_MWAIT) &&
1022 1035 idle_cpu_prefer_mwait) {
1023 1036 CPU->cpu_m.mcpu_mwait = cpuid_mwait_alloc(CPU);
1024 1037 /*
1025 1038 * Protect ourself from insane mwait size.
1026 1039 */
1027 1040 if (CPU->cpu_m.mcpu_mwait == NULL) {
1028 1041 #ifdef DEBUG
1029 1042 cmn_err(CE_NOTE, "Using hlt idle. Cannot "
1030 1043 "handle cpu 0 mwait size.");
1031 1044 #endif
1032 1045 idle_cpu_prefer_mwait = 0;
1033 1046 CPU->cpu_m.mcpu_idle_cpu = cpu_idle;
1034 1047 } else {
1035 1048 CPU->cpu_m.mcpu_idle_cpu = cpu_idle_mwait;
1036 1049 }
1037 1050 } else {
1038 1051 CPU->cpu_m.mcpu_idle_cpu = cpu_idle;
1039 1052 }
1040 1053 non_deep_idle_cpu = CPU->cpu_m.mcpu_idle_cpu;
1041 1054
1042 1055 /*
1043 1056 * Disable power saving deep idle loop?
1044 1057 */
1045 1058 if (idle_cpu_no_deep_c) {
1046 1059 idle_cpu = non_deep_idle_cpu;
1047 1060 }
1048 1061 #endif
1049 1062 }
1050 1063
1051 1064 mach_smpinit();
1052 1065 }
1053 1066
1054 1067 static void
1055 1068 mach_smpinit(void)
1056 1069 {
1057 1070 struct psm_ops *pops;
1058 1071 processorid_t cpu_id;
1059 1072 int cnt;
1060 1073 cpuset_t cpumask;
1061 1074
1062 1075 pops = mach_set[0];
1063 1076 CPUSET_ZERO(cpumask);
1064 1077
1065 1078 cpu_id = -1;
1066 1079 cpu_id = (*pops->psm_get_next_processorid)(cpu_id);
1067 1080 /*
1068 1081 * Only add boot_ncpus CPUs to mp_cpus. Other CPUs will be handled
1069 1082 * by CPU DR driver at runtime.
1070 1083 */
1071 1084 for (cnt = 0; cpu_id != -1 && cnt < boot_ncpus; cnt++) {
1072 1085 CPUSET_ADD(cpumask, cpu_id);
1073 1086 cpu_id = (*pops->psm_get_next_processorid)(cpu_id);
1074 1087 }
1075 1088
1076 1089 mp_cpus = cpumask;
1077 1090
1078 1091 /* MP related routines */
1079 1092 ap_mlsetup = pops->psm_post_cpu_start;
1080 1093 send_dirintf = pops->psm_send_ipi;
1081 1094
1082 1095 /* optional MP related routines */
1083 1096 if (pops->psm_shutdown)
1084 1097 psm_shutdownf = pops->psm_shutdown;
1085 1098 if (pops->psm_preshutdown)
1086 1099 psm_preshutdownf = pops->psm_preshutdown;
1087 1100 if (pops->psm_notify_func)
1088 1101 psm_notifyf = pops->psm_notify_func;
1089 1102 if (pops->psm_set_idlecpu)
1090 1103 psm_set_idle_cpuf = pops->psm_set_idlecpu;
1091 1104 if (pops->psm_unset_idlecpu)
1092 1105 psm_unset_idle_cpuf = pops->psm_unset_idlecpu;
1093 1106
1094 1107 psm_clkinit = pops->psm_clkinit;
1095 1108
1096 1109 if (pops->psm_timer_reprogram)
1097 1110 psm_timer_reprogram = pops->psm_timer_reprogram;
1098 1111
1099 1112 if (pops->psm_timer_enable)
1100 1113 psm_timer_enable = pops->psm_timer_enable;
1101 1114
1102 1115 if (pops->psm_timer_disable)
1103 1116 psm_timer_disable = pops->psm_timer_disable;
1104 1117
1105 1118 if (pops->psm_post_cyclic_setup)
1106 1119 psm_post_cyclic_setup = pops->psm_post_cyclic_setup;
1107 1120
1108 1121 if (pops->psm_state)
1109 1122 psm_state = pops->psm_state;
1110 1123
1111 1124 /*
1112 1125 * Set these vectors here so they can be used by Suspend/Resume
1113 1126 * on UP machines.
1114 1127 */
1115 1128 if (pops->psm_disable_intr)
1116 1129 psm_disable_intr = pops->psm_disable_intr;
1117 1130 if (pops->psm_enable_intr)
1118 1131 psm_enable_intr = pops->psm_enable_intr;
1119 1132
1120 1133 /* check for multiple CPUs */
1121 1134 if (cnt < 2 && plat_dr_support_cpu() == B_FALSE)
1122 1135 return;
1123 1136
1124 1137 /* check for MP platforms */
1125 1138 if (pops->psm_cpu_start == NULL)
1126 1139 return;
1127 1140
1128 1141 /*
1129 1142 * Set the dispatcher hook to enable cpu "wake up"
1130 1143 * when a thread becomes runnable.
1131 1144 */
1132 1145 if (idle_cpu_use_hlt) {
1133 1146 disp_enq_thread = cpu_wakeup;
1134 1147 #ifndef __xpv
1135 1148 if (is_x86_feature(x86_featureset, X86FSET_MWAIT) &&
1136 1149 idle_cpu_prefer_mwait)
1137 1150 disp_enq_thread = cpu_wakeup_mwait;
1138 1151 non_deep_idle_disp_enq_thread = disp_enq_thread;
1139 1152 #endif
1140 1153 }
1141 1154
1142 1155 psm_get_ipivect = pops->psm_get_ipivect;
1143 1156
1144 1157 (void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_intr",
1145 1158 (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI),
1146 1159 NULL, NULL, NULL, NULL);
1147 1160
1148 1161 (void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE);
1149 1162 }
1150 1163
1151 1164 static void
1152 1165 mach_picinit()
1153 1166 {
1154 1167 struct psm_ops *pops;
1155 1168
1156 1169 pops = mach_set[0];
1157 1170
1158 1171 /* register the interrupt handlers */
1159 1172 setlvl = pops->psm_intr_enter;
1160 1173 setlvlx = pops->psm_intr_exit;
1161 1174
1162 1175 /* initialize the interrupt hardware */
1163 1176 (*pops->psm_picinit)();
1164 1177
1165 1178 /* set interrupt mask for current ipl */
1166 1179 setspl = pops->psm_setspl;
1167 1180 cli();
1168 1181 setspl(CPU->cpu_pri);
1169 1182 }
1170 1183
1171 1184 uint_t cpu_freq; /* MHz */
1172 1185 uint64_t cpu_freq_hz; /* measured (in hertz) */
1173 1186
1174 1187 #define MEGA_HZ 1000000
1175 1188
1176 1189 #ifdef __xpv
1177 1190
1178 1191 int xpv_cpufreq_workaround = 1;
1179 1192 int xpv_cpufreq_verbose = 0;
1180 1193
1181 1194 #else /* __xpv */
1182 1195
1183 1196 static uint64_t
1184 1197 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks)
1185 1198 {
1186 1199 uint64_t cpu_hz;
1187 1200
1188 1201 if ((pit_counter == 0) || (*processor_clks == 0) ||
1189 1202 (*processor_clks > (((uint64_t)-1) / PIT_HZ)))
1190 1203 return (0);
1191 1204
1192 1205 cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter;
1193 1206
1194 1207 return (cpu_hz);
1195 1208 }
1196 1209
1197 1210 #endif /* __xpv */
1198 1211
1199 1212 static uint64_t
1200 1213 mach_getcpufreq(void)
1201 1214 {
1202 1215 #if defined(__xpv)
1203 1216 vcpu_time_info_t *vti = &CPU->cpu_m.mcpu_vcpu_info->time;
1204 1217 uint64_t cpu_hz;
1205 1218
1206 1219 /*
1207 1220 * During dom0 bringup, it was noted that on at least one older
1208 1221 * Intel HT machine, the hypervisor initially gives a tsc_to_system_mul
1209 1222 * value that is quite wrong (the 3.06GHz clock was reported
1210 1223 * as 4.77GHz)
1211 1224 *
1212 1225 * The curious thing is, that if you stop the kernel at entry,
1213 1226 * breakpoint here and inspect the value with kmdb, the value
1214 1227 * is correct - but if you don't stop and simply enable the
1215 1228 * printf statement (below), you can see the bad value printed
1216 1229 * here. Almost as if something kmdb did caused the hypervisor to
1217 1230 * figure it out correctly. And, note that the hypervisor
1218 1231 * eventually -does- figure it out correctly ... if you look at
1219 1232 * the field later in the life of dom0, it is correct.
1220 1233 *
1221 1234 * For now, on dom0, we employ a slightly cheesy workaround of
1222 1235 * using the DOM0_PHYSINFO hypercall.
1223 1236 */
1224 1237 if (DOMAIN_IS_INITDOMAIN(xen_info) && xpv_cpufreq_workaround) {
1225 1238 cpu_hz = 1000 * xpv_cpu_khz();
1226 1239 } else {
1227 1240 cpu_hz = (UINT64_C(1000000000) << 32) / vti->tsc_to_system_mul;
1228 1241
1229 1242 if (vti->tsc_shift < 0)
1230 1243 cpu_hz <<= -vti->tsc_shift;
1231 1244 else
1232 1245 cpu_hz >>= vti->tsc_shift;
1233 1246 }
1234 1247
1235 1248 if (xpv_cpufreq_verbose)
1236 1249 printf("mach_getcpufreq: system_mul 0x%x, shift %d, "
1237 1250 "cpu_hz %" PRId64 "Hz\n",
1238 1251 vti->tsc_to_system_mul, vti->tsc_shift, cpu_hz);
1239 1252
1240 1253 return (cpu_hz);
1241 1254 #else /* __xpv */
1242 1255 uint32_t pit_counter;
1243 1256 uint64_t processor_clks;
1244 1257
1245 1258 if (is_x86_feature(x86_featureset, X86FSET_TSC)) {
1246 1259 /*
1247 1260 * We have a TSC. freq_tsc() knows how to measure the number
1248 1261 * of clock cycles sampled against the PIT.
1249 1262 */
1250 1263 ulong_t flags = clear_int_flag();
1251 1264 processor_clks = freq_tsc(&pit_counter);
1252 1265 restore_int_flag(flags);
1253 1266 return (mach_calchz(pit_counter, &processor_clks));
1254 1267 } else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) {
1255 1268 #if defined(__amd64)
1256 1269 panic("mach_getcpufreq: no TSC!");
1257 1270 #elif defined(__i386)
1258 1271 /*
1259 1272 * We are a Cyrix based on a 6x86 core or an Intel Pentium
1260 1273 * for which freq_notsc() knows how to measure the number of
1261 1274 * elapsed clock cycles sampled against the PIT
1262 1275 */
1263 1276 ulong_t flags = clear_int_flag();
1264 1277 processor_clks = freq_notsc(&pit_counter);
1265 1278 restore_int_flag(flags);
1266 1279 return (mach_calchz(pit_counter, &processor_clks));
1267 1280 #endif /* __i386 */
1268 1281 }
1269 1282
1270 1283 /* We do not know how to calculate cpu frequency for this cpu. */
1271 1284 return (0);
1272 1285 #endif /* __xpv */
1273 1286 }
1274 1287
1275 1288 /*
1276 1289 * If the clock speed of a cpu is found to be reported incorrectly, do not add
1277 1290 * to this array, instead improve the accuracy of the algorithm that determines
1278 1291 * the clock speed of the processor or extend the implementation to support the
1279 1292 * vendor as appropriate. This is here only to support adjusting the speed on
1280 1293 * older slower processors that mach_fixcpufreq() would not be able to account
1281 1294 * for otherwise.
1282 1295 */
1283 1296 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 };
1284 1297
1285 1298 /*
1286 1299 * On fast processors the clock frequency that is measured may be off by
1287 1300 * a few MHz from the value printed on the part. This is a combination of
1288 1301 * the factors that for such fast parts being off by this much is within
1289 1302 * the tolerances for manufacture and because of the difficulties in the
1290 1303 * measurement that can lead to small error. This function uses some
1291 1304 * heuristics in order to tweak the value that was measured to match what
1292 1305 * is most likely printed on the part.
1293 1306 *
1294 1307 * Some examples:
1295 1308 * AMD Athlon 1000 mhz measured as 998 mhz
1296 1309 * Intel Pentium III Xeon 733 mhz measured as 731 mhz
1297 1310 * Intel Pentium IV 1500 mhz measured as 1495mhz
1298 1311 *
1299 1312 * If in the future this function is no longer sufficient to correct
1300 1313 * for the error in the measurement, then the algorithm used to perform
1301 1314 * the measurement will have to be improved in order to increase accuracy
1302 1315 * rather than adding horrible and questionable kludges here.
1303 1316 *
1304 1317 * This is called after the cyclics subsystem because of the potential
1305 1318 * that the heuristics within may give a worse estimate of the clock
1306 1319 * frequency than the value that was measured.
1307 1320 */
1308 1321 static void
1309 1322 mach_fixcpufreq(void)
1310 1323 {
1311 1324 uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i;
1312 1325
1313 1326 freq = (uint32_t)cpu_freq;
1314 1327
1315 1328 /*
1316 1329 * Find the nearest integer multiple of 200/3 (about 66) MHz to the
1317 1330 * measured speed taking into account that the 667 MHz parts were
1318 1331 * the first to round-up.
1319 1332 */
1320 1333 mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200);
1321 1334 near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3);
1322 1335 delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66);
1323 1336
1324 1337 /* Find the nearest integer multiple of 50 MHz to the measured speed */
1325 1338 mul = (freq + 25) / 50;
1326 1339 near50 = mul * 50;
1327 1340 delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50);
1328 1341
1329 1342 /* Find the closer of the two */
1330 1343 if (delta66 < delta50) {
1331 1344 fixed = near66;
1332 1345 delta = delta66;
1333 1346 } else {
1334 1347 fixed = near50;
1335 1348 delta = delta50;
1336 1349 }
1337 1350
1338 1351 if (fixed > INT_MAX)
1339 1352 return;
1340 1353
1341 1354 /*
1342 1355 * Some older parts have a core clock frequency that is not an
1343 1356 * integral multiple of 50 or 66 MHz. Check if one of the old
1344 1357 * clock frequencies is closer to the measured value than any
1345 1358 * of the integral multiples of 50 an 66, and if so set fixed
1346 1359 * and delta appropriately to represent the closest value.
1347 1360 */
1348 1361 i = sizeof (x86_cpu_freq) / sizeof (int);
1349 1362 while (i > 0) {
1350 1363 i--;
1351 1364
1352 1365 if (x86_cpu_freq[i] <= freq) {
1353 1366 mul = freq - x86_cpu_freq[i];
1354 1367
1355 1368 if (mul < delta) {
1356 1369 fixed = x86_cpu_freq[i];
1357 1370 delta = mul;
1358 1371 }
1359 1372
1360 1373 break;
1361 1374 }
1362 1375
1363 1376 mul = x86_cpu_freq[i] - freq;
1364 1377
1365 1378 if (mul < delta) {
1366 1379 fixed = x86_cpu_freq[i];
1367 1380 delta = mul;
1368 1381 }
1369 1382 }
1370 1383
1371 1384 /*
1372 1385 * Set a reasonable maximum for how much to correct the measured
1373 1386 * result by. This check is here to prevent the adjustment made
1374 1387 * by this function from being more harm than good. It is entirely
1375 1388 * possible that in the future parts will be made that are not
1376 1389 * integral multiples of 66 or 50 in clock frequency or that
1377 1390 * someone may overclock a part to some odd frequency. If the
1378 1391 * measured value is farther from the corrected value than
1379 1392 * allowed, then assume the corrected value is in error and use
1380 1393 * the measured value.
1381 1394 */
1382 1395 if (6 < delta)
1383 1396 return;
1384 1397
1385 1398 cpu_freq = (int)fixed;
1386 1399 }
1387 1400
1388 1401
1389 1402 static int
1390 1403 machhztomhz(uint64_t cpu_freq_hz)
1391 1404 {
1392 1405 uint64_t cpu_mhz;
1393 1406
1394 1407 /* Round to nearest MHZ */
1395 1408 cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ;
1396 1409
1397 1410 if (cpu_mhz > INT_MAX)
1398 1411 return (0);
1399 1412
1400 1413 return ((int)cpu_mhz);
1401 1414
1402 1415 }
1403 1416
1404 1417
1405 1418 static int
1406 1419 mach_clkinit(int preferred_mode, int *set_mode)
1407 1420 {
1408 1421 struct psm_ops *pops;
1409 1422 int resolution;
1410 1423
1411 1424 pops = mach_set[0];
1412 1425
1413 1426 cpu_freq_hz = mach_getcpufreq();
1414 1427
1415 1428 cpu_freq = machhztomhz(cpu_freq_hz);
1416 1429
1417 1430 if (!is_x86_feature(x86_featureset, X86FSET_TSC) || (cpu_freq == 0))
1418 1431 tsc_gethrtime_enable = 0;
1419 1432
1420 1433 #ifndef __xpv
1421 1434 if (tsc_gethrtime_enable) {
1422 1435 tsc_hrtimeinit(cpu_freq_hz);
1423 1436 } else
1424 1437 #endif
1425 1438 {
1426 1439 if (pops->psm_hrtimeinit)
1427 1440 (*pops->psm_hrtimeinit)();
1428 1441 gethrtimef = pops->psm_gethrtime;
1429 1442 gethrtimeunscaledf = gethrtimef;
1430 1443 /* scalehrtimef will remain dummy */
1431 1444 }
1432 1445
1433 1446 mach_fixcpufreq();
1434 1447
1435 1448 if (mach_ver[0] >= PSM_INFO_VER01_3) {
1436 1449 if (preferred_mode == TIMER_ONESHOT) {
1437 1450
1438 1451 resolution = (*pops->psm_clkinit)(0);
1439 1452 if (resolution != 0) {
1440 1453 *set_mode = TIMER_ONESHOT;
1441 1454 return (resolution);
1442 1455 }
1443 1456 }
1444 1457
1445 1458 /*
1446 1459 * either periodic mode was requested or could not set to
1447 1460 * one-shot mode
1448 1461 */
1449 1462 resolution = (*pops->psm_clkinit)(hz);
1450 1463 /*
1451 1464 * psm should be able to do periodic, so we do not check
1452 1465 * for return value of psm_clkinit here.
1453 1466 */
1454 1467 *set_mode = TIMER_PERIODIC;
1455 1468 return (resolution);
1456 1469 } else {
1457 1470 /*
1458 1471 * PSMI interface prior to PSMI_3 does not define a return
1459 1472 * value for psm_clkinit, so the return value is ignored.
1460 1473 */
1461 1474 (void) (*pops->psm_clkinit)(hz);
1462 1475 *set_mode = TIMER_PERIODIC;
1463 1476 return (nsec_per_tick);
1464 1477 }
1465 1478 }
1466 1479
1467 1480
1468 1481 /*ARGSUSED*/
1469 1482 static int
1470 1483 mach_softlvl_to_vect(int ipl)
1471 1484 {
1472 1485 setsoftint = av_set_softint_pending;
1473 1486 kdisetsoftint = kdi_av_set_softint_pending;
1474 1487
1475 1488 return (PSM_SV_SOFTWARE);
1476 1489 }
1477 1490
1478 1491 #ifdef DEBUG
1479 1492 /*
1480 1493 * This is here to allow us to simulate cpus that refuse to start.
1481 1494 */
1482 1495 cpuset_t cpufailset;
1483 1496 #endif
1484 1497
1485 1498 int
1486 1499 mach_cpu_start(struct cpu *cp, void *ctx)
1487 1500 {
1488 1501 struct psm_ops *pops = mach_set[0];
1489 1502 processorid_t id = cp->cpu_id;
1490 1503
1491 1504 #ifdef DEBUG
1492 1505 if (CPU_IN_SET(cpufailset, id))
1493 1506 return (0);
1494 1507 #endif
1495 1508 return ((*pops->psm_cpu_start)(id, ctx));
1496 1509 }
1497 1510
1498 1511 int
1499 1512 mach_cpuid_start(processorid_t id, void *ctx)
1500 1513 {
1501 1514 struct psm_ops *pops = mach_set[0];
1502 1515
1503 1516 #ifdef DEBUG
1504 1517 if (CPU_IN_SET(cpufailset, id))
1505 1518 return (0);
1506 1519 #endif
1507 1520 return ((*pops->psm_cpu_start)(id, ctx));
1508 1521 }
1509 1522
1510 1523 int
1511 1524 mach_cpu_stop(cpu_t *cp, void *ctx)
1512 1525 {
1513 1526 struct psm_ops *pops = mach_set[0];
1514 1527 psm_cpu_request_t request;
1515 1528
1516 1529 if (pops->psm_cpu_ops == NULL) {
1517 1530 return (ENOTSUP);
1518 1531 }
1519 1532
1520 1533 ASSERT(cp->cpu_id != -1);
1521 1534 request.pcr_cmd = PSM_CPU_STOP;
1522 1535 request.req.cpu_stop.cpuid = cp->cpu_id;
1523 1536 request.req.cpu_stop.ctx = ctx;
1524 1537
1525 1538 return ((*pops->psm_cpu_ops)(&request));
1526 1539 }
1527 1540
1528 1541 int
1529 1542 mach_cpu_add(mach_cpu_add_arg_t *argp, processorid_t *cpuidp)
1530 1543 {
1531 1544 int rc;
1532 1545 struct psm_ops *pops = mach_set[0];
1533 1546 psm_cpu_request_t request;
1534 1547
1535 1548 if (pops->psm_cpu_ops == NULL) {
1536 1549 return (ENOTSUP);
1537 1550 }
1538 1551
1539 1552 request.pcr_cmd = PSM_CPU_ADD;
1540 1553 request.req.cpu_add.argp = argp;
1541 1554 request.req.cpu_add.cpuid = -1;
1542 1555 rc = (*pops->psm_cpu_ops)(&request);
1543 1556 if (rc == 0) {
1544 1557 ASSERT(request.req.cpu_add.cpuid != -1);
1545 1558 *cpuidp = request.req.cpu_add.cpuid;
1546 1559 }
1547 1560
1548 1561 return (rc);
1549 1562 }
1550 1563
1551 1564 int
1552 1565 mach_cpu_remove(processorid_t cpuid)
1553 1566 {
1554 1567 struct psm_ops *pops = mach_set[0];
1555 1568 psm_cpu_request_t request;
1556 1569
1557 1570 if (pops->psm_cpu_ops == NULL) {
1558 1571 return (ENOTSUP);
1559 1572 }
1560 1573
1561 1574 request.pcr_cmd = PSM_CPU_REMOVE;
1562 1575 request.req.cpu_remove.cpuid = cpuid;
1563 1576
1564 1577 return ((*pops->psm_cpu_ops)(&request));
1565 1578 }
1566 1579
1567 1580 /*
1568 1581 * Default handler to create device node for CPU.
1569 1582 * One reference count will be held on created device node.
1570 1583 */
1571 1584 static int
1572 1585 mach_cpu_create_devinfo(cpu_t *cp, dev_info_t **dipp)
1573 1586 {
1574 1587 int rv, circ;
1575 1588 dev_info_t *dip;
1576 1589 static kmutex_t cpu_node_lock;
1577 1590 static dev_info_t *cpu_nex_devi = NULL;
1578 1591
1579 1592 ASSERT(cp != NULL);
1580 1593 ASSERT(dipp != NULL);
1581 1594 *dipp = NULL;
1582 1595
1583 1596 if (cpu_nex_devi == NULL) {
1584 1597 mutex_enter(&cpu_node_lock);
1585 1598 /* First check whether cpus exists. */
1586 1599 cpu_nex_devi = ddi_find_devinfo("cpus", -1, 0);
1587 1600 /* Create cpus if it doesn't exist. */
1588 1601 if (cpu_nex_devi == NULL) {
1589 1602 ndi_devi_enter(ddi_root_node(), &circ);
1590 1603 rv = ndi_devi_alloc(ddi_root_node(), "cpus",
1591 1604 (pnode_t)DEVI_SID_NODEID, &dip);
1592 1605 if (rv != NDI_SUCCESS) {
1593 1606 mutex_exit(&cpu_node_lock);
1594 1607 cmn_err(CE_CONT,
1595 1608 "?failed to create cpu nexus device.\n");
1596 1609 return (PSM_FAILURE);
1597 1610 }
1598 1611 ASSERT(dip != NULL);
1599 1612 (void) ndi_devi_online(dip, 0);
1600 1613 ndi_devi_exit(ddi_root_node(), circ);
1601 1614 cpu_nex_devi = dip;
1602 1615 }
1603 1616 mutex_exit(&cpu_node_lock);
1604 1617 }
1605 1618
1606 1619 /*
1607 1620 * create a child node for cpu identified as 'cpu_id'
1608 1621 */
1609 1622 ndi_devi_enter(cpu_nex_devi, &circ);
1610 1623 dip = ddi_add_child(cpu_nex_devi, "cpu", DEVI_SID_NODEID, -1);
1611 1624 if (dip == NULL) {
1612 1625 cmn_err(CE_CONT,
1613 1626 "?failed to create device node for cpu%d.\n", cp->cpu_id);
1614 1627 rv = PSM_FAILURE;
1615 1628 } else {
1616 1629 *dipp = dip;
1617 1630 (void) ndi_hold_devi(dip);
1618 1631 rv = PSM_SUCCESS;
1619 1632 }
1620 1633 ndi_devi_exit(cpu_nex_devi, circ);
1621 1634
1622 1635 return (rv);
1623 1636 }
1624 1637
1625 1638 /*
1626 1639 * Create cpu device node in device tree and online it.
1627 1640 * Return created dip with reference count held if requested.
1628 1641 */
1629 1642 int
1630 1643 mach_cpu_create_device_node(struct cpu *cp, dev_info_t **dipp)
1631 1644 {
1632 1645 int rv;
1633 1646 dev_info_t *dip = NULL;
1634 1647
1635 1648 ASSERT(psm_cpu_create_devinfo != NULL);
1636 1649 rv = psm_cpu_create_devinfo(cp, &dip);
1637 1650 if (rv == PSM_SUCCESS) {
1638 1651 cpuid_set_cpu_properties(dip, cp->cpu_id, cp->cpu_m.mcpu_cpi);
1639 1652 /* Recursively attach driver for parent nexus device. */
1640 1653 if (i_ddi_attach_node_hierarchy(ddi_get_parent(dip)) ==
1641 1654 DDI_SUCCESS) {
1642 1655 /* Configure cpu itself and descendants. */
1643 1656 (void) ndi_devi_online(dip,
1644 1657 NDI_ONLINE_ATTACH | NDI_CONFIG);
1645 1658 }
1646 1659 if (dipp != NULL) {
1647 1660 *dipp = dip;
1648 1661 } else {
1649 1662 (void) ndi_rele_devi(dip);
1650 1663 }
1651 1664 }
1652 1665
1653 1666 return (rv);
1654 1667 }
1655 1668
1656 1669 /*
1657 1670 * The dipp contains one of following values on return:
1658 1671 * - NULL if no device node found
1659 1672 * - pointer to device node if found
1660 1673 */
1661 1674 int
1662 1675 mach_cpu_get_device_node(struct cpu *cp, dev_info_t **dipp)
1663 1676 {
1664 1677 *dipp = NULL;
1665 1678 if (psm_cpu_get_devinfo != NULL) {
1666 1679 if (psm_cpu_get_devinfo(cp, dipp) == PSM_SUCCESS) {
1667 1680 return (PSM_SUCCESS);
1668 1681 }
1669 1682 }
1670 1683
1671 1684 return (PSM_FAILURE);
1672 1685 }
1673 1686
1674 1687 /*ARGSUSED*/
1675 1688 static int
1676 1689 mach_translate_irq(dev_info_t *dip, int irqno)
1677 1690 {
1678 1691 return (irqno); /* default to NO translation */
1679 1692 }
1680 1693
1681 1694 static void
1682 1695 mach_notify_error(int level, char *errmsg)
1683 1696 {
1684 1697 /*
1685 1698 * SL_FATAL is pass in once panicstr is set, deliver it
1686 1699 * as CE_PANIC. Also, translate SL_ codes back to CE_
1687 1700 * codes for the psmi handler
1688 1701 */
1689 1702 if (level & SL_FATAL)
1690 1703 (*notify_error)(CE_PANIC, errmsg);
1691 1704 else if (level & SL_WARN)
1692 1705 (*notify_error)(CE_WARN, errmsg);
1693 1706 else if (level & SL_NOTE)
1694 1707 (*notify_error)(CE_NOTE, errmsg);
1695 1708 else if (level & SL_CONSOLE)
1696 1709 (*notify_error)(CE_CONT, errmsg);
1697 1710 }
1698 1711
1699 1712 /*
1700 1713 * It provides the default basic intr_ops interface for the new DDI
1701 1714 * interrupt framework if the PSM doesn't have one.
1702 1715 *
1703 1716 * Input:
1704 1717 * dip - pointer to the dev_info structure of the requested device
1705 1718 * hdlp - pointer to the internal interrupt handle structure for the
1706 1719 * requested interrupt
1707 1720 * intr_op - opcode for this call
1708 1721 * result - pointer to the integer that will hold the result to be
1709 1722 * passed back if return value is PSM_SUCCESS
1710 1723 *
1711 1724 * Output:
1712 1725 * return value is either PSM_SUCCESS or PSM_FAILURE
1713 1726 */
1714 1727 static int
1715 1728 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp,
1716 1729 psm_intr_op_t intr_op, int *result)
1717 1730 {
1718 1731 struct intrspec *ispec;
1719 1732
1720 1733 switch (intr_op) {
1721 1734 case PSM_INTR_OP_CHECK_MSI:
1722 1735 *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI |
1723 1736 DDI_INTR_TYPE_MSIX);
1724 1737 break;
1725 1738 case PSM_INTR_OP_ALLOC_VECTORS:
1726 1739 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
1727 1740 *result = 1;
1728 1741 else
1729 1742 *result = 0;
1730 1743 break;
1731 1744 case PSM_INTR_OP_FREE_VECTORS:
1732 1745 break;
1733 1746 case PSM_INTR_OP_NAVAIL_VECTORS:
1734 1747 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
1735 1748 *result = 1;
1736 1749 else
1737 1750 *result = 0;
1738 1751 break;
1739 1752 case PSM_INTR_OP_XLATE_VECTOR:
1740 1753 ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp;
1741 1754 *result = psm_translate_irq(dip, ispec->intrspec_vec);
1742 1755 break;
1743 1756 case PSM_INTR_OP_GET_CAP:
1744 1757 *result = 0;
1745 1758 break;
1746 1759 case PSM_INTR_OP_GET_PENDING:
1747 1760 case PSM_INTR_OP_CLEAR_MASK:
1748 1761 case PSM_INTR_OP_SET_MASK:
1749 1762 case PSM_INTR_OP_GET_SHARED:
1750 1763 case PSM_INTR_OP_SET_PRI:
1751 1764 case PSM_INTR_OP_SET_CAP:
1752 1765 case PSM_INTR_OP_SET_CPU:
1753 1766 case PSM_INTR_OP_GET_INTR:
1754 1767 default:
1755 1768 return (PSM_FAILURE);
1756 1769 }
1757 1770 return (PSM_SUCCESS);
1758 1771 }
1759 1772 /*
1760 1773 * Return 1 if CMT load balancing policies should be
1761 1774 * implemented across instances of the specified hardware
1762 1775 * sharing relationship.
1763 1776 */
1764 1777 int
1765 1778 pg_cmt_load_bal_hw(pghw_type_t hw)
1766 1779 {
1767 1780 if (hw == PGHW_IPIPE ||
1768 1781 hw == PGHW_FPU ||
1769 1782 hw == PGHW_PROCNODE ||
1770 1783 hw == PGHW_CHIP)
1771 1784 return (1);
1772 1785 else
1773 1786 return (0);
1774 1787 }
1775 1788 /*
1776 1789 * Return 1 if thread affinity polices should be implemented
1777 1790 * for instances of the specifed hardware sharing relationship.
1778 1791 */
1779 1792 int
1780 1793 pg_cmt_affinity_hw(pghw_type_t hw)
1781 1794 {
1782 1795 if (hw == PGHW_CACHE)
1783 1796 return (1);
1784 1797 else
1785 1798 return (0);
1786 1799 }
1787 1800
1788 1801 /*
1789 1802 * Return number of counter events requested to measure hardware capacity and
1790 1803 * utilization and setup CPC requests for specified CPU as needed
1791 1804 *
1792 1805 * May return 0 when platform or processor specific code knows that no CPC
1793 1806 * events should be programmed on this CPU or -1 when platform or processor
1794 1807 * specific code doesn't know which counter events are best to use and common
1795 1808 * code should decide for itself
1796 1809 */
1797 1810 int
1798 1811 /* LINTED E_FUNC_ARG_UNUSED */
1799 1812 cu_plat_cpc_init(cpu_t *cp, kcpc_request_list_t *reqs, int nreqs)
1800 1813 {
1801 1814 const char *impl_name;
1802 1815
1803 1816 /*
1804 1817 * Return error if pcbe_ops not set
1805 1818 */
1806 1819 if (pcbe_ops == NULL)
1807 1820 return (-1);
1808 1821
1809 1822 /*
1810 1823 * Return that no CPC events should be programmed on hyperthreaded
1811 1824 * Pentium 4 and return error for all other x86 processors to tell
1812 1825 * common code to decide what counter events to program on those CPUs
1813 1826 * for measuring hardware capacity and utilization
1814 1827 */
1815 1828 impl_name = pcbe_ops->pcbe_impl_name();
1816 1829 if (impl_name != NULL && strcmp(impl_name, PCBE_IMPL_NAME_P4HT) == 0)
1817 1830 return (0);
1818 1831 else
1819 1832 return (-1);
1820 1833 }
|
↓ open down ↓ |
1445 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX