Print this page
8626 make pcplusmp and apix warning-free
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/i86pc/io/mp_platform_misc.c
+++ new/usr/src/uts/i86pc/io/mp_platform_misc.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
|
↓ open down ↓ |
12 lines elided |
↑ open up ↑ |
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
23 + * Copyright 2017 Joyent, Inc.
23 24 */
24 25 /*
25 26 * Copyright (c) 2010, Intel Corporation.
26 27 * All rights reserved.
27 28 */
28 29
29 30 /*
30 31 * PSMI 1.1 extensions are supported only in 2.6 and later versions.
31 32 * PSMI 1.2 extensions are supported only in 2.7 and later versions.
32 33 * PSMI 1.3 and 1.4 extensions are supported in Solaris 10.
33 34 * PSMI 1.5 extensions are supported in Solaris Nevada.
34 35 * PSMI 1.6 extensions are supported in Solaris Nevada.
35 36 * PSMI 1.7 extensions are supported in Solaris Nevada.
36 37 */
37 38 #define PSMI_1_7
38 39
39 40 #include <sys/processor.h>
40 41 #include <sys/time.h>
41 42 #include <sys/psm.h>
42 43 #include <sys/smp_impldefs.h>
43 44 #include <sys/inttypes.h>
44 45 #include <sys/cram.h>
45 46 #include <sys/acpi/acpi.h>
46 47 #include <sys/acpica.h>
47 48 #include <sys/psm_common.h>
48 49 #include <sys/apic.h>
49 50 #include <sys/apic_common.h>
50 51 #include <sys/pit.h>
51 52 #include <sys/ddi.h>
52 53 #include <sys/sunddi.h>
53 54 #include <sys/ddi_impldefs.h>
54 55 #include <sys/pci.h>
55 56 #include <sys/promif.h>
56 57 #include <sys/x86_archext.h>
57 58 #include <sys/cpc_impl.h>
58 59 #include <sys/uadmin.h>
59 60 #include <sys/panic.h>
60 61 #include <sys/debug.h>
61 62 #include <sys/archsystm.h>
62 63 #include <sys/trap.h>
63 64 #include <sys/machsystm.h>
64 65 #include <sys/cpuvar.h>
65 66 #include <sys/rm_platter.h>
66 67 #include <sys/privregs.h>
67 68 #include <sys/cyclic.h>
68 69 #include <sys/note.h>
69 70 #include <sys/pci_intr_lib.h>
70 71 #include <sys/sunndi.h>
71 72 #include <sys/hpet.h>
72 73 #include <sys/clock.h>
73 74
74 75 /*
75 76 * Part of mp_platfrom_common.c that's used only by pcplusmp & xpv_psm
76 77 * but not apix.
77 78 * These functions may be moved to xpv_psm later when apix and pcplusmp
78 79 * are merged together
79 80 */
80 81
81 82 /*
82 83 * Local Function Prototypes
83 84 */
84 85 static void apic_mark_vector(uchar_t oldvector, uchar_t newvector);
85 86 static void apic_xlate_vector_free_timeout_handler(void *arg);
86 87 static int apic_check_stuck_interrupt(apic_irq_t *irq_ptr, int old_bind_cpu,
87 88 int new_bind_cpu, int apicindex, int intin_no, int which_irq,
88 89 struct ioapic_reprogram_data *drep);
89 90 static int apic_setup_irq_table(dev_info_t *dip, int irqno,
90 91 struct apic_io_intr *intrp, struct intrspec *ispec, iflag_t *intr_flagp,
91 92 int type);
92 93 static void apic_try_deferred_reprogram(int ipl, int vect);
93 94 static void delete_defer_repro_ent(int which_irq);
94 95 static void apic_ioapic_wait_pending_clear(int ioapicindex,
95 96 int intin_no);
96 97
97 98 extern int apic_acpi_translate_pci_irq(dev_info_t *dip, int busid, int devid,
98 99 int ipin, int *pci_irqp, iflag_t *intr_flagp);
99 100 extern int apic_handle_pci_pci_bridge(dev_info_t *idip, int child_devno,
100 101 int child_ipin, struct apic_io_intr **intrp);
101 102 extern uchar_t acpi_find_ioapic(int irq);
102 103 extern struct apic_io_intr *apic_find_io_intr_w_busid(int irqno, int busid);
103 104 extern int apic_find_bus_id(int bustype);
104 105 extern int apic_find_intin(uchar_t ioapic, uchar_t intin);
105 106 extern void apic_record_rdt_entry(apic_irq_t *irqptr, int irq);
106 107
107 108 extern int apic_sci_vect;
108 109 extern iflag_t apic_sci_flags;
109 110 /* ACPI HPET interrupt configuration; -1 if HPET not used */
110 111 extern int apic_hpet_vect;
111 112 extern iflag_t apic_hpet_flags;
112 113 extern int apic_intr_policy;
113 114 extern char *psm_name;
114 115
115 116 /*
116 117 * number of bits per byte, from <sys/param.h>
117 118 */
118 119 #define UCHAR_MAX UINT8_MAX
119 120
120 121 /* Max wait time (in repetitions) for flags to clear in an RDT entry. */
121 122 extern int apic_max_reps_clear_pending;
122 123
123 124 /* The irq # is implicit in the array index: */
124 125 struct ioapic_reprogram_data apic_reprogram_info[APIC_MAX_VECTOR+1];
125 126 /*
126 127 * APIC_MAX_VECTOR + 1 is the maximum # of IRQs as well. ioapic_reprogram_info
127 128 * is indexed by IRQ number, NOT by vector number.
128 129 */
129 130
130 131 extern int apic_int_busy_mark;
131 132 extern int apic_int_free_mark;
132 133 extern int apic_diff_for_redistribution;
133 134 extern int apic_sample_factor_redistribution;
134 135 extern int apic_redist_cpu_skip;
135 136 extern int apic_num_imbalance;
136 137 extern int apic_num_rebind;
137 138
138 139 /* timeout for xlate_vector, mark_vector */
139 140 int apic_revector_timeout = 16 * 10000; /* 160 millisec */
140 141
141 142 extern int apic_defconf;
142 143 extern int apic_irq_translate;
143 144
144 145 extern int apic_use_acpi_madt_only; /* 1=ONLY use MADT from ACPI */
145 146
146 147 extern uchar_t apic_io_vectbase[MAX_IO_APIC];
147 148
148 149 extern boolean_t ioapic_mask_workaround[MAX_IO_APIC];
149 150
150 151 /*
151 152 * First available slot to be used as IRQ index into the apic_irq_table
152 153 * for those interrupts (like MSI/X) that don't have a physical IRQ.
153 154 */
154 155 extern int apic_first_avail_irq;
155 156
156 157 /*
157 158 * apic_defer_reprogram_lock ensures that only one processor is handling
158 159 * deferred interrupt programming at *_intr_exit time.
159 160 */
160 161 static lock_t apic_defer_reprogram_lock;
161 162
162 163 /*
163 164 * The current number of deferred reprogrammings outstanding
164 165 */
165 166 uint_t apic_reprogram_outstanding = 0;
166 167
167 168 #ifdef DEBUG
168 169 /*
169 170 * Counters that keep track of deferred reprogramming stats
170 171 */
171 172 uint_t apic_intr_deferrals = 0;
172 173 uint_t apic_intr_deliver_timeouts = 0;
173 174 uint_t apic_last_ditch_reprogram_failures = 0;
174 175 uint_t apic_deferred_setup_failures = 0;
175 176 uint_t apic_defer_repro_total_retries = 0;
176 177 uint_t apic_defer_repro_successes = 0;
177 178 uint_t apic_deferred_spurious_enters = 0;
178 179 #endif
179 180
180 181 extern int apic_io_max;
181 182 extern struct apic_io_intr *apic_io_intrp;
182 183
183 184 uchar_t apic_vector_to_irq[APIC_MAX_VECTOR+1];
184 185
185 186 extern uint32_t eisa_level_intr_mask;
186 187 /* At least MSB will be set if EISA bus */
187 188
188 189 extern int apic_pci_bus_total;
189 190 extern uchar_t apic_single_pci_busid;
190 191
191 192 /*
192 193 * Following declarations are for revectoring; used when ISRs at different
193 194 * IPLs share an irq.
194 195 */
195 196 static lock_t apic_revector_lock;
196 197 int apic_revector_pending = 0;
197 198 static uchar_t *apic_oldvec_to_newvec;
198 199 static uchar_t *apic_newvec_to_oldvec;
199 200
200 201 /* ACPI Interrupt Source Override Structure ptr */
201 202 extern ACPI_MADT_INTERRUPT_OVERRIDE *acpi_isop;
202 203 extern int acpi_iso_cnt;
203 204
204 205 /*
205 206 * Auto-configuration routines
206 207 */
207 208
208 209 /*
209 210 * Initialise vector->ipl and ipl->pri arrays. level_intr and irqtable
210 211 * are also set to NULL. vector->irq is set to a value which cannot map
211 212 * to a real irq to show that it is free.
212 213 */
213 214 void
214 215 apic_init_common(void)
215 216 {
216 217 int i, j, indx;
217 218 int *iptr;
218 219
219 220 /*
220 221 * Initialize apic_ipls from apic_vectortoipl. This array is
221 222 * used in apic_intr_enter to determine the IPL to use for the
222 223 * corresponding vector. On some systems, due to hardware errata
223 224 * and interrupt sharing, the IPL may not correspond to the IPL listed
224 225 * in apic_vectortoipl (see apic_addspl and apic_delspl).
225 226 */
226 227 for (i = 0; i < (APIC_AVAIL_VECTOR / APIC_VECTOR_PER_IPL); i++) {
227 228 indx = i * APIC_VECTOR_PER_IPL;
228 229
229 230 for (j = 0; j < APIC_VECTOR_PER_IPL; j++, indx++)
230 231 apic_ipls[indx] = apic_vectortoipl[i];
231 232 }
232 233
233 234 /* cpu 0 is always up (for now) */
234 235 apic_cpus[0].aci_status = APIC_CPU_ONLINE | APIC_CPU_INTR_ENABLE;
235 236
236 237 iptr = (int *)&apic_irq_table[0];
237 238 for (i = 0; i <= APIC_MAX_VECTOR; i++) {
238 239 apic_level_intr[i] = 0;
239 240 *iptr++ = NULL;
240 241 apic_vector_to_irq[i] = APIC_RESV_IRQ;
241 242
242 243 /* These *must* be initted to B_TRUE! */
243 244 apic_reprogram_info[i].done = B_TRUE;
244 245 apic_reprogram_info[i].irqp = NULL;
245 246 apic_reprogram_info[i].tries = 0;
246 247 apic_reprogram_info[i].bindcpu = 0;
247 248 }
248 249
249 250 /*
250 251 * Allocate a dummy irq table entry for the reserved entry.
251 252 * This takes care of the race between removing an irq and
252 253 * clock detecting a CPU in that irq during interrupt load
253 254 * sampling.
254 255 */
255 256 apic_irq_table[APIC_RESV_IRQ] =
256 257 kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
257 258
258 259 mutex_init(&airq_mutex, NULL, MUTEX_DEFAULT, NULL);
259 260 }
260 261
261 262 void
262 263 ioapic_init_intr(int mask_apic)
263 264 {
264 265 int ioapic_ix;
265 266 struct intrspec ispec;
266 267 apic_irq_t *irqptr;
267 268 int i, j;
268 269 ulong_t iflag;
269 270
270 271 LOCK_INIT_CLEAR(&apic_revector_lock);
271 272 LOCK_INIT_CLEAR(&apic_defer_reprogram_lock);
272 273
273 274 /* mask interrupt vectors */
274 275 for (j = 0; j < apic_io_max && mask_apic; j++) {
275 276 int intin_max;
276 277
277 278 ioapic_ix = j;
278 279 /* Bits 23-16 define the maximum redirection entries */
279 280 intin_max = (ioapic_read(ioapic_ix, APIC_VERS_CMD) >> 16)
280 281 & 0xff;
281 282 for (i = 0; i <= intin_max; i++)
282 283 ioapic_write(ioapic_ix, APIC_RDT_CMD + 2 * i, AV_MASK);
283 284 }
284 285
285 286 /*
286 287 * Hack alert: deal with ACPI SCI interrupt chicken/egg here
287 288 */
288 289 if (apic_sci_vect > 0) {
289 290 /*
290 291 * acpica has already done add_avintr(); we just
291 292 * to finish the job by mimicing translate_irq()
292 293 *
293 294 * Fake up an intrspec and setup the tables
294 295 */
295 296 ispec.intrspec_vec = apic_sci_vect;
296 297 ispec.intrspec_pri = SCI_IPL;
297 298
298 299 if (apic_setup_irq_table(NULL, apic_sci_vect, NULL,
299 300 &ispec, &apic_sci_flags, DDI_INTR_TYPE_FIXED) < 0) {
300 301 cmn_err(CE_WARN, "!apic: SCI setup failed");
301 302 return;
302 303 }
303 304 irqptr = apic_irq_table[apic_sci_vect];
304 305
305 306 iflag = intr_clear();
306 307 lock_set(&apic_ioapic_lock);
307 308
308 309 /* Program I/O APIC */
309 310 (void) apic_setup_io_intr(irqptr, apic_sci_vect, B_FALSE);
310 311
311 312 lock_clear(&apic_ioapic_lock);
312 313 intr_restore(iflag);
313 314
314 315 irqptr->airq_share++;
315 316 }
316 317
317 318 /*
318 319 * Hack alert: deal with ACPI HPET interrupt chicken/egg here.
319 320 */
320 321 if (apic_hpet_vect > 0) {
321 322 /*
322 323 * hpet has already done add_avintr(); we just need
323 324 * to finish the job by mimicing translate_irq()
324 325 *
325 326 * Fake up an intrspec and setup the tables
326 327 */
327 328 ispec.intrspec_vec = apic_hpet_vect;
328 329 ispec.intrspec_pri = CBE_HIGH_PIL;
329 330
330 331 if (apic_setup_irq_table(NULL, apic_hpet_vect, NULL,
331 332 &ispec, &apic_hpet_flags, DDI_INTR_TYPE_FIXED) < 0) {
332 333 cmn_err(CE_WARN, "!apic: HPET setup failed");
333 334 return;
334 335 }
335 336 irqptr = apic_irq_table[apic_hpet_vect];
336 337
337 338 iflag = intr_clear();
338 339 lock_set(&apic_ioapic_lock);
339 340
340 341 /* Program I/O APIC */
341 342 (void) apic_setup_io_intr(irqptr, apic_hpet_vect, B_FALSE);
342 343
343 344 lock_clear(&apic_ioapic_lock);
344 345 intr_restore(iflag);
345 346
346 347 irqptr->airq_share++;
347 348 }
348 349 }
349 350
350 351 /*
351 352 * Add mask bits to disable interrupt vector from happening
352 353 * at or above IPL. In addition, it should remove mask bits
353 354 * to enable interrupt vectors below the given IPL.
354 355 *
355 356 * Both add and delspl are complicated by the fact that different interrupts
356 357 * may share IRQs. This can happen in two ways.
357 358 * 1. The same H/W line is shared by more than 1 device
358 359 * 1a. with interrupts at different IPLs
359 360 * 1b. with interrupts at same IPL
360 361 * 2. We ran out of vectors at a given IPL and started sharing vectors.
361 362 * 1b and 2 should be handled gracefully, except for the fact some ISRs
362 363 * will get called often when no interrupt is pending for the device.
363 364 * For 1a, we handle it at the higher IPL.
364 365 */
365 366 /*ARGSUSED*/
366 367 int
367 368 apic_addspl_common(int irqno, int ipl, int min_ipl, int max_ipl)
368 369 {
369 370 uchar_t vector;
370 371 ulong_t iflag;
371 372 apic_irq_t *irqptr, *irqheadptr;
372 373 int irqindex;
373 374
374 375 ASSERT(max_ipl <= UCHAR_MAX);
375 376 irqindex = IRQINDEX(irqno);
376 377
377 378 if ((irqindex == -1) || (!apic_irq_table[irqindex]))
378 379 return (PSM_FAILURE);
379 380
380 381 mutex_enter(&airq_mutex);
381 382 irqptr = irqheadptr = apic_irq_table[irqindex];
382 383
383 384 DDI_INTR_IMPLDBG((CE_CONT, "apic_addspl: dip=0x%p type=%d irqno=0x%x "
384 385 "vector=0x%x\n", (void *)irqptr->airq_dip,
385 386 irqptr->airq_mps_intr_index, irqno, irqptr->airq_vector));
386 387
387 388 while (irqptr) {
388 389 if (VIRTIRQ(irqindex, irqptr->airq_share_id) == irqno)
389 390 break;
390 391 irqptr = irqptr->airq_next;
391 392 }
392 393 irqptr->airq_share++;
393 394
394 395 mutex_exit(&airq_mutex);
395 396
396 397 /* return if it is not hardware interrupt */
397 398 if (irqptr->airq_mps_intr_index == RESERVE_INDEX)
398 399 return (PSM_SUCCESS);
399 400
400 401 /* Or if there are more interupts at a higher IPL */
401 402 if (ipl != max_ipl)
402 403 return (PSM_SUCCESS);
403 404
404 405 /*
405 406 * if apic_picinit() has not been called yet, just return.
406 407 * At the end of apic_picinit(), we will call setup_io_intr().
407 408 */
408 409
409 410 if (!apic_picinit_called)
410 411 return (PSM_SUCCESS);
411 412
412 413 /*
413 414 * Upgrade vector if max_ipl is not earlier ipl. If we cannot allocate,
414 415 * return failure.
415 416 */
416 417 if (irqptr->airq_ipl != max_ipl &&
417 418 !ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
418 419
419 420 vector = apic_allocate_vector(max_ipl, irqindex, 1);
420 421 if (vector == 0) {
421 422 irqptr->airq_share--;
422 423 return (PSM_FAILURE);
423 424 }
424 425 irqptr = irqheadptr;
425 426 apic_mark_vector(irqptr->airq_vector, vector);
426 427 while (irqptr) {
427 428 irqptr->airq_vector = vector;
428 429 irqptr->airq_ipl = (uchar_t)max_ipl;
429 430 /*
430 431 * reprogram irq being added and every one else
431 432 * who is not in the UNINIT state
432 433 */
433 434 if ((VIRTIRQ(irqindex, irqptr->airq_share_id) ==
434 435 irqno) || (irqptr->airq_temp_cpu != IRQ_UNINIT)) {
435 436 apic_record_rdt_entry(irqptr, irqindex);
436 437
437 438 iflag = intr_clear();
438 439 lock_set(&apic_ioapic_lock);
439 440
440 441 (void) apic_setup_io_intr(irqptr, irqindex,
441 442 B_FALSE);
442 443
443 444 lock_clear(&apic_ioapic_lock);
444 445 intr_restore(iflag);
445 446 }
446 447 irqptr = irqptr->airq_next;
447 448 }
448 449 return (PSM_SUCCESS);
449 450
450 451 } else if (irqptr->airq_ipl != max_ipl &&
451 452 ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
452 453 /*
453 454 * We cannot upgrade the vector, but we can change
454 455 * the IPL that this vector induces.
455 456 *
456 457 * Note that we subtract APIC_BASE_VECT from the vector
457 458 * here because this array is used in apic_intr_enter
458 459 * (no need to add APIC_BASE_VECT in that hot code
459 460 * path since we can do it in the rarely-executed path
460 461 * here).
461 462 */
462 463 apic_ipls[irqptr->airq_vector - APIC_BASE_VECT] =
463 464 (uchar_t)max_ipl;
464 465
465 466 irqptr = irqheadptr;
466 467 while (irqptr) {
467 468 irqptr->airq_ipl = (uchar_t)max_ipl;
468 469 irqptr = irqptr->airq_next;
469 470 }
470 471
471 472 return (PSM_SUCCESS);
472 473 }
473 474
474 475 ASSERT(irqptr);
475 476
476 477 iflag = intr_clear();
477 478 lock_set(&apic_ioapic_lock);
478 479
479 480 (void) apic_setup_io_intr(irqptr, irqindex, B_FALSE);
480 481
481 482 lock_clear(&apic_ioapic_lock);
482 483 intr_restore(iflag);
483 484
484 485 return (PSM_SUCCESS);
485 486 }
486 487
487 488 /*
488 489 * Recompute mask bits for the given interrupt vector.
489 490 * If there is no interrupt servicing routine for this
490 491 * vector, this function should disable interrupt vector
491 492 * from happening at all IPLs. If there are still
492 493 * handlers using the given vector, this function should
493 494 * disable the given vector from happening below the lowest
494 495 * IPL of the remaining hadlers.
495 496 */
496 497 /*ARGSUSED*/
497 498 int
498 499 apic_delspl_common(int irqno, int ipl, int min_ipl, int max_ipl)
499 500 {
500 501 uchar_t vector;
501 502 uint32_t bind_cpu;
502 503 int intin, irqindex;
503 504 int ioapic_ix;
504 505 apic_irq_t *irqptr, *preirqptr, *irqheadptr, *irqp;
505 506 ulong_t iflag;
506 507
507 508 mutex_enter(&airq_mutex);
508 509 irqindex = IRQINDEX(irqno);
509 510 irqptr = preirqptr = irqheadptr = apic_irq_table[irqindex];
510 511
511 512 DDI_INTR_IMPLDBG((CE_CONT, "apic_delspl: dip=0x%p type=%d irqno=0x%x "
512 513 "vector=0x%x\n", (void *)irqptr->airq_dip,
513 514 irqptr->airq_mps_intr_index, irqno, irqptr->airq_vector));
514 515
515 516 while (irqptr) {
516 517 if (VIRTIRQ(irqindex, irqptr->airq_share_id) == irqno)
517 518 break;
518 519 preirqptr = irqptr;
519 520 irqptr = irqptr->airq_next;
520 521 }
521 522 ASSERT(irqptr);
522 523
523 524 irqptr->airq_share--;
524 525
525 526 mutex_exit(&airq_mutex);
526 527
527 528 /*
528 529 * If there are more interrupts at a higher IPL, we don't need
529 530 * to disable anything.
530 531 */
531 532 if (ipl < max_ipl)
532 533 return (PSM_SUCCESS);
533 534
534 535 /* return if it is not hardware interrupt */
535 536 if (irqptr->airq_mps_intr_index == RESERVE_INDEX)
536 537 return (PSM_SUCCESS);
537 538
538 539 if (!apic_picinit_called) {
539 540 /*
540 541 * Clear irq_struct. If two devices shared an intpt
541 542 * line & 1 unloaded before picinit, we are hosed. But, then
542 543 * we hope the machine survive.
543 544 */
544 545 irqptr->airq_mps_intr_index = FREE_INDEX;
545 546 irqptr->airq_temp_cpu = IRQ_UNINIT;
546 547 apic_free_vector(irqptr->airq_vector);
547 548 return (PSM_SUCCESS);
548 549 }
549 550 /*
550 551 * Downgrade vector to new max_ipl if needed. If we cannot allocate,
551 552 * use old IPL. Not very elegant, but it should work.
552 553 */
553 554 if ((irqptr->airq_ipl != max_ipl) && (max_ipl != PSM_INVALID_IPL) &&
554 555 !ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
555 556 apic_irq_t *irqp;
556 557 if ((vector = apic_allocate_vector(max_ipl, irqno, 1))) {
557 558 apic_mark_vector(irqheadptr->airq_vector, vector);
558 559 irqp = irqheadptr;
559 560 while (irqp) {
560 561 irqp->airq_vector = vector;
561 562 irqp->airq_ipl = (uchar_t)max_ipl;
562 563 if (irqp->airq_temp_cpu != IRQ_UNINIT) {
563 564 apic_record_rdt_entry(irqp, irqindex);
564 565
565 566 iflag = intr_clear();
566 567 lock_set(&apic_ioapic_lock);
567 568
568 569 (void) apic_setup_io_intr(irqp,
569 570 irqindex, B_FALSE);
570 571
571 572 lock_clear(&apic_ioapic_lock);
572 573 intr_restore(iflag);
573 574 }
574 575 irqp = irqp->airq_next;
575 576 }
576 577 }
577 578
578 579 } else if (irqptr->airq_ipl != max_ipl &&
579 580 max_ipl != PSM_INVALID_IPL &&
580 581 ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
581 582
582 583 /*
583 584 * We cannot downgrade the IPL of the vector below the vector's
584 585 * hardware priority. If we did, it would be possible for a
585 586 * higher-priority hardware vector to interrupt a CPU running at an IPL
586 587 * lower than the hardware priority of the interrupting vector (but
587 588 * higher than the soft IPL of this IRQ). When this happens, we would
588 589 * then try to drop the IPL BELOW what it was (effectively dropping
589 590 * below base_spl) which would be potentially catastrophic.
590 591 *
591 592 * (e.g. Suppose the hardware vector associated with this IRQ is 0x40
592 593 * (hardware IPL of 4). Further assume that the old IPL of this IRQ
593 594 * was 4, but the new IPL is 1. If we forced vector 0x40 to result in
594 595 * an IPL of 1, it would be possible for the processor to be executing
595 596 * at IPL 3 and for an interrupt to come in on vector 0x40, interrupting
596 597 * the currently-executing ISR. When apic_intr_enter consults
597 598 * apic_irqs[], it will return 1, bringing the IPL of the CPU down to 1
598 599 * so even though the processor was running at IPL 4, an IPL 1
599 600 * interrupt will have interrupted it, which must not happen)).
600 601 *
601 602 * Effectively, this means that the hardware priority corresponding to
602 603 * the IRQ's IPL (in apic_ipls[]) cannot be lower than the vector's
603 604 * hardware priority.
604 605 *
605 606 * (In the above example, then, after removal of the IPL 4 device's
606 607 * interrupt handler, the new IPL will continue to be 4 because the
607 608 * hardware priority that IPL 1 implies is lower than the hardware
608 609 * priority of the vector used.)
609 610 */
610 611 /* apic_ipls is indexed by vector, starting at APIC_BASE_VECT */
611 612 const int apic_ipls_index = irqptr->airq_vector -
612 613 APIC_BASE_VECT;
613 614 const int vect_inherent_hwpri = irqptr->airq_vector >>
614 615 APIC_IPL_SHIFT;
615 616
616 617 /*
617 618 * If there are still devices using this IRQ, determine the
618 619 * new ipl to use.
619 620 */
620 621 if (irqptr->airq_share) {
621 622 int vect_desired_hwpri, hwpri;
622 623
623 624 ASSERT(max_ipl < MAXIPL);
624 625 vect_desired_hwpri = apic_ipltopri[max_ipl] >>
625 626 APIC_IPL_SHIFT;
626 627
627 628 /*
628 629 * If the desired IPL's hardware priority is lower
629 630 * than that of the vector, use the hardware priority
630 631 * of the vector to determine the new IPL.
631 632 */
632 633 hwpri = (vect_desired_hwpri < vect_inherent_hwpri) ?
633 634 vect_inherent_hwpri : vect_desired_hwpri;
634 635
635 636 /*
636 637 * Now, to get the right index for apic_vectortoipl,
|
↓ open down ↓ |
604 lines elided |
↑ open up ↑ |
637 638 * we need to subtract APIC_BASE_VECT from the
638 639 * hardware-vector-equivalent (in hwpri). Since hwpri
639 640 * is already shifted, we shift APIC_BASE_VECT before
640 641 * doing the subtraction.
641 642 */
642 643 hwpri -= (APIC_BASE_VECT >> APIC_IPL_SHIFT);
643 644
644 645 ASSERT(hwpri >= 0);
645 646 ASSERT(hwpri < MAXIPL);
646 647 max_ipl = apic_vectortoipl[hwpri];
647 - apic_ipls[apic_ipls_index] = max_ipl;
648 + apic_ipls[apic_ipls_index] = (uchar_t)max_ipl;
648 649
649 650 irqp = irqheadptr;
650 651 while (irqp) {
651 652 irqp->airq_ipl = (uchar_t)max_ipl;
652 653 irqp = irqp->airq_next;
653 654 }
654 655 } else {
655 656 /*
656 657 * No more devices on this IRQ, so reset this vector's
657 658 * element in apic_ipls to the original IPL for this
658 659 * vector
659 660 */
660 661 apic_ipls[apic_ipls_index] =
661 662 apic_vectortoipl[vect_inherent_hwpri];
662 663 }
663 664 }
664 665
665 666 /*
666 667 * If there are still active interrupts, we are done.
667 668 */
668 669 if (irqptr->airq_share)
669 670 return (PSM_SUCCESS);
670 671
671 672 iflag = intr_clear();
672 673 lock_set(&apic_ioapic_lock);
673 674
674 675 if (irqptr->airq_mps_intr_index == MSI_INDEX) {
675 676 /*
676 677 * Disable the MSI vector
677 678 * Make sure we only disable on the last
678 679 * of the multi-MSI support
679 680 */
680 681 if (i_ddi_intr_get_current_nenables(irqptr->airq_dip) == 1) {
681 682 apic_pci_msi_disable_mode(irqptr->airq_dip,
682 683 DDI_INTR_TYPE_MSI);
683 684 }
684 685 } else if (irqptr->airq_mps_intr_index == MSIX_INDEX) {
685 686 /*
686 687 * Disable the MSI-X vector
687 688 * needs to clear its mask and addr/data for each MSI-X
688 689 */
689 690 apic_pci_msi_unconfigure(irqptr->airq_dip, DDI_INTR_TYPE_MSIX,
690 691 irqptr->airq_origirq);
691 692 /*
692 693 * Make sure we only disable on the last MSI-X
693 694 */
694 695 if (i_ddi_intr_get_current_nenables(irqptr->airq_dip) == 1) {
695 696 apic_pci_msi_disable_mode(irqptr->airq_dip,
696 697 DDI_INTR_TYPE_MSIX);
697 698 }
698 699 } else {
699 700 /*
700 701 * The assumption here is that this is safe, even for
701 702 * systems with IOAPICs that suffer from the hardware
702 703 * erratum because all devices have been quiesced before
703 704 * they unregister their interrupt handlers. If that
704 705 * assumption turns out to be false, this mask operation
705 706 * can induce the same erratum result we're trying to
706 707 * avoid.
707 708 */
708 709 ioapic_ix = irqptr->airq_ioapicindex;
709 710 intin = irqptr->airq_intin_no;
710 711 ioapic_write(ioapic_ix, APIC_RDT_CMD + 2 * intin, AV_MASK);
711 712 }
712 713
713 714 apic_vt_ops->apic_intrmap_free_entry(&irqptr->airq_intrmap_private);
714 715
715 716 /*
716 717 * This irq entry is the only one in the chain.
717 718 */
718 719 if (irqheadptr->airq_next == NULL) {
719 720 ASSERT(irqheadptr == irqptr);
720 721 bind_cpu = irqptr->airq_temp_cpu;
721 722 if (((uint32_t)bind_cpu != IRQ_UNBOUND) &&
722 723 ((uint32_t)bind_cpu != IRQ_UNINIT)) {
723 724 ASSERT(apic_cpu_in_range(bind_cpu));
724 725 if (bind_cpu & IRQ_USER_BOUND) {
725 726 /* If hardbound, temp_cpu == cpu */
726 727 bind_cpu &= ~IRQ_USER_BOUND;
727 728 apic_cpus[bind_cpu].aci_bound--;
728 729 } else
729 730 apic_cpus[bind_cpu].aci_temp_bound--;
730 731 }
731 732 irqptr->airq_temp_cpu = IRQ_UNINIT;
732 733 irqptr->airq_mps_intr_index = FREE_INDEX;
733 734 lock_clear(&apic_ioapic_lock);
734 735 intr_restore(iflag);
735 736 apic_free_vector(irqptr->airq_vector);
736 737 return (PSM_SUCCESS);
737 738 }
738 739
739 740 /*
740 741 * If we get here, we are sharing the vector and there are more than
741 742 * one active irq entries in the chain.
742 743 */
743 744 lock_clear(&apic_ioapic_lock);
744 745 intr_restore(iflag);
745 746
746 747 mutex_enter(&airq_mutex);
747 748 /* Remove the irq entry from the chain */
748 749 if (irqptr == irqheadptr) { /* The irq entry is at the head */
749 750 apic_irq_table[irqindex] = irqptr->airq_next;
750 751 } else {
751 752 preirqptr->airq_next = irqptr->airq_next;
752 753 }
753 754 /* Free the irq entry */
754 755 kmem_free(irqptr, sizeof (apic_irq_t));
755 756 mutex_exit(&airq_mutex);
756 757
757 758 return (PSM_SUCCESS);
758 759 }
759 760
760 761 /*
761 762 * apic_introp_xlate() replaces apic_translate_irq() and is
762 763 * called only from apic_intr_ops(). With the new ADII framework,
763 764 * the priority can no longer be retrieved through i_ddi_get_intrspec().
764 765 * It has to be passed in from the caller.
765 766 *
766 767 * Return value:
767 768 * Success: irqno for the given device
768 769 * Failure: -1
769 770 */
770 771 int
771 772 apic_introp_xlate(dev_info_t *dip, struct intrspec *ispec, int type)
772 773 {
773 774 char dev_type[16];
774 775 int dev_len, pci_irq, newirq, bustype, devid, busid, i;
775 776 int irqno = ispec->intrspec_vec;
776 777 ddi_acc_handle_t cfg_handle;
777 778 uchar_t ipin;
778 779 struct apic_io_intr *intrp;
779 780 iflag_t intr_flag;
780 781 ACPI_SUBTABLE_HEADER *hp;
781 782 ACPI_MADT_INTERRUPT_OVERRIDE *isop;
782 783 apic_irq_t *airqp;
783 784 int parent_is_pci_or_pciex = 0;
784 785 int child_is_pciex = 0;
785 786
786 787 DDI_INTR_IMPLDBG((CE_CONT, "apic_introp_xlate: dip=0x%p name=%s "
787 788 "type=%d irqno=0x%x\n", (void *)dip, ddi_get_name(dip), type,
788 789 irqno));
789 790
790 791 dev_len = sizeof (dev_type);
791 792 if (ddi_getlongprop_buf(DDI_DEV_T_ANY, ddi_get_parent(dip),
792 793 DDI_PROP_DONTPASS, "device_type", (caddr_t)dev_type,
793 794 &dev_len) == DDI_PROP_SUCCESS) {
794 795 if ((strcmp(dev_type, "pci") == 0) ||
795 796 (strcmp(dev_type, "pciex") == 0))
796 797 parent_is_pci_or_pciex = 1;
797 798 }
798 799
799 800 if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip,
800 801 DDI_PROP_DONTPASS, "compatible", (caddr_t)dev_type,
801 802 &dev_len) == DDI_PROP_SUCCESS) {
802 803 if (strstr(dev_type, "pciex"))
803 804 child_is_pciex = 1;
804 805 }
805 806
806 807 if (DDI_INTR_IS_MSI_OR_MSIX(type)) {
807 808 if ((airqp = apic_find_irq(dip, ispec, type)) != NULL) {
808 809 airqp->airq_iflag.bustype =
809 810 child_is_pciex ? BUS_PCIE : BUS_PCI;
810 811 return (apic_vector_to_irq[airqp->airq_vector]);
811 812 }
812 813 return (apic_setup_irq_table(dip, irqno, NULL, ispec,
813 814 NULL, type));
814 815 }
815 816
816 817 bustype = 0;
817 818
818 819 /* check if we have already translated this irq */
819 820 mutex_enter(&airq_mutex);
820 821 newirq = apic_min_device_irq;
821 822 for (; newirq <= apic_max_device_irq; newirq++) {
822 823 airqp = apic_irq_table[newirq];
823 824 while (airqp) {
824 825 if ((airqp->airq_dip == dip) &&
825 826 (airqp->airq_origirq == irqno) &&
826 827 (airqp->airq_mps_intr_index != FREE_INDEX)) {
827 828
828 829 mutex_exit(&airq_mutex);
829 830 return (VIRTIRQ(newirq, airqp->airq_share_id));
830 831 }
831 832 airqp = airqp->airq_next;
832 833 }
833 834 }
834 835 mutex_exit(&airq_mutex);
835 836
836 837 if (apic_defconf)
837 838 goto defconf;
838 839
839 840 if ((dip == NULL) || (!apic_irq_translate && !apic_enable_acpi))
840 841 goto nonpci;
841 842
842 843 if (parent_is_pci_or_pciex) {
843 844 /* pci device */
844 845 if (acpica_get_bdf(dip, &busid, &devid, NULL) != 0)
845 846 goto nonpci;
846 847 if (busid == 0 && apic_pci_bus_total == 1)
847 848 busid = (int)apic_single_pci_busid;
848 849
849 850 if (pci_config_setup(dip, &cfg_handle) != DDI_SUCCESS)
850 851 return (-1);
851 852 ipin = pci_config_get8(cfg_handle, PCI_CONF_IPIN) - PCI_INTA;
852 853 pci_config_teardown(&cfg_handle);
853 854 if (apic_enable_acpi && !apic_use_acpi_madt_only) {
854 855 if (apic_acpi_translate_pci_irq(dip, busid, devid,
855 856 ipin, &pci_irq, &intr_flag) != ACPI_PSM_SUCCESS)
856 857 return (-1);
857 858
858 859 intr_flag.bustype = child_is_pciex ? BUS_PCIE : BUS_PCI;
859 860 return (apic_setup_irq_table(dip, pci_irq, NULL, ispec,
860 861 &intr_flag, type));
861 862 } else {
862 863 pci_irq = ((devid & 0x1f) << 2) | (ipin & 0x3);
863 864 if ((intrp = apic_find_io_intr_w_busid(pci_irq, busid))
864 865 == NULL) {
865 866 if ((pci_irq = apic_handle_pci_pci_bridge(dip,
866 867 devid, ipin, &intrp)) == -1)
867 868 return (-1);
868 869 }
869 870 return (apic_setup_irq_table(dip, pci_irq, intrp, ispec,
870 871 NULL, type));
871 872 }
872 873 } else if (strcmp(dev_type, "isa") == 0)
873 874 bustype = BUS_ISA;
874 875 else if (strcmp(dev_type, "eisa") == 0)
875 876 bustype = BUS_EISA;
876 877
877 878 nonpci:
878 879 if (apic_enable_acpi && !apic_use_acpi_madt_only) {
879 880 /* search iso entries first */
880 881 if (acpi_iso_cnt != 0) {
881 882 hp = (ACPI_SUBTABLE_HEADER *)acpi_isop;
882 883 i = 0;
883 884 while (i < acpi_iso_cnt) {
884 885 if (hp->Type ==
885 886 ACPI_MADT_TYPE_INTERRUPT_OVERRIDE) {
886 887 isop =
887 888 (ACPI_MADT_INTERRUPT_OVERRIDE *) hp;
888 889 if (isop->Bus == 0 &&
889 890 isop->SourceIrq == irqno) {
890 891 newirq = isop->GlobalIrq;
891 892 intr_flag.intr_po =
892 893 isop->IntiFlags &
893 894 ACPI_MADT_POLARITY_MASK;
894 895 intr_flag.intr_el =
895 896 (isop->IntiFlags &
896 897 ACPI_MADT_TRIGGER_MASK)
897 898 >> 2;
898 899 intr_flag.bustype = BUS_ISA;
899 900
900 901 return (apic_setup_irq_table(
901 902 dip, newirq, NULL, ispec,
902 903 &intr_flag, type));
903 904
904 905 }
905 906 i++;
906 907 }
907 908 hp = (ACPI_SUBTABLE_HEADER *)(((char *)hp) +
908 909 hp->Length);
909 910 }
910 911 }
911 912 intr_flag.intr_po = INTR_PO_ACTIVE_HIGH;
912 913 intr_flag.intr_el = INTR_EL_EDGE;
913 914 intr_flag.bustype = BUS_ISA;
914 915 return (apic_setup_irq_table(dip, irqno, NULL, ispec,
915 916 &intr_flag, type));
916 917 } else {
917 918 if (bustype == 0) /* not initialized */
918 919 bustype = eisa_level_intr_mask ? BUS_EISA : BUS_ISA;
919 920 for (i = 0; i < 2; i++) {
920 921 if (((busid = apic_find_bus_id(bustype)) != -1) &&
921 922 ((intrp = apic_find_io_intr_w_busid(irqno, busid))
922 923 != NULL)) {
923 924 if ((newirq = apic_setup_irq_table(dip, irqno,
924 925 intrp, ispec, NULL, type)) != -1) {
925 926 return (newirq);
926 927 }
927 928 goto defconf;
928 929 }
929 930 bustype = (bustype == BUS_EISA) ? BUS_ISA : BUS_EISA;
930 931 }
931 932 }
932 933
933 934 /* MPS default configuration */
934 935 defconf:
935 936 newirq = apic_setup_irq_table(dip, irqno, NULL, ispec, NULL, type);
936 937 if (newirq == -1)
937 938 return (-1);
|
↓ open down ↓ |
280 lines elided |
↑ open up ↑ |
938 939 ASSERT(IRQINDEX(newirq) == irqno);
939 940 ASSERT(apic_irq_table[irqno]);
940 941 return (newirq);
941 942 }
942 943
943 944 /*
944 945 * Attempt to share vector with someone else
945 946 */
946 947 static int
947 948 apic_share_vector(int irqno, iflag_t *intr_flagp, short intr_index, int ipl,
948 - uchar_t ioapicindex, uchar_t ipin, apic_irq_t **irqptrp)
949 + uchar_t ioapicindex, uchar_t ipin, apic_irq_t **irqptrp)
949 950 {
950 951 #ifdef DEBUG
951 952 apic_irq_t *tmpirqp = NULL;
952 953 #endif /* DEBUG */
953 954 apic_irq_t *irqptr, dummyirq;
954 955 int newirq, chosen_irq = -1, share = 127;
955 956 int lowest, highest, i;
956 957 uchar_t share_id;
957 958
958 959 DDI_INTR_IMPLDBG((CE_CONT, "apic_share_vector: irqno=0x%x "
959 960 "intr_index=0x%x ipl=0x%x\n", irqno, intr_index, ipl));
960 961
961 962 highest = apic_ipltopri[ipl] + APIC_VECTOR_MASK;
962 963 lowest = apic_ipltopri[ipl-1] + APIC_VECTOR_PER_IPL;
963 964
964 965 if (highest < lowest) /* Both ipl and ipl-1 map to same pri */
965 966 lowest -= APIC_VECTOR_PER_IPL;
966 967 dummyirq.airq_mps_intr_index = intr_index;
967 968 dummyirq.airq_ioapicindex = ioapicindex;
968 969 dummyirq.airq_intin_no = ipin;
969 970 if (intr_flagp)
970 971 dummyirq.airq_iflag = *intr_flagp;
971 972 apic_record_rdt_entry(&dummyirq, irqno);
972 973 for (i = lowest; i <= highest; i++) {
973 974 newirq = apic_vector_to_irq[i];
974 975 if (newirq == APIC_RESV_IRQ)
975 976 continue;
976 977 irqptr = apic_irq_table[newirq];
977 978
978 979 if ((dummyirq.airq_rdt_entry & 0xFF00) !=
979 980 (irqptr->airq_rdt_entry & 0xFF00))
980 981 /* not compatible */
981 982 continue;
982 983
983 984 if (irqptr->airq_share < share) {
984 985 share = irqptr->airq_share;
985 986 chosen_irq = newirq;
986 987 }
987 988 }
988 989 if (chosen_irq != -1) {
989 990 /*
990 991 * Assign a share id which is free or which is larger
991 992 * than the largest one.
992 993 */
993 994 share_id = 1;
994 995 mutex_enter(&airq_mutex);
995 996 irqptr = apic_irq_table[chosen_irq];
996 997 while (irqptr) {
997 998 if (irqptr->airq_mps_intr_index == FREE_INDEX) {
998 999 share_id = irqptr->airq_share_id;
999 1000 break;
1000 1001 }
1001 1002 if (share_id <= irqptr->airq_share_id)
1002 1003 share_id = irqptr->airq_share_id + 1;
1003 1004 #ifdef DEBUG
1004 1005 tmpirqp = irqptr;
1005 1006 #endif /* DEBUG */
1006 1007 irqptr = irqptr->airq_next;
1007 1008 }
1008 1009 if (!irqptr) {
1009 1010 irqptr = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
1010 1011 irqptr->airq_temp_cpu = IRQ_UNINIT;
1011 1012 irqptr->airq_next =
1012 1013 apic_irq_table[chosen_irq]->airq_next;
1013 1014 apic_irq_table[chosen_irq]->airq_next = irqptr;
1014 1015 #ifdef DEBUG
1015 1016 tmpirqp = apic_irq_table[chosen_irq];
1016 1017 #endif /* DEBUG */
1017 1018 }
1018 1019 irqptr->airq_mps_intr_index = intr_index;
1019 1020 irqptr->airq_ioapicindex = ioapicindex;
1020 1021 irqptr->airq_intin_no = ipin;
1021 1022 if (intr_flagp)
1022 1023 irqptr->airq_iflag = *intr_flagp;
1023 1024 irqptr->airq_vector = apic_irq_table[chosen_irq]->airq_vector;
1024 1025 irqptr->airq_share_id = share_id;
1025 1026 apic_record_rdt_entry(irqptr, irqno);
1026 1027 *irqptrp = irqptr;
1027 1028 #ifdef DEBUG
1028 1029 /* shuffle the pointers to test apic_delspl path */
1029 1030 if (tmpirqp) {
1030 1031 tmpirqp->airq_next = irqptr->airq_next;
1031 1032 irqptr->airq_next = apic_irq_table[chosen_irq];
1032 1033 apic_irq_table[chosen_irq] = irqptr;
1033 1034 }
1034 1035 #endif /* DEBUG */
1035 1036 mutex_exit(&airq_mutex);
1036 1037 return (VIRTIRQ(chosen_irq, share_id));
1037 1038 }
1038 1039 return (-1);
1039 1040 }
1040 1041
1041 1042 /*
1042 1043 * Allocate/Initialize the apic_irq_table[] entry for given irqno. If the entry
|
↓ open down ↓ |
84 lines elided |
↑ open up ↑ |
1043 1044 * is used already, we will try to allocate a new irqno.
1044 1045 *
1045 1046 * Return value:
1046 1047 * Success: irqno
1047 1048 * Failure: -1
1048 1049 */
1049 1050 static int
1050 1051 apic_setup_irq_table(dev_info_t *dip, int irqno, struct apic_io_intr *intrp,
1051 1052 struct intrspec *ispec, iflag_t *intr_flagp, int type)
1052 1053 {
1053 - int origirq = ispec->intrspec_vec;
1054 - uchar_t ipl = ispec->intrspec_pri;
1054 + int origirq;
1055 + uchar_t ipl;
1055 1056 int newirq, intr_index;
1056 1057 uchar_t ipin, ioapic, ioapicindex, vector;
1057 1058 apic_irq_t *irqptr;
1058 1059 major_t major;
1059 1060 dev_info_t *sdip;
1060 1061
1062 + ASSERT(ispec != NULL);
1063 +
1064 + origirq = ispec->intrspec_vec;
1065 + ipl = ispec->intrspec_pri;
1066 +
1061 1067 DDI_INTR_IMPLDBG((CE_CONT, "apic_setup_irq_table: dip=0x%p type=%d "
1062 1068 "irqno=0x%x origirq=0x%x\n", (void *)dip, type, irqno, origirq));
1063 1069
1064 - ASSERT(ispec != NULL);
1065 -
1066 1070 major = (dip != NULL) ? ddi_driver_major(dip) : 0;
1067 1071
1068 1072 if (DDI_INTR_IS_MSI_OR_MSIX(type)) {
1069 1073 /* MSI/X doesn't need to setup ioapic stuffs */
1070 1074 ioapicindex = 0xff;
1071 1075 ioapic = 0xff;
1072 1076 ipin = (uchar_t)0xff;
1073 1077 intr_index = (type == DDI_INTR_TYPE_MSI) ? MSI_INDEX :
1074 1078 MSIX_INDEX;
1075 1079 mutex_enter(&airq_mutex);
1076 1080 if ((irqno = apic_allocate_irq(apic_first_avail_irq)) == -1) {
1077 1081 mutex_exit(&airq_mutex);
1078 1082 /* need an irq for MSI/X to index into autovect[] */
1079 1083 cmn_err(CE_WARN, "No interrupt irq: %s instance %d",
1080 1084 ddi_get_name(dip), ddi_get_instance(dip));
1081 1085 return (-1);
1082 1086 }
1083 1087 mutex_exit(&airq_mutex);
1084 1088
1085 1089 } else if (intrp != NULL) {
1086 1090 intr_index = (int)(intrp - apic_io_intrp);
1087 1091 ioapic = intrp->intr_destid;
1088 1092 ipin = intrp->intr_destintin;
1089 1093 /* Find ioapicindex. If destid was ALL, we will exit with 0. */
1090 1094 for (ioapicindex = apic_io_max - 1; ioapicindex; ioapicindex--)
1091 1095 if (apic_io_id[ioapicindex] == ioapic)
1092 1096 break;
1093 1097 ASSERT((ioapic == apic_io_id[ioapicindex]) ||
1094 1098 (ioapic == INTR_ALL_APIC));
1095 1099
1096 1100 /* check whether this intin# has been used by another irqno */
1097 1101 if ((newirq = apic_find_intin(ioapicindex, ipin)) != -1) {
1098 1102 return (newirq);
1099 1103 }
1100 1104
1101 1105 } else if (intr_flagp != NULL) {
1102 1106 /* ACPI case */
1103 1107 intr_index = ACPI_INDEX;
1104 1108 ioapicindex = acpi_find_ioapic(irqno);
1105 1109 ASSERT(ioapicindex != 0xFF);
1106 1110 ioapic = apic_io_id[ioapicindex];
1107 1111 ipin = irqno - apic_io_vectbase[ioapicindex];
1108 1112 if (apic_irq_table[irqno] &&
1109 1113 apic_irq_table[irqno]->airq_mps_intr_index == ACPI_INDEX) {
1110 1114 ASSERT(apic_irq_table[irqno]->airq_intin_no == ipin &&
1111 1115 apic_irq_table[irqno]->airq_ioapicindex ==
1112 1116 ioapicindex);
1113 1117 return (irqno);
|
↓ open down ↓ |
38 lines elided |
↑ open up ↑ |
1114 1118 }
1115 1119
1116 1120 } else {
1117 1121 /* default configuration */
1118 1122 ioapicindex = 0;
1119 1123 ioapic = apic_io_id[ioapicindex];
1120 1124 ipin = (uchar_t)irqno;
1121 1125 intr_index = DEFAULT_INDEX;
1122 1126 }
1123 1127
1124 - if (ispec == NULL) {
1125 - APIC_VERBOSE_IOAPIC((CE_WARN, "No intrspec for irqno = %x\n",
1126 - irqno));
1127 - } else if ((vector = apic_allocate_vector(ipl, irqno, 0)) == 0) {
1128 + if ((vector = apic_allocate_vector(ipl, irqno, 0)) == 0) {
1128 1129 if ((newirq = apic_share_vector(irqno, intr_flagp, intr_index,
1129 1130 ipl, ioapicindex, ipin, &irqptr)) != -1) {
1130 1131 irqptr->airq_ipl = ipl;
1131 1132 irqptr->airq_origirq = (uchar_t)origirq;
1132 1133 irqptr->airq_dip = dip;
1133 1134 irqptr->airq_major = major;
1134 1135 sdip = apic_irq_table[IRQINDEX(newirq)]->airq_dip;
1135 1136 /* This is OK to do really */
1136 1137 if (sdip == NULL) {
1137 1138 cmn_err(CE_WARN, "Sharing vectors: %s"
1138 1139 " instance %d and SCI",
1139 1140 ddi_get_name(dip), ddi_get_instance(dip));
1140 1141 } else {
1141 1142 cmn_err(CE_WARN, "Sharing vectors: %s"
1142 1143 " instance %d and %s instance %d",
1143 1144 ddi_get_name(sdip), ddi_get_instance(sdip),
1144 1145 ddi_get_name(dip), ddi_get_instance(dip));
1145 1146 }
1146 1147 return (newirq);
1147 1148 }
1148 1149 /* try high priority allocation now that share has failed */
1149 1150 if ((vector = apic_allocate_vector(ipl, irqno, 1)) == 0) {
1150 1151 cmn_err(CE_WARN, "No interrupt vector: %s instance %d",
1151 1152 ddi_get_name(dip), ddi_get_instance(dip));
1152 1153 return (-1);
1153 1154 }
1154 1155 }
1155 1156
1156 1157 mutex_enter(&airq_mutex);
1157 1158 if (apic_irq_table[irqno] == NULL) {
1158 1159 irqptr = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
1159 1160 irqptr->airq_temp_cpu = IRQ_UNINIT;
1160 1161 apic_irq_table[irqno] = irqptr;
1161 1162 } else {
1162 1163 irqptr = apic_irq_table[irqno];
1163 1164 if (irqptr->airq_mps_intr_index != FREE_INDEX) {
1164 1165 /*
1165 1166 * The slot is used by another irqno, so allocate
1166 1167 * a free irqno for this interrupt
1167 1168 */
1168 1169 newirq = apic_allocate_irq(apic_first_avail_irq);
1169 1170 if (newirq == -1) {
1170 1171 mutex_exit(&airq_mutex);
1171 1172 return (-1);
1172 1173 }
1173 1174 irqno = newirq;
1174 1175 irqptr = apic_irq_table[irqno];
1175 1176 if (irqptr == NULL) {
1176 1177 irqptr = kmem_zalloc(sizeof (apic_irq_t),
1177 1178 KM_SLEEP);
1178 1179 irqptr->airq_temp_cpu = IRQ_UNINIT;
1179 1180 apic_irq_table[irqno] = irqptr;
1180 1181 }
1181 1182 vector = apic_modify_vector(vector, newirq);
1182 1183 }
1183 1184 }
1184 1185 apic_max_device_irq = max(irqno, apic_max_device_irq);
1185 1186 apic_min_device_irq = min(irqno, apic_min_device_irq);
1186 1187 mutex_exit(&airq_mutex);
1187 1188 irqptr->airq_ioapicindex = ioapicindex;
1188 1189 irqptr->airq_intin_no = ipin;
1189 1190 irqptr->airq_ipl = ipl;
1190 1191 irqptr->airq_vector = vector;
1191 1192 irqptr->airq_origirq = (uchar_t)origirq;
1192 1193 irqptr->airq_share_id = 0;
1193 1194 irqptr->airq_mps_intr_index = (short)intr_index;
1194 1195 irqptr->airq_dip = dip;
1195 1196 irqptr->airq_major = major;
1196 1197 irqptr->airq_cpu = apic_bind_intr(dip, irqno, ioapic, ipin);
1197 1198 if (intr_flagp)
1198 1199 irqptr->airq_iflag = *intr_flagp;
1199 1200
1200 1201 if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
1201 1202 /* setup I/O APIC entry for non-MSI/X interrupts */
1202 1203 apic_record_rdt_entry(irqptr, irqno);
1203 1204 }
1204 1205 return (irqno);
1205 1206 }
1206 1207
1207 1208 /*
1208 1209 * return the cpu to which this intr should be bound.
1209 1210 * Check properties or any other mechanism to see if user wants it
1210 1211 * bound to a specific CPU. If so, return the cpu id with high bit set.
1211 1212 * If not, use the policy to choose a cpu and return the id.
1212 1213 */
1213 1214 uint32_t
1214 1215 apic_bind_intr(dev_info_t *dip, int irq, uchar_t ioapicid, uchar_t intin)
1215 1216 {
1216 1217 int instance, instno, prop_len, bind_cpu, count;
1217 1218 uint_t i, rc;
1218 1219 uint32_t cpu;
1219 1220 major_t major;
1220 1221 char *name, *drv_name, *prop_val, *cptr;
|
↓ open down ↓ |
83 lines elided |
↑ open up ↑ |
1221 1222 char prop_name[32];
1222 1223 ulong_t iflag;
1223 1224
1224 1225
1225 1226 if (apic_intr_policy == INTR_LOWEST_PRIORITY)
1226 1227 return (IRQ_UNBOUND);
1227 1228
1228 1229 if (apic_nproc == 1)
1229 1230 return (0);
1230 1231
1231 - drv_name = NULL;
1232 - rc = DDI_PROP_NOT_FOUND;
1233 - major = (major_t)-1;
1234 - if (dip != NULL) {
1235 - name = ddi_get_name(dip);
1236 - major = ddi_name_to_major(name);
1237 - drv_name = ddi_major_to_name(major);
1238 - instance = ddi_get_instance(dip);
1239 - if (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) {
1240 - i = apic_min_device_irq;
1241 - for (; i <= apic_max_device_irq; i++) {
1232 + if (dip == NULL) {
1233 + iflag = intr_clear();
1234 + lock_set(&apic_ioapic_lock);
1235 + bind_cpu = apic_get_next_bind_cpu();
1236 + lock_clear(&apic_ioapic_lock);
1237 + intr_restore(iflag);
1242 1238
1243 - if ((i == irq) || (apic_irq_table[i] == NULL) ||
1244 - (apic_irq_table[i]->airq_mps_intr_index
1245 - == FREE_INDEX))
1246 - continue;
1239 + cmn_err(CE_CONT, "!%s: irq 0x%x "
1240 + "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n",
1241 + psm_name, irq, apic_irq_table[irq]->airq_vector, ioapicid,
1242 + intin, bind_cpu & ~IRQ_USER_BOUND);
1247 1243
1248 - if ((apic_irq_table[i]->airq_major == major) &&
1249 - (!(apic_irq_table[i]->airq_cpu &
1250 - IRQ_USER_BOUND))) {
1244 + return ((uint32_t)bind_cpu);
1245 + }
1251 1246
1252 - cpu = apic_irq_table[i]->airq_cpu;
1247 + name = ddi_get_name(dip);
1248 + major = ddi_name_to_major(name);
1249 + drv_name = ddi_major_to_name(major);
1250 + instance = ddi_get_instance(dip);
1251 + if (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) {
1252 + i = apic_min_device_irq;
1253 + for (; i <= apic_max_device_irq; i++) {
1254 + if ((i == irq) || (apic_irq_table[i] == NULL) ||
1255 + (apic_irq_table[i]->airq_mps_intr_index
1256 + == FREE_INDEX))
1257 + continue;
1253 1258
1254 - cmn_err(CE_CONT,
1255 - "!%s: %s (%s) instance #%d "
1256 - "irq 0x%x vector 0x%x ioapic 0x%x "
1257 - "intin 0x%x is bound to cpu %d\n",
1258 - psm_name,
1259 - name, drv_name, instance, irq,
1260 - apic_irq_table[irq]->airq_vector,
1261 - ioapicid, intin, cpu);
1262 - return (cpu);
1263 - }
1259 + if ((apic_irq_table[i]->airq_major == major) &&
1260 + (!(apic_irq_table[i]->airq_cpu & IRQ_USER_BOUND))) {
1261 + cpu = apic_irq_table[i]->airq_cpu;
1262 +
1263 + cmn_err(CE_CONT,
1264 + "!%s: %s (%s) instance #%d "
1265 + "irq 0x%x vector 0x%x ioapic 0x%x "
1266 + "intin 0x%x is bound to cpu %d\n",
1267 + psm_name,
1268 + name, drv_name, instance, irq,
1269 + apic_irq_table[irq]->airq_vector,
1270 + ioapicid, intin, cpu);
1271 + return (cpu);
1264 1272 }
1265 1273 }
1266 - /*
1267 - * search for "drvname"_intpt_bind_cpus property first, the
1268 - * syntax of the property should be "a[,b,c,...]" where
1269 - * instance 0 binds to cpu a, instance 1 binds to cpu b,
1270 - * instance 3 binds to cpu c...
1271 - * ddi_getlongprop() will search /option first, then /
1272 - * if "drvname"_intpt_bind_cpus doesn't exist, then find
1273 - * intpt_bind_cpus property. The syntax is the same, and
1274 - * it applies to all the devices if its "drvname" specific
1275 - * property doesn't exist
1276 - */
1277 - (void) strcpy(prop_name, drv_name);
1278 - (void) strcat(prop_name, "_intpt_bind_cpus");
1279 - rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0, prop_name,
1280 - (caddr_t)&prop_val, &prop_len);
1281 - if (rc != DDI_PROP_SUCCESS) {
1282 - rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0,
1283 - "intpt_bind_cpus", (caddr_t)&prop_val, &prop_len);
1284 - }
1285 1274 }
1275 + /*
1276 + * search for "drvname"_intpt_bind_cpus property first, the
1277 + * syntax of the property should be "a[,b,c,...]" where
1278 + * instance 0 binds to cpu a, instance 1 binds to cpu b,
1279 + * instance 3 binds to cpu c...
1280 + * ddi_getlongprop() will search /option first, then /
1281 + * if "drvname"_intpt_bind_cpus doesn't exist, then find
1282 + * intpt_bind_cpus property. The syntax is the same, and
1283 + * it applies to all the devices if its "drvname" specific
1284 + * property doesn't exist
1285 + */
1286 + (void) strcpy(prop_name, drv_name);
1287 + (void) strcat(prop_name, "_intpt_bind_cpus");
1288 + rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0, prop_name,
1289 + (caddr_t)&prop_val, &prop_len);
1290 + if (rc != DDI_PROP_SUCCESS) {
1291 + rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0,
1292 + "intpt_bind_cpus", (caddr_t)&prop_val, &prop_len);
1293 + }
1286 1294 if (rc == DDI_PROP_SUCCESS) {
1287 1295 for (i = count = 0; i < (prop_len - 1); i++)
1288 1296 if (prop_val[i] == ',')
1289 1297 count++;
1290 1298 if (prop_val[i-1] != ',')
1291 1299 count++;
1292 1300 /*
1293 1301 * if somehow the binding instances defined in the
1294 1302 * property are not enough for this instno., then
1295 1303 * reuse the pattern for the next instance until
1296 1304 * it reaches the requested instno
1297 1305 */
1298 1306 instno = instance % count;
1299 1307 i = 0;
1300 1308 cptr = prop_val;
1301 1309 while (i < instno)
1302 1310 if (*cptr++ == ',')
1303 1311 i++;
1304 1312 bind_cpu = stoi(&cptr);
1305 1313 kmem_free(prop_val, prop_len);
1306 1314 /* if specific CPU is bogus, then default to next cpu */
1307 1315 if (!apic_cpu_in_range(bind_cpu)) {
1308 1316 cmn_err(CE_WARN, "%s: %s=%s: CPU %d not present",
1309 1317 psm_name, prop_name, prop_val, bind_cpu);
|
↓ open down ↓ |
14 lines elided |
↑ open up ↑ |
1310 1318 rc = DDI_PROP_NOT_FOUND;
1311 1319 } else {
1312 1320 /* indicate that we are bound at user request */
1313 1321 bind_cpu |= IRQ_USER_BOUND;
1314 1322 }
1315 1323 /*
1316 1324 * no need to check apic_cpus[].aci_status, if specific CPU is
1317 1325 * not up, then post_cpu_start will handle it.
1318 1326 */
1319 1327 }
1328 +
1320 1329 if (rc != DDI_PROP_SUCCESS) {
1321 1330 iflag = intr_clear();
1322 1331 lock_set(&apic_ioapic_lock);
1323 1332 bind_cpu = apic_get_next_bind_cpu();
1324 1333 lock_clear(&apic_ioapic_lock);
1325 1334 intr_restore(iflag);
1326 1335 }
1327 1336
1328 - if (drv_name != NULL)
1329 - cmn_err(CE_CONT, "!%s: %s (%s) instance %d irq 0x%x "
1330 - "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n",
1331 - psm_name, name, drv_name, instance, irq,
1332 - apic_irq_table[irq]->airq_vector, ioapicid, intin,
1333 - bind_cpu & ~IRQ_USER_BOUND);
1334 - else
1335 - cmn_err(CE_CONT, "!%s: irq 0x%x "
1336 - "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n",
1337 - psm_name, irq, apic_irq_table[irq]->airq_vector, ioapicid,
1338 - intin, bind_cpu & ~IRQ_USER_BOUND);
1337 + cmn_err(CE_CONT, "!%s: %s (%s) instance %d irq 0x%x "
1338 + "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n",
1339 + psm_name, name, drv_name, instance, irq,
1340 + apic_irq_table[irq]->airq_vector, ioapicid, intin,
1341 + bind_cpu & ~IRQ_USER_BOUND);
1339 1342
1340 1343 return ((uint32_t)bind_cpu);
1341 1344 }
1342 1345
1343 1346 /*
1344 1347 * Mark vector as being in the process of being deleted. Interrupts
1345 1348 * may still come in on some CPU. The moment an interrupt comes with
1346 1349 * the new vector, we know we can free the old one. Called only from
1347 1350 * addspl and delspl with interrupts disabled. Because an interrupt
1348 1351 * can be shared, but no interrupt from either device may come in,
1349 1352 * we also use a timeout mechanism, which we arbitrarily set to
1350 1353 * apic_revector_timeout microseconds.
1351 1354 */
1352 1355 static void
1353 1356 apic_mark_vector(uchar_t oldvector, uchar_t newvector)
1354 1357 {
1355 1358 ulong_t iflag;
1356 1359
1357 1360 iflag = intr_clear();
1358 1361 lock_set(&apic_revector_lock);
1359 1362 if (!apic_oldvec_to_newvec) {
1360 1363 apic_oldvec_to_newvec =
1361 1364 kmem_zalloc(sizeof (newvector) * APIC_MAX_VECTOR * 2,
1362 1365 KM_NOSLEEP);
1363 1366
1364 1367 if (!apic_oldvec_to_newvec) {
1365 1368 /*
1366 1369 * This failure is not catastrophic.
1367 1370 * But, the oldvec will never be freed.
1368 1371 */
1369 1372 apic_error |= APIC_ERR_MARK_VECTOR_FAIL;
1370 1373 lock_clear(&apic_revector_lock);
1371 1374 intr_restore(iflag);
1372 1375 return;
1373 1376 }
1374 1377 apic_newvec_to_oldvec = &apic_oldvec_to_newvec[APIC_MAX_VECTOR];
1375 1378 }
1376 1379
1377 1380 /* See if we already did this for drivers which do double addintrs */
1378 1381 if (apic_oldvec_to_newvec[oldvector] != newvector) {
1379 1382 apic_oldvec_to_newvec[oldvector] = newvector;
1380 1383 apic_newvec_to_oldvec[newvector] = oldvector;
1381 1384 apic_revector_pending++;
1382 1385 }
1383 1386 lock_clear(&apic_revector_lock);
1384 1387 intr_restore(iflag);
1385 1388 (void) timeout(apic_xlate_vector_free_timeout_handler,
1386 1389 (void *)(uintptr_t)oldvector, drv_usectohz(apic_revector_timeout));
1387 1390 }
1388 1391
1389 1392 /*
1390 1393 * xlate_vector is called from intr_enter if revector_pending is set.
1391 1394 * It will xlate it if needed and mark the old vector as free.
1392 1395 */
1393 1396 uchar_t
1394 1397 apic_xlate_vector(uchar_t vector)
1395 1398 {
1396 1399 uchar_t newvector, oldvector = 0;
1397 1400
1398 1401 lock_set(&apic_revector_lock);
1399 1402 /* Do we really need to do this ? */
1400 1403 if (!apic_revector_pending) {
1401 1404 lock_clear(&apic_revector_lock);
1402 1405 return (vector);
1403 1406 }
1404 1407 if ((newvector = apic_oldvec_to_newvec[vector]) != 0)
1405 1408 oldvector = vector;
1406 1409 else {
1407 1410 /*
1408 1411 * The incoming vector is new . See if a stale entry is
1409 1412 * remaining
1410 1413 */
1411 1414 if ((oldvector = apic_newvec_to_oldvec[vector]) != 0)
1412 1415 newvector = vector;
1413 1416 }
1414 1417
1415 1418 if (oldvector) {
1416 1419 apic_revector_pending--;
1417 1420 apic_oldvec_to_newvec[oldvector] = 0;
1418 1421 apic_newvec_to_oldvec[newvector] = 0;
1419 1422 apic_free_vector(oldvector);
1420 1423 lock_clear(&apic_revector_lock);
1421 1424 /* There could have been more than one reprogramming! */
1422 1425 return (apic_xlate_vector(newvector));
1423 1426 }
1424 1427 lock_clear(&apic_revector_lock);
1425 1428 return (vector);
1426 1429 }
1427 1430
1428 1431 void
1429 1432 apic_xlate_vector_free_timeout_handler(void *arg)
1430 1433 {
1431 1434 ulong_t iflag;
1432 1435 uchar_t oldvector, newvector;
1433 1436
1434 1437 oldvector = (uchar_t)(uintptr_t)arg;
1435 1438 iflag = intr_clear();
1436 1439 lock_set(&apic_revector_lock);
1437 1440 if ((newvector = apic_oldvec_to_newvec[oldvector]) != 0) {
1438 1441 apic_free_vector(oldvector);
1439 1442 apic_oldvec_to_newvec[oldvector] = 0;
1440 1443 apic_newvec_to_oldvec[newvector] = 0;
1441 1444 apic_revector_pending--;
1442 1445 }
1443 1446
1444 1447 lock_clear(&apic_revector_lock);
1445 1448 intr_restore(iflag);
1446 1449 }
1447 1450
1448 1451 /*
1449 1452 * Bind interrupt corresponding to irq_ptr to bind_cpu.
1450 1453 * Must be called with interrupts disabled and apic_ioapic_lock held
1451 1454 */
1452 1455 int
1453 1456 apic_rebind(apic_irq_t *irq_ptr, int bind_cpu,
1454 1457 struct ioapic_reprogram_data *drep)
1455 1458 {
1456 1459 int ioapicindex, intin_no;
1457 1460 uint32_t airq_temp_cpu;
1458 1461 apic_cpus_info_t *cpu_infop;
1459 1462 uint32_t rdt_entry;
1460 1463 int which_irq;
1461 1464 ioapic_rdt_t irdt;
1462 1465
1463 1466 which_irq = apic_vector_to_irq[irq_ptr->airq_vector];
1464 1467
1465 1468 intin_no = irq_ptr->airq_intin_no;
1466 1469 ioapicindex = irq_ptr->airq_ioapicindex;
1467 1470 airq_temp_cpu = irq_ptr->airq_temp_cpu;
1468 1471 if (airq_temp_cpu != IRQ_UNINIT && airq_temp_cpu != IRQ_UNBOUND) {
1469 1472 if (airq_temp_cpu & IRQ_USER_BOUND)
1470 1473 /* Mask off high bit so it can be used as array index */
1471 1474 airq_temp_cpu &= ~IRQ_USER_BOUND;
1472 1475
1473 1476 ASSERT(apic_cpu_in_range(airq_temp_cpu));
1474 1477 }
1475 1478
1476 1479 /*
1477 1480 * Can't bind to a CPU that's not accepting interrupts:
1478 1481 */
1479 1482 cpu_infop = &apic_cpus[bind_cpu & ~IRQ_USER_BOUND];
1480 1483 if (!(cpu_infop->aci_status & APIC_CPU_INTR_ENABLE))
1481 1484 return (1);
1482 1485
1483 1486 /*
1484 1487 * If we are about to change the interrupt vector for this interrupt,
1485 1488 * and this interrupt is level-triggered, attached to an IOAPIC,
1486 1489 * has been delivered to a CPU and that CPU has not handled it
1487 1490 * yet, we cannot reprogram the IOAPIC now.
1488 1491 */
1489 1492 if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr->airq_mps_intr_index)) {
1490 1493
1491 1494 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex,
1492 1495 intin_no);
1493 1496
1494 1497 if ((irq_ptr->airq_vector != RDT_VECTOR(rdt_entry)) &&
1495 1498 apic_check_stuck_interrupt(irq_ptr, airq_temp_cpu,
1496 1499 bind_cpu, ioapicindex, intin_no, which_irq, drep) != 0) {
1497 1500
1498 1501 return (0);
1499 1502 }
1500 1503
1501 1504 /*
1502 1505 * NOTE: We do not unmask the RDT here, as an interrupt MAY
1503 1506 * still come in before we have a chance to reprogram it below.
1504 1507 * The reprogramming below will simultaneously change and
1505 1508 * unmask the RDT entry.
1506 1509 */
1507 1510
1508 1511 if ((uint32_t)bind_cpu == IRQ_UNBOUND) {
1509 1512 irdt.ir_lo = AV_LDEST | AV_LOPRI |
1510 1513 irq_ptr->airq_rdt_entry;
1511 1514
1512 1515 irdt.ir_hi = AV_TOALL >> APIC_ID_BIT_OFFSET;
1513 1516
1514 1517 apic_vt_ops->apic_intrmap_alloc_entry(
1515 1518 &irq_ptr->airq_intrmap_private, NULL,
1516 1519 DDI_INTR_TYPE_FIXED, 1, ioapicindex);
1517 1520 apic_vt_ops->apic_intrmap_map_entry(
1518 1521 irq_ptr->airq_intrmap_private, (void *)&irdt,
1519 1522 DDI_INTR_TYPE_FIXED, 1);
1520 1523 apic_vt_ops->apic_intrmap_record_rdt(
1521 1524 irq_ptr->airq_intrmap_private, &irdt);
1522 1525
1523 1526 /* Write the RDT entry -- no specific CPU binding */
1524 1527 WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin_no,
1525 1528 irdt.ir_hi | AV_TOALL);
1526 1529
1527 1530 if (airq_temp_cpu != IRQ_UNINIT && airq_temp_cpu !=
1528 1531 IRQ_UNBOUND)
1529 1532 apic_cpus[airq_temp_cpu].aci_temp_bound--;
1530 1533
1531 1534 /*
1532 1535 * Write the vector, trigger, and polarity portion of
1533 1536 * the RDT
1534 1537 */
1535 1538 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin_no,
1536 1539 irdt.ir_lo);
1537 1540
1538 1541 irq_ptr->airq_temp_cpu = IRQ_UNBOUND;
1539 1542 return (0);
1540 1543 }
1541 1544 }
1542 1545
1543 1546 if (bind_cpu & IRQ_USER_BOUND) {
1544 1547 cpu_infop->aci_bound++;
1545 1548 } else {
1546 1549 cpu_infop->aci_temp_bound++;
1547 1550 }
1548 1551 ASSERT(apic_cpu_in_range(bind_cpu));
1549 1552
1550 1553 if ((airq_temp_cpu != IRQ_UNBOUND) && (airq_temp_cpu != IRQ_UNINIT)) {
1551 1554 apic_cpus[airq_temp_cpu].aci_temp_bound--;
1552 1555 }
1553 1556 if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr->airq_mps_intr_index)) {
1554 1557
1555 1558 irdt.ir_lo = AV_PDEST | AV_FIXED | irq_ptr->airq_rdt_entry;
1556 1559 irdt.ir_hi = cpu_infop->aci_local_id;
1557 1560
1558 1561 apic_vt_ops->apic_intrmap_alloc_entry(
1559 1562 &irq_ptr->airq_intrmap_private, NULL, DDI_INTR_TYPE_FIXED,
1560 1563 1, ioapicindex);
1561 1564 apic_vt_ops->apic_intrmap_map_entry(
1562 1565 irq_ptr->airq_intrmap_private,
1563 1566 (void *)&irdt, DDI_INTR_TYPE_FIXED, 1);
1564 1567 apic_vt_ops->apic_intrmap_record_rdt(
1565 1568 irq_ptr->airq_intrmap_private, &irdt);
1566 1569
1567 1570 /* Write the RDT entry -- bind to a specific CPU: */
1568 1571 WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin_no,
1569 1572 irdt.ir_hi);
1570 1573
1571 1574 /* Write the vector, trigger, and polarity portion of the RDT */
1572 1575 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin_no,
1573 1576 irdt.ir_lo);
1574 1577
1575 1578 } else {
1576 1579 int type = (irq_ptr->airq_mps_intr_index == MSI_INDEX) ?
1577 1580 DDI_INTR_TYPE_MSI : DDI_INTR_TYPE_MSIX;
1578 1581 if (type == DDI_INTR_TYPE_MSI) {
1579 1582 if (irq_ptr->airq_ioapicindex ==
1580 1583 irq_ptr->airq_origirq) {
1581 1584 /* first one */
1582 1585 DDI_INTR_IMPLDBG((CE_CONT, "apic_rebind: call "
1583 1586 "apic_pci_msi_enable_vector\n"));
1584 1587 apic_pci_msi_enable_vector(irq_ptr,
1585 1588 type, which_irq, irq_ptr->airq_vector,
1586 1589 irq_ptr->airq_intin_no,
1587 1590 cpu_infop->aci_local_id);
1588 1591 }
1589 1592 if ((irq_ptr->airq_ioapicindex +
1590 1593 irq_ptr->airq_intin_no - 1) ==
1591 1594 irq_ptr->airq_origirq) { /* last one */
1592 1595 DDI_INTR_IMPLDBG((CE_CONT, "apic_rebind: call "
1593 1596 "apic_pci_msi_enable_mode\n"));
1594 1597 apic_pci_msi_enable_mode(irq_ptr->airq_dip,
1595 1598 type, which_irq);
1596 1599 }
1597 1600 } else { /* MSI-X */
1598 1601 apic_pci_msi_enable_vector(irq_ptr, type,
1599 1602 irq_ptr->airq_origirq, irq_ptr->airq_vector, 1,
1600 1603 cpu_infop->aci_local_id);
1601 1604 apic_pci_msi_enable_mode(irq_ptr->airq_dip, type,
1602 1605 irq_ptr->airq_origirq);
1603 1606 }
1604 1607 }
1605 1608 irq_ptr->airq_temp_cpu = (uint32_t)bind_cpu;
1606 1609 apic_redist_cpu_skip &= ~(1 << (bind_cpu & ~IRQ_USER_BOUND));
1607 1610 return (0);
1608 1611 }
1609 1612
1610 1613 static void
1611 1614 apic_last_ditch_clear_remote_irr(int ioapic_ix, int intin_no)
1612 1615 {
1613 1616 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no)
1614 1617 & AV_REMOTE_IRR) != 0) {
1615 1618 /*
1616 1619 * Trying to clear the bit through normal
1617 1620 * channels has failed. So as a last-ditch
1618 1621 * effort, try to set the trigger mode to
1619 1622 * edge, then to level. This has been
1620 1623 * observed to work on many systems.
1621 1624 */
1622 1625 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1623 1626 intin_no,
1624 1627 READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1625 1628 intin_no) & ~AV_LEVEL);
1626 1629
1627 1630 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1628 1631 intin_no,
1629 1632 READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1630 1633 intin_no) | AV_LEVEL);
1631 1634
1632 1635 /*
1633 1636 * If the bit's STILL set, this interrupt may
1634 1637 * be hosed.
1635 1638 */
1636 1639 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1637 1640 intin_no) & AV_REMOTE_IRR) != 0) {
1638 1641
1639 1642 prom_printf("%s: Remote IRR still "
1640 1643 "not clear for IOAPIC %d intin %d.\n"
1641 1644 "\tInterrupts to this pin may cease "
1642 1645 "functioning.\n", psm_name, ioapic_ix,
1643 1646 intin_no);
1644 1647 #ifdef DEBUG
1645 1648 apic_last_ditch_reprogram_failures++;
1646 1649 #endif
1647 1650 }
1648 1651 }
1649 1652 }
1650 1653
1651 1654 /*
1652 1655 * This function is protected by apic_ioapic_lock coupled with the
1653 1656 * fact that interrupts are disabled.
1654 1657 */
1655 1658 static void
1656 1659 delete_defer_repro_ent(int which_irq)
1657 1660 {
1658 1661 ASSERT(which_irq >= 0);
1659 1662 ASSERT(which_irq <= 255);
1660 1663 ASSERT(LOCK_HELD(&apic_ioapic_lock));
1661 1664
1662 1665 if (apic_reprogram_info[which_irq].done)
1663 1666 return;
1664 1667
1665 1668 apic_reprogram_info[which_irq].done = B_TRUE;
1666 1669
1667 1670 #ifdef DEBUG
1668 1671 apic_defer_repro_total_retries +=
1669 1672 apic_reprogram_info[which_irq].tries;
1670 1673
1671 1674 apic_defer_repro_successes++;
1672 1675 #endif
1673 1676
1674 1677 if (--apic_reprogram_outstanding == 0) {
1675 1678
1676 1679 setlvlx = psm_intr_exit_fn();
1677 1680 }
1678 1681 }
1679 1682
1680 1683
1681 1684 /*
1682 1685 * Interrupts must be disabled during this function to prevent
1683 1686 * self-deadlock. Interrupts are disabled because this function
1684 1687 * is called from apic_check_stuck_interrupt(), which is called
1685 1688 * from apic_rebind(), which requires its caller to disable interrupts.
1686 1689 */
1687 1690 static void
1688 1691 add_defer_repro_ent(apic_irq_t *irq_ptr, int which_irq, int new_bind_cpu)
1689 1692 {
1690 1693 ASSERT(which_irq >= 0);
1691 1694 ASSERT(which_irq <= 255);
1692 1695 ASSERT(!interrupts_enabled());
1693 1696
1694 1697 /*
1695 1698 * On the off-chance that there's already a deferred
1696 1699 * reprogramming on this irq, check, and if so, just update the
1697 1700 * CPU and irq pointer to which the interrupt is targeted, then return.
1698 1701 */
1699 1702 if (!apic_reprogram_info[which_irq].done) {
1700 1703 apic_reprogram_info[which_irq].bindcpu = new_bind_cpu;
1701 1704 apic_reprogram_info[which_irq].irqp = irq_ptr;
1702 1705 return;
1703 1706 }
1704 1707
1705 1708 apic_reprogram_info[which_irq].irqp = irq_ptr;
1706 1709 apic_reprogram_info[which_irq].bindcpu = new_bind_cpu;
1707 1710 apic_reprogram_info[which_irq].tries = 0;
1708 1711 /*
1709 1712 * This must be the last thing set, since we're not
1710 1713 * grabbing any locks, apic_try_deferred_reprogram() will
1711 1714 * make its decision about using this entry iff done
1712 1715 * is false.
1713 1716 */
1714 1717 apic_reprogram_info[which_irq].done = B_FALSE;
1715 1718
1716 1719 /*
1717 1720 * If there were previously no deferred reprogrammings, change
1718 1721 * setlvlx to call apic_try_deferred_reprogram()
1719 1722 */
1720 1723 if (++apic_reprogram_outstanding == 1) {
1721 1724
1722 1725 setlvlx = apic_try_deferred_reprogram;
1723 1726 }
1724 1727 }
1725 1728
1726 1729 static void
1727 1730 apic_try_deferred_reprogram(int prev_ipl, int irq)
1728 1731 {
1729 1732 int reproirq;
1730 1733 ulong_t iflag;
1731 1734 struct ioapic_reprogram_data *drep;
1732 1735
1733 1736 (*psm_intr_exit_fn())(prev_ipl, irq);
1734 1737
1735 1738 if (!lock_try(&apic_defer_reprogram_lock)) {
1736 1739 return;
1737 1740 }
1738 1741
1739 1742 /*
1740 1743 * Acquire the apic_ioapic_lock so that any other operations that
1741 1744 * may affect the apic_reprogram_info state are serialized.
1742 1745 * It's still possible for the last deferred reprogramming to clear
1743 1746 * between the time we entered this function and the time we get to
1744 1747 * the for loop below. In that case, *setlvlx will have been set
1745 1748 * back to *_intr_exit and drep will be NULL. (There's no way to
1746 1749 * stop that from happening -- we would need to grab a lock before
1747 1750 * calling *setlvlx, which is neither realistic nor prudent).
1748 1751 */
1749 1752 iflag = intr_clear();
1750 1753 lock_set(&apic_ioapic_lock);
1751 1754
1752 1755 /*
1753 1756 * For each deferred RDT entry, try to reprogram it now. Note that
1754 1757 * there is no lock acquisition to read apic_reprogram_info because
1755 1758 * '.done' is set only after the other fields in the structure are set.
1756 1759 */
1757 1760
1758 1761 drep = NULL;
1759 1762 for (reproirq = 0; reproirq <= APIC_MAX_VECTOR; reproirq++) {
1760 1763 if (apic_reprogram_info[reproirq].done == B_FALSE) {
1761 1764 drep = &apic_reprogram_info[reproirq];
1762 1765 break;
1763 1766 }
1764 1767 }
1765 1768
1766 1769 /*
1767 1770 * Either we found a deferred action to perform, or
1768 1771 * we entered this function spuriously, after *setlvlx
1769 1772 * was restored to point to *_intr_exit. Any other
1770 1773 * permutation is invalid.
1771 1774 */
1772 1775 ASSERT(drep != NULL || *setlvlx == psm_intr_exit_fn());
1773 1776
1774 1777 /*
1775 1778 * Though we can't really do anything about errors
1776 1779 * at this point, keep track of them for reporting.
1777 1780 * Note that it is very possible for apic_setup_io_intr
1778 1781 * to re-register this very timeout if the Remote IRR bit
1779 1782 * has not yet cleared.
1780 1783 */
1781 1784
1782 1785 #ifdef DEBUG
1783 1786 if (drep != NULL) {
1784 1787 if (apic_setup_io_intr(drep, reproirq, B_TRUE) != 0) {
1785 1788 apic_deferred_setup_failures++;
1786 1789 }
1787 1790 } else {
1788 1791 apic_deferred_spurious_enters++;
1789 1792 }
1790 1793 #else
1791 1794 if (drep != NULL)
1792 1795 (void) apic_setup_io_intr(drep, reproirq, B_TRUE);
1793 1796 #endif
1794 1797
1795 1798 lock_clear(&apic_ioapic_lock);
1796 1799 intr_restore(iflag);
1797 1800
1798 1801 lock_clear(&apic_defer_reprogram_lock);
1799 1802 }
1800 1803
1801 1804 static void
1802 1805 apic_ioapic_wait_pending_clear(int ioapic_ix, int intin_no)
1803 1806 {
1804 1807 int waited;
1805 1808
1806 1809 /*
1807 1810 * Wait for the delivery pending bit to clear.
1808 1811 */
1809 1812 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no) &
1810 1813 (AV_LEVEL|AV_PENDING)) == (AV_LEVEL|AV_PENDING)) {
1811 1814
1812 1815 /*
1813 1816 * If we're still waiting on the delivery of this interrupt,
1814 1817 * continue to wait here until it is delivered (this should be
1815 1818 * a very small amount of time, but include a timeout just in
1816 1819 * case).
1817 1820 */
1818 1821 for (waited = 0; waited < apic_max_reps_clear_pending;
1819 1822 waited++) {
1820 1823 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1821 1824 intin_no) & AV_PENDING) == 0) {
1822 1825 break;
1823 1826 }
1824 1827 }
1825 1828 }
1826 1829 }
1827 1830
1828 1831
1829 1832 /*
1830 1833 * Checks to see if the IOAPIC interrupt entry specified has its Remote IRR
1831 1834 * bit set. Calls functions that modify the function that setlvlx points to,
1832 1835 * so that the reprogramming can be retried very shortly.
1833 1836 *
1834 1837 * This function will mask the RDT entry if the interrupt is level-triggered.
1835 1838 * (The caller is responsible for unmasking the RDT entry.)
1836 1839 *
1837 1840 * Returns non-zero if the caller should defer IOAPIC reprogramming.
1838 1841 */
1839 1842 static int
1840 1843 apic_check_stuck_interrupt(apic_irq_t *irq_ptr, int old_bind_cpu,
1841 1844 int new_bind_cpu, int ioapic_ix, int intin_no, int which_irq,
1842 1845 struct ioapic_reprogram_data *drep)
1843 1846 {
1844 1847 int32_t rdt_entry;
1845 1848 int waited;
1846 1849 int reps = 0;
1847 1850
1848 1851 /*
1849 1852 * Wait for the delivery pending bit to clear.
1850 1853 */
1851 1854 do {
1852 1855 ++reps;
1853 1856
1854 1857 apic_ioapic_wait_pending_clear(ioapic_ix, intin_no);
1855 1858
1856 1859 /*
1857 1860 * Mask the RDT entry, but only if it's a level-triggered
1858 1861 * interrupt
1859 1862 */
1860 1863 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1861 1864 intin_no);
1862 1865 if ((rdt_entry & (AV_LEVEL|AV_MASK)) == AV_LEVEL) {
1863 1866
1864 1867 /* Mask it */
1865 1868 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no,
1866 1869 AV_MASK | rdt_entry);
1867 1870 }
1868 1871
1869 1872 if ((rdt_entry & AV_LEVEL) == AV_LEVEL) {
1870 1873 /*
1871 1874 * If there was a race and an interrupt was injected
1872 1875 * just before we masked, check for that case here.
1873 1876 * Then, unmask the RDT entry and try again. If we're
1874 1877 * on our last try, don't unmask (because we want the
1875 1878 * RDT entry to remain masked for the rest of the
1876 1879 * function).
1877 1880 */
1878 1881 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1879 1882 intin_no);
1880 1883 if ((rdt_entry & AV_PENDING) &&
1881 1884 (reps < apic_max_reps_clear_pending)) {
1882 1885 /* Unmask it */
1883 1886 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1884 1887 intin_no, rdt_entry & ~AV_MASK);
1885 1888 }
1886 1889 }
1887 1890
1888 1891 } while ((rdt_entry & AV_PENDING) &&
1889 1892 (reps < apic_max_reps_clear_pending));
1890 1893
1891 1894 #ifdef DEBUG
1892 1895 if (rdt_entry & AV_PENDING)
1893 1896 apic_intr_deliver_timeouts++;
1894 1897 #endif
1895 1898
1896 1899 /*
1897 1900 * If the remote IRR bit is set, then the interrupt has been sent
1898 1901 * to a CPU for processing. We have no choice but to wait for
1899 1902 * that CPU to process the interrupt, at which point the remote IRR
1900 1903 * bit will be cleared.
1901 1904 */
1902 1905 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no) &
1903 1906 (AV_LEVEL|AV_REMOTE_IRR)) == (AV_LEVEL|AV_REMOTE_IRR)) {
1904 1907
1905 1908 /*
1906 1909 * If the CPU that this RDT is bound to is NOT the current
1907 1910 * CPU, wait until that CPU handles the interrupt and ACKs
1908 1911 * it. If this interrupt is not bound to any CPU (that is,
1909 1912 * if it's bound to the logical destination of "anyone"), it
1910 1913 * may have been delivered to the current CPU so handle that
1911 1914 * case by deferring the reprogramming (below).
1912 1915 */
1913 1916 if ((old_bind_cpu != IRQ_UNBOUND) &&
1914 1917 (old_bind_cpu != IRQ_UNINIT) &&
1915 1918 (old_bind_cpu != psm_get_cpu_id())) {
1916 1919 for (waited = 0; waited < apic_max_reps_clear_pending;
1917 1920 waited++) {
1918 1921 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1919 1922 intin_no) & AV_REMOTE_IRR) == 0) {
1920 1923
1921 1924 delete_defer_repro_ent(which_irq);
1922 1925
1923 1926 /* Remote IRR has cleared! */
1924 1927 return (0);
1925 1928 }
1926 1929 }
1927 1930 }
1928 1931
1929 1932 /*
1930 1933 * If we waited and the Remote IRR bit is still not cleared,
1931 1934 * AND if we've invoked the timeout APIC_REPROGRAM_MAX_TIMEOUTS
1932 1935 * times for this interrupt, try the last-ditch workaround:
1933 1936 */
1934 1937 if (drep && drep->tries >= APIC_REPROGRAM_MAX_TRIES) {
1935 1938
1936 1939 apic_last_ditch_clear_remote_irr(ioapic_ix, intin_no);
1937 1940
1938 1941 /* Mark this one as reprogrammed: */
1939 1942 delete_defer_repro_ent(which_irq);
1940 1943
1941 1944 return (0);
1942 1945 } else {
1943 1946 #ifdef DEBUG
1944 1947 apic_intr_deferrals++;
1945 1948 #endif
1946 1949
1947 1950 /*
1948 1951 * If waiting for the Remote IRR bit (above) didn't
1949 1952 * allow it to clear, defer the reprogramming.
1950 1953 * Add a new deferred-programming entry if the
1951 1954 * caller passed a NULL one (and update the existing one
1952 1955 * in case anything changed).
1953 1956 */
1954 1957 add_defer_repro_ent(irq_ptr, which_irq, new_bind_cpu);
1955 1958 if (drep)
1956 1959 drep->tries++;
1957 1960
1958 1961 /* Inform caller to defer IOAPIC programming: */
1959 1962 return (1);
1960 1963 }
1961 1964
1962 1965 }
1963 1966
1964 1967 /* Remote IRR is clear */
1965 1968 delete_defer_repro_ent(which_irq);
1966 1969
1967 1970 return (0);
1968 1971 }
1969 1972
1970 1973 /*
1971 1974 * Called to migrate all interrupts at an irq to another cpu.
1972 1975 * Must be called with interrupts disabled and apic_ioapic_lock held
1973 1976 */
1974 1977 int
1975 1978 apic_rebind_all(apic_irq_t *irq_ptr, int bind_cpu)
1976 1979 {
1977 1980 apic_irq_t *irqptr = irq_ptr;
1978 1981 int retval = 0;
1979 1982
1980 1983 while (irqptr) {
1981 1984 if (irqptr->airq_temp_cpu != IRQ_UNINIT)
1982 1985 retval |= apic_rebind(irqptr, bind_cpu, NULL);
1983 1986 irqptr = irqptr->airq_next;
1984 1987 }
1985 1988
1986 1989 return (retval);
1987 1990 }
1988 1991
1989 1992 /*
1990 1993 * apic_intr_redistribute does all the messy computations for identifying
1991 1994 * which interrupt to move to which CPU. Currently we do just one interrupt
1992 1995 * at a time. This reduces the time we spent doing all this within clock
1993 1996 * interrupt. When it is done in idle, we could do more than 1.
1994 1997 * First we find the most busy and the most free CPU (time in ISR only)
1995 1998 * skipping those CPUs that has been identified as being ineligible (cpu_skip)
1996 1999 * Then we look for IRQs which are closest to the difference between the
1997 2000 * most busy CPU and the average ISR load. We try to find one whose load
1998 2001 * is less than difference.If none exists, then we chose one larger than the
1999 2002 * difference, provided it does not make the most idle CPU worse than the
2000 2003 * most busy one. In the end, we clear all the busy fields for CPUs. For
2001 2004 * IRQs, they are cleared as they are scanned.
2002 2005 */
2003 2006 void
2004 2007 apic_intr_redistribute(void)
2005 2008 {
2006 2009 int busiest_cpu, most_free_cpu;
2007 2010 int cpu_free, cpu_busy, max_busy, min_busy;
2008 2011 int min_free, diff;
2009 2012 int average_busy, cpus_online;
2010 2013 int i, busy;
2011 2014 ulong_t iflag;
2012 2015 apic_cpus_info_t *cpu_infop;
2013 2016 apic_irq_t *min_busy_irq = NULL;
2014 2017 apic_irq_t *max_busy_irq = NULL;
2015 2018
2016 2019 busiest_cpu = most_free_cpu = -1;
2017 2020 cpu_free = cpu_busy = max_busy = average_busy = 0;
2018 2021 min_free = apic_sample_factor_redistribution;
2019 2022 cpus_online = 0;
2020 2023 /*
2021 2024 * Below we will check for CPU_INTR_ENABLE, bound, temp_bound, temp_cpu
2022 2025 * without ioapic_lock. That is OK as we are just doing statistical
2023 2026 * sampling anyway and any inaccuracy now will get corrected next time
2024 2027 * The call to rebind which actually changes things will make sure
2025 2028 * we are consistent.
2026 2029 */
2027 2030 for (i = 0; i < apic_nproc; i++) {
2028 2031 if (apic_cpu_in_range(i) &&
2029 2032 !(apic_redist_cpu_skip & (1 << i)) &&
2030 2033 (apic_cpus[i].aci_status & APIC_CPU_INTR_ENABLE)) {
2031 2034
2032 2035 cpu_infop = &apic_cpus[i];
2033 2036 /*
2034 2037 * If no unbound interrupts or only 1 total on this
2035 2038 * CPU, skip
2036 2039 */
2037 2040 if (!cpu_infop->aci_temp_bound ||
2038 2041 (cpu_infop->aci_bound + cpu_infop->aci_temp_bound)
2039 2042 == 1) {
2040 2043 apic_redist_cpu_skip |= 1 << i;
2041 2044 continue;
2042 2045 }
2043 2046
2044 2047 busy = cpu_infop->aci_busy;
2045 2048 average_busy += busy;
2046 2049 cpus_online++;
2047 2050 if (max_busy < busy) {
2048 2051 max_busy = busy;
2049 2052 busiest_cpu = i;
2050 2053 }
2051 2054 if (min_free > busy) {
2052 2055 min_free = busy;
2053 2056 most_free_cpu = i;
2054 2057 }
2055 2058 if (busy > apic_int_busy_mark) {
2056 2059 cpu_busy |= 1 << i;
2057 2060 } else {
2058 2061 if (busy < apic_int_free_mark)
2059 2062 cpu_free |= 1 << i;
2060 2063 }
2061 2064 }
2062 2065 }
2063 2066 if ((cpu_busy && cpu_free) ||
2064 2067 (max_busy >= (min_free + apic_diff_for_redistribution))) {
2065 2068
2066 2069 apic_num_imbalance++;
2067 2070 #ifdef DEBUG
2068 2071 if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) {
2069 2072 prom_printf(
2070 2073 "redistribute busy=%x free=%x max=%x min=%x",
2071 2074 cpu_busy, cpu_free, max_busy, min_free);
2072 2075 }
2073 2076 #endif /* DEBUG */
2074 2077
2075 2078
2076 2079 average_busy /= cpus_online;
2077 2080
2078 2081 diff = max_busy - average_busy;
2079 2082 min_busy = max_busy; /* start with the max possible value */
2080 2083 max_busy = 0;
2081 2084 min_busy_irq = max_busy_irq = NULL;
2082 2085 i = apic_min_device_irq;
2083 2086 for (; i <= apic_max_device_irq; i++) {
2084 2087 apic_irq_t *irq_ptr;
2085 2088 /* Change to linked list per CPU ? */
2086 2089 if ((irq_ptr = apic_irq_table[i]) == NULL)
2087 2090 continue;
2088 2091 /* Check for irq_busy & decide which one to move */
2089 2092 /* Also zero them for next round */
2090 2093 if ((irq_ptr->airq_temp_cpu == busiest_cpu) &&
2091 2094 irq_ptr->airq_busy) {
2092 2095 if (irq_ptr->airq_busy < diff) {
2093 2096 /*
2094 2097 * Check for least busy CPU,
2095 2098 * best fit or what ?
2096 2099 */
2097 2100 if (max_busy < irq_ptr->airq_busy) {
2098 2101 /*
2099 2102 * Most busy within the
2100 2103 * required differential
2101 2104 */
2102 2105 max_busy = irq_ptr->airq_busy;
2103 2106 max_busy_irq = irq_ptr;
2104 2107 }
2105 2108 } else {
2106 2109 if (min_busy > irq_ptr->airq_busy) {
2107 2110 /*
2108 2111 * least busy, but more than
2109 2112 * the reqd diff
2110 2113 */
2111 2114 if (min_busy <
2112 2115 (diff + average_busy -
2113 2116 min_free)) {
2114 2117 /*
2115 2118 * Making sure new cpu
2116 2119 * will not end up
2117 2120 * worse
2118 2121 */
2119 2122 min_busy =
2120 2123 irq_ptr->airq_busy;
2121 2124
2122 2125 min_busy_irq = irq_ptr;
2123 2126 }
2124 2127 }
2125 2128 }
2126 2129 }
2127 2130 irq_ptr->airq_busy = 0;
2128 2131 }
2129 2132
2130 2133 if (max_busy_irq != NULL) {
2131 2134 #ifdef DEBUG
2132 2135 if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) {
2133 2136 prom_printf("rebinding %x to %x",
2134 2137 max_busy_irq->airq_vector, most_free_cpu);
2135 2138 }
2136 2139 #endif /* DEBUG */
2137 2140 iflag = intr_clear();
2138 2141 if (lock_try(&apic_ioapic_lock)) {
2139 2142 if (apic_rebind_all(max_busy_irq,
2140 2143 most_free_cpu) == 0) {
2141 2144 /* Make change permenant */
2142 2145 max_busy_irq->airq_cpu =
2143 2146 (uint32_t)most_free_cpu;
2144 2147 }
2145 2148 lock_clear(&apic_ioapic_lock);
2146 2149 }
2147 2150 intr_restore(iflag);
2148 2151
2149 2152 } else if (min_busy_irq != NULL) {
2150 2153 #ifdef DEBUG
2151 2154 if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) {
2152 2155 prom_printf("rebinding %x to %x",
2153 2156 min_busy_irq->airq_vector, most_free_cpu);
2154 2157 }
2155 2158 #endif /* DEBUG */
2156 2159
2157 2160 iflag = intr_clear();
2158 2161 if (lock_try(&apic_ioapic_lock)) {
2159 2162 if (apic_rebind_all(min_busy_irq,
2160 2163 most_free_cpu) == 0) {
2161 2164 /* Make change permenant */
2162 2165 min_busy_irq->airq_cpu =
2163 2166 (uint32_t)most_free_cpu;
2164 2167 }
2165 2168 lock_clear(&apic_ioapic_lock);
2166 2169 }
2167 2170 intr_restore(iflag);
2168 2171
2169 2172 } else {
2170 2173 if (cpu_busy != (1 << busiest_cpu)) {
2171 2174 apic_redist_cpu_skip |= 1 << busiest_cpu;
2172 2175 /*
2173 2176 * We leave cpu_skip set so that next time we
2174 2177 * can choose another cpu
2175 2178 */
2176 2179 }
2177 2180 }
2178 2181 apic_num_rebind++;
2179 2182 } else {
2180 2183 /*
2181 2184 * found nothing. Could be that we skipped over valid CPUs
2182 2185 * or we have balanced everything. If we had a variable
2183 2186 * ticks_for_redistribution, it could be increased here.
2184 2187 * apic_int_busy, int_free etc would also need to be
2185 2188 * changed.
2186 2189 */
2187 2190 if (apic_redist_cpu_skip)
2188 2191 apic_redist_cpu_skip = 0;
2189 2192 }
2190 2193 for (i = 0; i < apic_nproc; i++) {
2191 2194 if (apic_cpu_in_range(i)) {
2192 2195 apic_cpus[i].aci_busy = 0;
2193 2196 }
2194 2197 }
2195 2198 }
2196 2199
2197 2200 void
2198 2201 apic_cleanup_busy(void)
2199 2202 {
2200 2203 int i;
2201 2204 apic_irq_t *irq_ptr;
2202 2205
2203 2206 for (i = 0; i < apic_nproc; i++) {
2204 2207 if (apic_cpu_in_range(i)) {
2205 2208 apic_cpus[i].aci_busy = 0;
2206 2209 }
2207 2210 }
2208 2211
2209 2212 for (i = apic_min_device_irq; i <= apic_max_device_irq; i++) {
2210 2213 if ((irq_ptr = apic_irq_table[i]) != NULL)
2211 2214 irq_ptr->airq_busy = 0;
2212 2215 }
2213 2216 }
|
↓ open down ↓ |
865 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX