1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25 /*
26 * Copyright 2014 Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
27 * Copyright (c) 2014 by Delphix. All rights reserved.
28 */
29
30 #include <sys/cpuvar.h>
31 #include <sys/psm.h>
32 #include <sys/archsystm.h>
33 #include <sys/apic.h>
34 #include <sys/sunddi.h>
35 #include <sys/ddi_impldefs.h>
36 #include <sys/mach_intr.h>
37 #include <sys/sysmacros.h>
38 #include <sys/trap.h>
39 #include <sys/x86_archext.h>
40 #include <sys/privregs.h>
41 #include <sys/psm_common.h>
42
43 /* Function prototypes of local apic and X2APIC */
44 static uint64_t local_apic_read(uint32_t reg);
45 static void local_apic_write(uint32_t reg, uint64_t value);
46 static int get_local_apic_pri(void);
47 static void local_apic_write_task_reg(uint64_t value);
48 static void local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
49 static uint64_t local_x2apic_read(uint32_t msr);
50 static void local_x2apic_write(uint32_t msr, uint64_t value);
51 static int get_local_x2apic_pri(void);
52 static void local_x2apic_write_task_reg(uint64_t value);
53 static void local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
54
55 /*
56 * According to the X2APIC specification:
57 *
58 * xAPIC global enable X2APIC enable Description
59 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10])
60 * -----------------------------------------------------------
61 * 0 0 APIC is disabled
62 * 0 1 Invalid
63 * 1 0 APIC is enabled in xAPIC mode
64 * 1 1 APIC is enabled in X2APIC mode
65 * -----------------------------------------------------------
66 */
67 int x2apic_enable = 1;
68 apic_mode_t apic_mode = LOCAL_APIC; /* Default mode is Local APIC */
69
70 /* See apic_directed_EOI_supported(). Currently 3-state variable. */
71 volatile int apic_directed_eoi_state = 2;
72
73 /* Uses MMIO (Memory Mapped IO) */
74 static apic_reg_ops_t local_apic_regs_ops = {
75 local_apic_read,
76 local_apic_write,
77 get_local_apic_pri,
78 local_apic_write_task_reg,
79 local_apic_write_int_cmd,
80 apic_send_EOI,
81 };
82
83 /* X2APIC : Uses RDMSR/WRMSR instructions to access APIC registers */
84 static apic_reg_ops_t x2apic_regs_ops = {
85 local_x2apic_read,
86 local_x2apic_write,
87 get_local_x2apic_pri,
88 local_x2apic_write_task_reg,
89 local_x2apic_write_int_cmd,
90 apic_send_EOI,
91 };
92
93 int apic_have_32bit_cr8 = 0;
94
95 /* The default ops is local APIC (Memory Mapped IO) */
96 apic_reg_ops_t *apic_reg_ops = &local_apic_regs_ops;
97
98 /*
99 * APIC register ops related data sturctures and functions.
100 */
101 void apic_send_EOI();
102 void apic_send_directed_EOI(uint32_t irq);
103
104 #define X2APIC_ENABLE_BIT 10
105
106 /*
107 * Local APIC Implementation
108 */
109 static uint64_t
110 local_apic_read(uint32_t reg)
111 {
112 return ((uint32_t)apicadr[reg]);
113 }
114
115 static void
116 local_apic_write(uint32_t reg, uint64_t value)
117 {
118 apicadr[reg] = (uint32_t)value;
119 }
120
121 static int
122 get_local_apic_pri(void)
123 {
124 #if defined(__amd64)
125 return ((int)getcr8());
126 #else
127 if (apic_have_32bit_cr8)
128 return ((int)getcr8());
129 return (apicadr[APIC_TASK_REG]);
130 #endif
131 }
132
133 static void
134 local_apic_write_task_reg(uint64_t value)
135 {
136 #if defined(__amd64)
137 setcr8((ulong_t)(value >> APIC_IPL_SHIFT));
138 #else
139 if (apic_have_32bit_cr8)
140 setcr8((ulong_t)(value >> APIC_IPL_SHIFT));
141 else
142 apicadr[APIC_TASK_REG] = (uint32_t)value;
143 #endif
144 }
145
146 static void
147 local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
148 {
149 apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET;
150 apicadr[APIC_INT_CMD1] = cmd1;
151 }
152
153 /*
154 * X2APIC Implementation.
155 */
156 static uint64_t
157 local_x2apic_read(uint32_t msr)
158 {
159 uint64_t i;
160
161 i = (uint64_t)(rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)) & 0xffffffff);
162 return (i);
163 }
164
165 static void
166 local_x2apic_write(uint32_t msr, uint64_t value)
167 {
168 uint64_t tmp;
169
170 if (msr != APIC_EOI_REG) {
171 tmp = rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2));
172 tmp = (tmp & 0xffffffff00000000) | value;
173 } else {
174 tmp = 0;
175 }
176
177 wrmsr((REG_X2APIC_BASE_MSR + (msr >> 2)), tmp);
178 }
179
180 static int
181 get_local_x2apic_pri(void)
182 {
183 return (rdmsr(REG_X2APIC_BASE_MSR + (APIC_TASK_REG >> 2)));
184 }
185
186 static void
187 local_x2apic_write_task_reg(uint64_t value)
188 {
189 X2APIC_WRITE(APIC_TASK_REG, value);
190 }
191
192 static void
193 local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
194 {
195 wrmsr((REG_X2APIC_BASE_MSR + (APIC_INT_CMD1 >> 2)),
196 (((uint64_t)cpu_id << 32) | cmd1));
197 }
198
199 /*ARGSUSED*/
200 void
201 apic_send_EOI(uint32_t irq)
202 {
203 apic_reg_ops->apic_write(APIC_EOI_REG, 0);
204 }
205
206 /*
207 * Support for Directed EOI capability is available in both the xAPIC
208 * and x2APIC mode.
209 */
210 void
211 apic_send_directed_EOI(uint32_t irq)
212 {
213 uchar_t ioapicindex;
214 uchar_t vector;
215 apic_irq_t *apic_irq;
216 short intr_index;
217
218 /*
219 * Following the EOI to the local APIC unit, perform a directed
220 * EOI to the IOxAPIC generating the interrupt by writing to its
221 * EOI register.
222 *
223 * A broadcast EOI is not generated.
224 */
225 apic_reg_ops->apic_write(APIC_EOI_REG, 0);
226
227 apic_irq = apic_irq_table[irq];
228 while (apic_irq) {
229 intr_index = apic_irq->airq_mps_intr_index;
230 if (intr_index == ACPI_INDEX || intr_index >= 0) {
231 ioapicindex = apic_irq->airq_ioapicindex;
232 vector = apic_irq->airq_vector;
233 ioapic_write_eoi(ioapicindex, vector);
234 }
235 apic_irq = apic_irq->airq_next;
236 }
237 }
238
239 int
240 apic_detect_x2apic(void)
241 {
242 if (x2apic_enable == 0)
243 return (0);
244
245 return (is_x86_feature(x86_featureset, X86FSET_X2APIC));
246 }
247
248 void
249 apic_enable_x2apic(void)
250 {
251 uint64_t apic_base_msr;
252
253 if (apic_local_mode() == LOCAL_X2APIC) {
254 /* BIOS apparently has enabled X2APIC */
255 if (apic_mode != LOCAL_X2APIC)
256 x2apic_update_psm();
257 return;
258 }
259
260 /*
261 * This is the first time we are enabling X2APIC on this CPU
262 */
263 apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
264 apic_base_msr = apic_base_msr | (0x1 << X2APIC_ENABLE_BIT);
265 wrmsr(REG_APIC_BASE_MSR, apic_base_msr);
266
267 if (apic_mode != LOCAL_X2APIC)
268 x2apic_update_psm();
269 }
270
271 /*
272 * Determine which mode the current CPU is in. See the table above.
273 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10])
274 */
275 int
276 apic_local_mode(void)
277 {
278 uint64_t apic_base_msr;
279 int bit = ((0x1 << (X2APIC_ENABLE_BIT + 1)) |
280 (0x1 << X2APIC_ENABLE_BIT));
281
282 apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
283
284 if ((apic_base_msr & bit) == bit)
285 return (LOCAL_X2APIC);
286 else
287 return (LOCAL_APIC);
288 }
289
290 void
291 apic_set_directed_EOI_handler()
292 {
293 apic_reg_ops->apic_send_eoi = apic_send_directed_EOI;
294 }
295
296 int
297 apic_directed_EOI_supported()
298 {
299 uint32_t ver;
300
301 /*
302 * There are some known issues with some versions of Linux KVM and QEMU
303 * where by directed EOIs do not properly function and instead get
304 * coalesced at the hypervisor, causing the host not to see interrupts.
305 * Thus, when the platform is KVM, we would like to disable it by
306 * default, but keep it available otherwise.
307 *
308 * We use a three-state variable (apic_directed_eoi_state) to determine
309 * how we handle directed EOI.
310 *
311 * 0 --> Don't do directed EOI at all.
312 * 1 --> Do directed EOI if available, no matter the HW environment.
313 * 2 --> Don't do directed EOI on KVM, but do it otherwise if available.
314 *
315 * If some grinning weirdo put something else in there, treat it as '2'
316 * (i.e. the current default).
317 *
318 * Note, at this time illumos KVM does not identify as KVM. If it does,
319 * we'll need to do some work to determine if it should be caught by
320 * this or if it should show up as its own value of platform_type.
321 */
322 switch (apic_directed_eoi_state) {
323 case 0:
324 /* Don't do it at all. */
325 return (0);
326 case 1:
327 break;
328 case 2:
329 default:
330 /* Only do it if we aren't on KVM. */
331 if (get_hwenv() == HW_KVM)
332 return (0);
333 /* FALLTHRU */
334 }
335
336 ver = apic_reg_ops->apic_read(APIC_VERS_REG);
337 if (ver & APIC_DIRECTED_EOI_BIT)
338 return (1);
339
340 return (0);
341 }
342
343 /*
344 * Change apic_reg_ops depending upon the apic_mode.
345 */
346 void
347 apic_change_ops()
348 {
349 if (apic_mode == LOCAL_APIC)
350 apic_reg_ops = &local_apic_regs_ops;
351 else if (apic_mode == LOCAL_X2APIC)
352 apic_reg_ops = &x2apic_regs_ops;
353 }
354
355 /*
356 * Generates an interprocessor interrupt to another CPU when X2APIC mode is
357 * enabled.
358 */
359 void
360 x2apic_send_ipi(int cpun, int ipl)
361 {
362 int vector;
363 ulong_t flag;
364
365 ASSERT(apic_mode == LOCAL_X2APIC);
366
367 /*
368 * With X2APIC, Intel relaxed the semantics of the
369 * WRMSR instruction such that references to the X2APIC
370 * MSR registers are no longer serializing instructions.
371 * The code that initiates IPIs assumes that some sort
372 * of memory serialization occurs. The old APIC code
373 * did a write to uncachable memory mapped registers.
374 * Any reference to uncached memory is a serializing
375 * operation. To mimic those semantics here, we do an
376 * atomic operation, which translates to a LOCK OR instruction,
377 * which is serializing.
378 */
379 atomic_or_ulong(&flag, 1);
380
381 vector = apic_resv_vector[ipl];
382
383 flag = intr_clear();
384
385 /*
386 * According to X2APIC specification in section '2.3.5.1' of
387 * Interrupt Command Register Semantics, the semantics of
388 * programming Interrupt Command Register to dispatch an interrupt
389 * is simplified. A single MSR write to the 64-bit ICR is required
390 * for dispatching an interrupt. Specifically with the 64-bit MSR
391 * interface to ICR, system software is not required to check the
392 * status of the delivery status bit prior to writing to the ICR
393 * to send an IPI. With the removal of the Delivery Status bit,
394 * system software no longer has a reason to read the ICR. It remains
395 * readable only to aid in debugging.
396 */
397 #ifdef DEBUG
398 APIC_AV_PENDING_SET();
399 #endif /* DEBUG */
400
401 if ((cpun == psm_get_cpu_id())) {
402 X2APIC_WRITE(X2APIC_SELF_IPI, vector);
403 } else {
404 apic_reg_ops->apic_write_int_cmd(
405 apic_cpus[cpun].aci_local_id, vector);
406 }
407
408 intr_restore(flag);
409 }
410
411 /*
412 * Generates IPI to another CPU depending on the local APIC mode.
413 * apic_send_ipi() and x2apic_send_ipi() depends on the configured
414 * mode of the local APIC, but that may not match the actual mode
415 * early in CPU startup.
416 *
417 * Any changes made to this routine must be accompanied by similar
418 * changes to apic_send_ipi().
419 */
420 void
421 apic_common_send_ipi(int cpun, int ipl)
422 {
423 int vector;
424 ulong_t flag;
425 int mode = apic_local_mode();
426
427 if (mode == LOCAL_X2APIC) {
428 x2apic_send_ipi(cpun, ipl);
429 return;
430 }
431
432 ASSERT(mode == LOCAL_APIC);
433
434 vector = apic_resv_vector[ipl];
435 ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
436 flag = intr_clear();
437 while (local_apic_regs_ops.apic_read(APIC_INT_CMD1) & AV_PENDING)
438 apic_ret();
439 local_apic_regs_ops.apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
440 vector);
441 intr_restore(flag);
442 }