1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25 /*
26 * Copyright 2014 Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
27 * Copyright (c) 2014 by Delphix. All rights reserved.
28 * Copyright 2017 Joyent, Inc.
29 */
30
31 #include <sys/cpuvar.h>
32 #include <sys/psm.h>
33 #include <sys/archsystm.h>
34 #include <sys/apic.h>
35 #include <sys/sunddi.h>
36 #include <sys/ddi_impldefs.h>
37 #include <sys/mach_intr.h>
38 #include <sys/sysmacros.h>
39 #include <sys/trap.h>
40 #include <sys/x86_archext.h>
41 #include <sys/privregs.h>
42 #include <sys/psm_common.h>
43
44 /* Function prototypes of X2APIC */
45 static uint64_t local_x2apic_read(uint32_t msr);
46 static void local_x2apic_write(uint32_t msr, uint64_t value);
47 static int get_local_x2apic_pri(void);
48 static void local_x2apic_write_task_reg(uint64_t value);
49 static void local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
50
51 /*
52 * According to the X2APIC specification:
53 *
54 * xAPIC global enable X2APIC enable Description
55 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10])
56 * -----------------------------------------------------------
57 * 0 0 APIC is disabled
58 * 0 1 Invalid
59 * 1 0 APIC is enabled in xAPIC mode
60 * 1 1 APIC is enabled in X2APIC mode
61 * -----------------------------------------------------------
62 */
63 int x2apic_enable = 1;
64
65 /* X2APIC : Uses RDMSR/WRMSR instructions to access APIC registers */
66 static apic_reg_ops_t x2apic_regs_ops = {
67 local_x2apic_read,
68 local_x2apic_write,
69 get_local_x2apic_pri,
70 local_x2apic_write_task_reg,
71 local_x2apic_write_int_cmd,
72 apic_send_EOI,
73 };
74
75 /*
76 * X2APIC Implementation.
77 */
78 static uint64_t
79 local_x2apic_read(uint32_t msr)
80 {
81 uint64_t i;
82
83 i = (uint64_t)(rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)) & 0xffffffff);
84 return (i);
85 }
86
87 static void
88 local_x2apic_write(uint32_t msr, uint64_t value)
89 {
90 uint64_t tmp;
91
92 if (msr != APIC_EOI_REG) {
93 tmp = rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2));
94 tmp = (tmp & 0xffffffff00000000) | value;
95 } else {
96 tmp = 0;
97 }
98
99 wrmsr((REG_X2APIC_BASE_MSR + (msr >> 2)), tmp);
100 }
101
102 static int
103 get_local_x2apic_pri(void)
104 {
105 return (rdmsr(REG_X2APIC_BASE_MSR + (APIC_TASK_REG >> 2)));
106 }
107
108 static void
109 local_x2apic_write_task_reg(uint64_t value)
110 {
111 X2APIC_WRITE(APIC_TASK_REG, value);
112 }
113
114 static void
115 local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
116 {
117 wrmsr((REG_X2APIC_BASE_MSR + (APIC_INT_CMD1 >> 2)),
118 (((uint64_t)cpu_id << 32) | cmd1));
119 }
120
121 int
122 apic_detect_x2apic(void)
123 {
124 if (x2apic_enable == 0)
125 return (0);
126
127 return (is_x86_feature(x86_featureset, X86FSET_X2APIC));
128 }
129
130 void
131 apic_enable_x2apic(void)
132 {
133 uint64_t apic_base_msr;
134
135 if (apic_local_mode() == LOCAL_X2APIC) {
136 /* BIOS apparently has enabled X2APIC */
137 if (apic_mode != LOCAL_X2APIC)
138 x2apic_update_psm();
139 return;
140 }
141
142 /*
143 * This is the first time we are enabling X2APIC on this CPU
144 */
145 apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
146 apic_base_msr = apic_base_msr | (0x1 << X2APIC_ENABLE_BIT);
147 wrmsr(REG_APIC_BASE_MSR, apic_base_msr);
148
149 if (apic_mode != LOCAL_X2APIC)
150 x2apic_update_psm();
151 }
152
153 /*
154 * Change apic_reg_ops depending upon the apic_mode.
155 */
156 void
157 apic_change_ops()
158 {
159 if (apic_mode == LOCAL_APIC)
160 apic_reg_ops = &local_apic_regs_ops;
161 else if (apic_mode == LOCAL_X2APIC)
162 apic_reg_ops = &x2apic_regs_ops;
163 }
164
165 /*
166 * Generates an interprocessor interrupt to another CPU when X2APIC mode is
167 * enabled.
168 */
169 void
170 x2apic_send_ipi(int cpun, int ipl)
171 {
172 int vector;
173 ulong_t flag;
174
175 ASSERT(apic_mode == LOCAL_X2APIC);
176
177 /*
178 * With X2APIC, Intel relaxed the semantics of the
179 * WRMSR instruction such that references to the X2APIC
180 * MSR registers are no longer serializing instructions.
181 * The code that initiates IPIs assumes that some sort
182 * of memory serialization occurs. The old APIC code
183 * did a write to uncachable memory mapped registers.
184 * Any reference to uncached memory is a serializing
185 * operation. To mimic those semantics here, we do an
186 * atomic operation, which translates to a LOCK OR instruction,
187 * which is serializing.
188 */
189 atomic_or_ulong(&flag, 1);
190
191 vector = apic_resv_vector[ipl];
192
193 flag = intr_clear();
194
195 /*
196 * According to X2APIC specification in section '2.3.5.1' of
197 * Interrupt Command Register Semantics, the semantics of
198 * programming Interrupt Command Register to dispatch an interrupt
199 * is simplified. A single MSR write to the 64-bit ICR is required
200 * for dispatching an interrupt. Specifically with the 64-bit MSR
201 * interface to ICR, system software is not required to check the
202 * status of the delivery status bit prior to writing to the ICR
203 * to send an IPI. With the removal of the Delivery Status bit,
204 * system software no longer has a reason to read the ICR. It remains
205 * readable only to aid in debugging.
206 */
207 #ifdef DEBUG
208 APIC_AV_PENDING_SET();
209 #endif /* DEBUG */
210
211 if ((cpun == psm_get_cpu_id())) {
212 X2APIC_WRITE(X2APIC_SELF_IPI, vector);
213 } else {
214 apic_reg_ops->apic_write_int_cmd(
215 apic_cpus[cpun].aci_local_id, vector);
216 }
217
218 intr_restore(flag);
219 }
220
221 /*
222 * Generates IPI to another CPU depending on the local APIC mode.
223 * apic_send_ipi() and x2apic_send_ipi() depends on the configured
224 * mode of the local APIC, but that may not match the actual mode
225 * early in CPU startup.
226 *
227 * Any changes made to this routine must be accompanied by similar
228 * changes to apic_send_ipi().
229 */
230 void
231 apic_common_send_ipi(int cpun, int ipl)
232 {
233 int vector;
234 ulong_t flag;
235 int mode = apic_local_mode();
236
237 if (mode == LOCAL_X2APIC) {
238 x2apic_send_ipi(cpun, ipl);
239 return;
240 }
241
242 ASSERT(mode == LOCAL_APIC);
243
244 vector = apic_resv_vector[ipl];
245 ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
246 flag = intr_clear();
247 while (local_apic_regs_ops.apic_read(APIC_INT_CMD1) & AV_PENDING)
248 apic_ret();
249 local_apic_regs_ops.apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
250 vector);
251 intr_restore(flag);
252 }