91 static void apic_init_intr(void);
92
93 /*
94 * standard MP entries
95 */
96 static int apic_probe(void);
97 static int apic_getclkirq(int ipl);
98 static void apic_init(void);
99 static void apic_picinit(void);
100 static int apic_post_cpu_start(void);
101 static int apic_intr_enter(int ipl, int *vect);
102 static void apic_setspl(int ipl);
103 static void x2apic_setspl(int ipl);
104 static int apic_addspl(int ipl, int vector, int min_ipl, int max_ipl);
105 static int apic_delspl(int ipl, int vector, int min_ipl, int max_ipl);
106 static int apic_disable_intr(processorid_t cpun);
107 static void apic_enable_intr(processorid_t cpun);
108 static int apic_get_ipivect(int ipl, int type);
109 static void apic_post_cyclic_setup(void *arg);
110
111 /*
112 * The following vector assignments influence the value of ipltopri and
113 * vectortoipl. Note that vectors 0 - 0x1f are not used. We can program
114 * idle to 0 and IPL 0 to 0xf to differentiate idle in case
115 * we care to do so in future. Note some IPLs which are rarely used
116 * will share the vector ranges and heavily used IPLs (5 and 6) have
117 * a wide range.
118 *
119 * This array is used to initialize apic_ipls[] (in apic_init()).
120 *
121 * IPL Vector range. as passed to intr_enter
122 * 0 none.
123 * 1,2,3 0x20-0x2f 0x0-0xf
124 * 4 0x30-0x3f 0x10-0x1f
125 * 5 0x40-0x5f 0x20-0x3f
126 * 6 0x60-0x7f 0x40-0x5f
127 * 7,8,9 0x80-0x8f 0x60-0x6f
128 * 10 0x90-0x9f 0x70-0x7f
129 * 11 0xa0-0xaf 0x80-0x8f
130 * ... ...
1051 avail = 0;
1052 for (i = APIC_FIRST_FREE_IRQ; i < APIC_RESV_IRQ; i++) {
1053 if ((apic_irq_table[i] == NULL) ||
1054 apic_irq_table[i]->airq_mps_intr_index == FREE_INDEX) {
1055 if (++avail >= count)
1056 return (PSM_SUCCESS);
1057 }
1058 }
1059 return (PSM_FAILURE);
1060 }
1061
1062 /*
1063 * This function allocates "count" MSI vector(s) for the given "dip/pri/type"
1064 */
1065 int
1066 apic_alloc_msi_vectors(dev_info_t *dip, int inum, int count, int pri,
1067 int behavior)
1068 {
1069 int rcount, i;
1070 uchar_t start, irqno;
1071 uint32_t cpu;
1072 major_t major;
1073 apic_irq_t *irqptr;
1074
1075 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msi_vectors: dip=0x%p "
1076 "inum=0x%x pri=0x%x count=0x%x behavior=%d\n",
1077 (void *)dip, inum, pri, count, behavior));
1078
1079 if (count > 1) {
1080 if (behavior == DDI_INTR_ALLOC_STRICT &&
1081 apic_multi_msi_enable == 0)
1082 return (0);
1083 if (apic_multi_msi_enable == 0)
1084 count = 1;
1085 }
1086
1087 if ((rcount = apic_navail_vector(dip, pri)) > count)
1088 rcount = count;
1089 else if (rcount == 0 || (rcount < count &&
1090 behavior == DDI_INTR_ALLOC_STRICT))
1091 return (0);
1123 * apic_check_free_irqs() check earlier
1124 */
1125 mutex_exit(&airq_mutex);
1126 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msi_vectors: "
1127 "apic_allocate_irq failed\n"));
1128 return (i);
1129 }
1130 apic_max_device_irq = max(irqno, apic_max_device_irq);
1131 apic_min_device_irq = min(irqno, apic_min_device_irq);
1132 irqptr = apic_irq_table[irqno];
1133 #ifdef DEBUG
1134 if (apic_vector_to_irq[start + i] != APIC_RESV_IRQ)
1135 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msi_vectors: "
1136 "apic_vector_to_irq is not APIC_RESV_IRQ\n"));
1137 #endif
1138 apic_vector_to_irq[start + i] = (uchar_t)irqno;
1139
1140 irqptr->airq_vector = (uchar_t)(start + i);
1141 irqptr->airq_ioapicindex = (uchar_t)inum; /* start */
1142 irqptr->airq_intin_no = (uchar_t)rcount;
1143 irqptr->airq_ipl = pri;
1144 irqptr->airq_vector = start + i;
1145 irqptr->airq_origirq = (uchar_t)(inum + i);
1146 irqptr->airq_share_id = 0;
1147 irqptr->airq_mps_intr_index = MSI_INDEX;
1148 irqptr->airq_dip = dip;
1149 irqptr->airq_major = major;
1150 if (i == 0) /* they all bound to the same cpu */
1151 cpu = irqptr->airq_cpu = apic_bind_intr(dip, irqno,
1152 0xff, 0xff);
1153 else
1154 irqptr->airq_cpu = cpu;
1155 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msi_vectors: irq=0x%x "
1156 "dip=0x%p vector=0x%x origirq=0x%x pri=0x%x\n", irqno,
1157 (void *)irqptr->airq_dip, irqptr->airq_vector,
1158 irqptr->airq_origirq, pri));
1159 }
1160 mutex_exit(&airq_mutex);
1161 return (rcount);
1162 }
1163
1200 */
1201 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msix_vectors: "
1202 "apic_allocate_irq failed\n"));
1203 rcount = i;
1204 goto out;
1205 }
1206 if ((vector = apic_allocate_vector(pri, irqno, 1)) == 0) {
1207 /*
1208 * shouldn't happen because of the
1209 * apic_navail_vector() call earlier
1210 */
1211 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msix_vectors: "
1212 "apic_allocate_vector failed\n"));
1213 rcount = i;
1214 goto out;
1215 }
1216 apic_max_device_irq = max(irqno, apic_max_device_irq);
1217 apic_min_device_irq = min(irqno, apic_min_device_irq);
1218 irqptr = apic_irq_table[irqno];
1219 irqptr->airq_vector = (uchar_t)vector;
1220 irqptr->airq_ipl = pri;
1221 irqptr->airq_origirq = (uchar_t)(inum + i);
1222 irqptr->airq_share_id = 0;
1223 irqptr->airq_mps_intr_index = MSIX_INDEX;
1224 irqptr->airq_dip = dip;
1225 irqptr->airq_major = major;
1226 irqptr->airq_cpu = apic_bind_intr(dip, irqno, 0xff, 0xff);
1227 }
1228 out:
1229 mutex_exit(&airq_mutex);
1230 return (rcount);
1231 }
1232
1233 /*
1234 * Allocate a free vector for irq at ipl. Takes care of merging of multiple
1235 * IPLs into a single APIC level as well as stretching some IPLs onto multiple
1236 * levels. APIC_HI_PRI_VECTS interrupts are reserved for high priority
1237 * requests and allocated only when pri is set.
1238 */
1239 uchar_t
1240 apic_allocate_vector(int ipl, int irq, int pri)
1242 int lowest, highest, i;
1243
1244 highest = apic_ipltopri[ipl] + APIC_VECTOR_MASK;
1245 lowest = apic_ipltopri[ipl - 1] + APIC_VECTOR_PER_IPL;
1246
1247 if (highest < lowest) /* Both ipl and ipl - 1 map to same pri */
1248 lowest -= APIC_VECTOR_PER_IPL;
1249
1250 #ifdef DEBUG
1251 if (apic_restrict_vector) /* for testing shared interrupt logic */
1252 highest = lowest + apic_restrict_vector + APIC_HI_PRI_VECTS;
1253 #endif /* DEBUG */
1254 if (pri == 0)
1255 highest -= APIC_HI_PRI_VECTS;
1256
1257 for (i = lowest; i <= highest; i++) {
1258 if (APIC_CHECK_RESERVE_VECTORS(i))
1259 continue;
1260 if (apic_vector_to_irq[i] == APIC_RESV_IRQ) {
1261 apic_vector_to_irq[i] = (uchar_t)irq;
1262 return (i);
1263 }
1264 }
1265
1266 return (0);
1267 }
1268
1269 /* Mark vector as not being used by any irq */
1270 void
1271 apic_free_vector(uchar_t vector)
1272 {
1273 apic_vector_to_irq[vector] = APIC_RESV_IRQ;
1274 }
1275
1276 /*
1277 * Call rebind to do the actual programming.
1278 * Must be called with interrupts disabled and apic_ioapic_lock held
1279 * 'p' is polymorphic -- if this function is called to process a deferred
1280 * reprogramming, p is of type 'struct ioapic_reprogram_data *', from which
1281 * the irq pointer is retrieved. If not doing deferred reprogramming,
1282 * p is of the type 'apic_irq_t *'.
|
91 static void apic_init_intr(void);
92
93 /*
94 * standard MP entries
95 */
96 static int apic_probe(void);
97 static int apic_getclkirq(int ipl);
98 static void apic_init(void);
99 static void apic_picinit(void);
100 static int apic_post_cpu_start(void);
101 static int apic_intr_enter(int ipl, int *vect);
102 static void apic_setspl(int ipl);
103 static void x2apic_setspl(int ipl);
104 static int apic_addspl(int ipl, int vector, int min_ipl, int max_ipl);
105 static int apic_delspl(int ipl, int vector, int min_ipl, int max_ipl);
106 static int apic_disable_intr(processorid_t cpun);
107 static void apic_enable_intr(processorid_t cpun);
108 static int apic_get_ipivect(int ipl, int type);
109 static void apic_post_cyclic_setup(void *arg);
110
111 #define UCHAR_MAX UINT8_MAX
112
113 /*
114 * The following vector assignments influence the value of ipltopri and
115 * vectortoipl. Note that vectors 0 - 0x1f are not used. We can program
116 * idle to 0 and IPL 0 to 0xf to differentiate idle in case
117 * we care to do so in future. Note some IPLs which are rarely used
118 * will share the vector ranges and heavily used IPLs (5 and 6) have
119 * a wide range.
120 *
121 * This array is used to initialize apic_ipls[] (in apic_init()).
122 *
123 * IPL Vector range. as passed to intr_enter
124 * 0 none.
125 * 1,2,3 0x20-0x2f 0x0-0xf
126 * 4 0x30-0x3f 0x10-0x1f
127 * 5 0x40-0x5f 0x20-0x3f
128 * 6 0x60-0x7f 0x40-0x5f
129 * 7,8,9 0x80-0x8f 0x60-0x6f
130 * 10 0x90-0x9f 0x70-0x7f
131 * 11 0xa0-0xaf 0x80-0x8f
132 * ... ...
1053 avail = 0;
1054 for (i = APIC_FIRST_FREE_IRQ; i < APIC_RESV_IRQ; i++) {
1055 if ((apic_irq_table[i] == NULL) ||
1056 apic_irq_table[i]->airq_mps_intr_index == FREE_INDEX) {
1057 if (++avail >= count)
1058 return (PSM_SUCCESS);
1059 }
1060 }
1061 return (PSM_FAILURE);
1062 }
1063
1064 /*
1065 * This function allocates "count" MSI vector(s) for the given "dip/pri/type"
1066 */
1067 int
1068 apic_alloc_msi_vectors(dev_info_t *dip, int inum, int count, int pri,
1069 int behavior)
1070 {
1071 int rcount, i;
1072 uchar_t start, irqno;
1073 uint32_t cpu = 0;
1074 major_t major;
1075 apic_irq_t *irqptr;
1076
1077 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msi_vectors: dip=0x%p "
1078 "inum=0x%x pri=0x%x count=0x%x behavior=%d\n",
1079 (void *)dip, inum, pri, count, behavior));
1080
1081 if (count > 1) {
1082 if (behavior == DDI_INTR_ALLOC_STRICT &&
1083 apic_multi_msi_enable == 0)
1084 return (0);
1085 if (apic_multi_msi_enable == 0)
1086 count = 1;
1087 }
1088
1089 if ((rcount = apic_navail_vector(dip, pri)) > count)
1090 rcount = count;
1091 else if (rcount == 0 || (rcount < count &&
1092 behavior == DDI_INTR_ALLOC_STRICT))
1093 return (0);
1125 * apic_check_free_irqs() check earlier
1126 */
1127 mutex_exit(&airq_mutex);
1128 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msi_vectors: "
1129 "apic_allocate_irq failed\n"));
1130 return (i);
1131 }
1132 apic_max_device_irq = max(irqno, apic_max_device_irq);
1133 apic_min_device_irq = min(irqno, apic_min_device_irq);
1134 irqptr = apic_irq_table[irqno];
1135 #ifdef DEBUG
1136 if (apic_vector_to_irq[start + i] != APIC_RESV_IRQ)
1137 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msi_vectors: "
1138 "apic_vector_to_irq is not APIC_RESV_IRQ\n"));
1139 #endif
1140 apic_vector_to_irq[start + i] = (uchar_t)irqno;
1141
1142 irqptr->airq_vector = (uchar_t)(start + i);
1143 irqptr->airq_ioapicindex = (uchar_t)inum; /* start */
1144 irqptr->airq_intin_no = (uchar_t)rcount;
1145 ASSERT(pri >= 0 && pri <= UCHAR_MAX);
1146 irqptr->airq_ipl = (uchar_t)pri;
1147 irqptr->airq_vector = start + i;
1148 irqptr->airq_origirq = (uchar_t)(inum + i);
1149 irqptr->airq_share_id = 0;
1150 irqptr->airq_mps_intr_index = MSI_INDEX;
1151 irqptr->airq_dip = dip;
1152 irqptr->airq_major = major;
1153 if (i == 0) /* they all bound to the same cpu */
1154 cpu = irqptr->airq_cpu = apic_bind_intr(dip, irqno,
1155 0xff, 0xff);
1156 else
1157 irqptr->airq_cpu = cpu;
1158 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msi_vectors: irq=0x%x "
1159 "dip=0x%p vector=0x%x origirq=0x%x pri=0x%x\n", irqno,
1160 (void *)irqptr->airq_dip, irqptr->airq_vector,
1161 irqptr->airq_origirq, pri));
1162 }
1163 mutex_exit(&airq_mutex);
1164 return (rcount);
1165 }
1166
1203 */
1204 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msix_vectors: "
1205 "apic_allocate_irq failed\n"));
1206 rcount = i;
1207 goto out;
1208 }
1209 if ((vector = apic_allocate_vector(pri, irqno, 1)) == 0) {
1210 /*
1211 * shouldn't happen because of the
1212 * apic_navail_vector() call earlier
1213 */
1214 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msix_vectors: "
1215 "apic_allocate_vector failed\n"));
1216 rcount = i;
1217 goto out;
1218 }
1219 apic_max_device_irq = max(irqno, apic_max_device_irq);
1220 apic_min_device_irq = min(irqno, apic_min_device_irq);
1221 irqptr = apic_irq_table[irqno];
1222 irqptr->airq_vector = (uchar_t)vector;
1223 ASSERT(pri >= 0 && pri <= UCHAR_MAX);
1224 irqptr->airq_ipl = (uchar_t)pri;
1225 irqptr->airq_origirq = (uchar_t)(inum + i);
1226 irqptr->airq_share_id = 0;
1227 irqptr->airq_mps_intr_index = MSIX_INDEX;
1228 irqptr->airq_dip = dip;
1229 irqptr->airq_major = major;
1230 irqptr->airq_cpu = apic_bind_intr(dip, irqno, 0xff, 0xff);
1231 }
1232 out:
1233 mutex_exit(&airq_mutex);
1234 return (rcount);
1235 }
1236
1237 /*
1238 * Allocate a free vector for irq at ipl. Takes care of merging of multiple
1239 * IPLs into a single APIC level as well as stretching some IPLs onto multiple
1240 * levels. APIC_HI_PRI_VECTS interrupts are reserved for high priority
1241 * requests and allocated only when pri is set.
1242 */
1243 uchar_t
1244 apic_allocate_vector(int ipl, int irq, int pri)
1246 int lowest, highest, i;
1247
1248 highest = apic_ipltopri[ipl] + APIC_VECTOR_MASK;
1249 lowest = apic_ipltopri[ipl - 1] + APIC_VECTOR_PER_IPL;
1250
1251 if (highest < lowest) /* Both ipl and ipl - 1 map to same pri */
1252 lowest -= APIC_VECTOR_PER_IPL;
1253
1254 #ifdef DEBUG
1255 if (apic_restrict_vector) /* for testing shared interrupt logic */
1256 highest = lowest + apic_restrict_vector + APIC_HI_PRI_VECTS;
1257 #endif /* DEBUG */
1258 if (pri == 0)
1259 highest -= APIC_HI_PRI_VECTS;
1260
1261 for (i = lowest; i <= highest; i++) {
1262 if (APIC_CHECK_RESERVE_VECTORS(i))
1263 continue;
1264 if (apic_vector_to_irq[i] == APIC_RESV_IRQ) {
1265 apic_vector_to_irq[i] = (uchar_t)irq;
1266 ASSERT(i >= 0 && i <= UCHAR_MAX);
1267 return ((uchar_t)i);
1268 }
1269 }
1270
1271 return (0);
1272 }
1273
1274 /* Mark vector as not being used by any irq */
1275 void
1276 apic_free_vector(uchar_t vector)
1277 {
1278 apic_vector_to_irq[vector] = APIC_RESV_IRQ;
1279 }
1280
1281 /*
1282 * Call rebind to do the actual programming.
1283 * Must be called with interrupts disabled and apic_ioapic_lock held
1284 * 'p' is polymorphic -- if this function is called to process a deferred
1285 * reprogramming, p is of type 'struct ioapic_reprogram_data *', from which
1286 * the irq pointer is retrieved. If not doing deferred reprogramming,
1287 * p is of the type 'apic_irq_t *'.
|